MyoaEngineering
/
ShoppingCrawler
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

[PPN-211113] Implement spring config client

This commit is contained in:
woo-jin.shin 2021-11-17 09:39:49 +09:00
parent 0afa52aa53
commit 5b4b44f093
22 changed files with 332 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
build.gradle
View File

@ -32,6 +32,17 @@ allprojects {
}
}
ext {
set('springCloudVersion', "2020.0.4")
}
dependencyManagement {
imports {
mavenBom "org.springframework.cloud:spring-cloud-dependencies:${springCloudVersion}"
}
}
/* task initSourceFolders {
sourceSets*.java.srcDirs*.each {
if( !it.exists() ) {

3
processor/build.gradle
View File

@ -1,3 +1,4 @@
dependencies {
developmentOnly 'org.springframework.boot:spring-boot-devtools'
runtimeOnly 'com.h2database:h2'
@ -9,6 +10,8 @@ dependencies {
implementation 'org.springframework.boot:spring-boot-starter-webflux'
implementation 'org.springframework.boot:spring-boot-starter-data-jpa'
implementation 'org.springframework.boot:spring-boot-starter-jdbc'
implementation 'org.springframework.cloud:spring-cloud-starter-config'
implementation 'org.springframework.boot:spring-boot-starter-actuator'
implementation 'com.rometools:rome:1.16.0'
implementation 'org.jsoup:jsoup:1.14.2'
implementation 'com.h2database:h2:1.4.200'

6
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/client/PpomppuBoardFeedRetriever.java
View File

@ -1,9 +1,11 @@
package com.myoa.engineering.crawl.ppomppu.processor.infrastructure.client;
import com.myoa.engineering.crawl.ppomppu.processor.configuration.factory.WebClientFilterFactory;
import com.myoa.engineering.crawl.ppomppu.processor.configuration.factory.WebFluxExchangeStragiesFactory;
import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebClientFilterFactory;
import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebFluxExchangeStragiesFactory;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import lombok.extern.slf4j.Slf4j;
import org.springframework.core.ParameterizedTypeReference;
import org.springframework.stereotype.Component;
import org.springframework.web.reactive.function.client.WebClient;
import org.springframework.web.reactive.function.client.WebClientRequestException;

6
processor/src/main/resources/application-development.yml
View File

@ -3,6 +3,12 @@ spring:
activate:
on-profile: development
import:
- "configserver:http://192.168.0.100:11080"
- classpath:/development/webclient.yml
- classpath:/development/temppassword.yml
- classpath:/development/database.yml
profiles:
group:
development: development, datasource-development
# import: optional:configserver:http://localhost:11080 # can be start up even config server was not found.

8
processor/src/main/resources/application.yml
View File

@ -1,4 +1,6 @@
spring:
application:
name: ppn-processor
main:
allow-bean-definition-overriding: true
profiles:
@ -12,3 +14,9 @@ server:
error:
whitelabel:
enabled: false
management:
endpoints:
web:
exposure:
include: refresh

2
receiver/src/main/resources/application.yml
View File

@ -1,4 +1,6 @@
spring:
application:
name: ppn-receiver
main:
allow-bean-definition-overriding: true
profiles:

23
sender/build.gradle Normal file
View File

@ -0,0 +1,23 @@
dependencies {
developmentOnly 'org.springframework.boot:spring-boot-devtools'
runtimeOnly 'mysql:mysql-connector-java'
compileOnly 'org.projectlombok:lombok'
implementation project(':support')
// https://projectreactor.io/docs/core/release/reference/#debug-activate
implementation 'org.springframework.boot:spring-boot-starter-webflux'
implementation 'org.springframework.boot:spring-boot-starter-actuator'
implementation 'org.springframework.cloud:spring-cloud-starter-config'
annotationProcessor 'org.springframework.boot:spring-boot-configuration-processor'
annotationProcessor 'org.projectlombok:lombok'
testImplementation 'org.springframework.boot:spring-boot-starter-test'
}
test {
useJUnitPlatform()
testLogging {
events "passed", "skipped", "failed"
}
}

8
sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/SenderApplication.java
View File

@ -1,10 +1,18 @@
package com.myoa.engineering.crawl.ppomppu.sender;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
/**
* SenderApplication
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-08-20
*
*/
@SpringBootApplication
public class SenderApplication {
public static void main(String[] args) {
SpringApplication.run(SenderApplication.class, args);
}
}

37
sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/controller/TestAPIController.java Normal file
View File

@ -0,0 +1,37 @@
package com.myoa.engineering.crawl.ppomppu.sender.controller;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import com.myoa.engineering.crawl.ppomppu.sender.dto.SlackMessageDTO;
import com.myoa.engineering.crawl.ppomppu.sender.infrastructure.client.SlackMessageSender;
import reactor.core.publisher.Mono;
/**
* TestAPIController
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-11-15
*
*/
@RestController
@RequestMapping("/api/v1")
public class TestAPIController {
private final SlackMessageSender sender;
public TestAPIController() {
this.sender = new SlackMessageSender("xoxb-2688454277126-2695026012277-K2Ib13lKokmTiBSnSMrc0Bp2");
}
@GetMapping("/test")
public Mono<String> test() {
return sender.sendMessage(SlackMessageDTO.builder()
.text("test!")
.iconEmoji("monge")
.channel("notify_shopping")
.username("shopping notifier")
.build());
}
}

12
sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/dto/MessageDTO.java Normal file
View File

@ -0,0 +1,12 @@
package com.myoa.engineering.crawl.ppomppu.sender.dto;
import java.io.Serializable;
/**
* MessageDTO
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-11-14
*
*/
public interface MessageDTO extends Serializable {
}

36
sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/dto/SlackMessageDTO.java Normal file
View File

@ -0,0 +1,36 @@
package com.myoa.engineering.crawl.ppomppu.sender.dto;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.Builder;
import lombok.Getter;
import lombok.NoArgsConstructor;
/**
* SlackMessageDTO
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-11-14
*
*/
@Getter
@NoArgsConstructor
public class SlackMessageDTO implements MessageDTO {
private final static long serialVersionUID = 4737608709660494713L;
private String text;
private String channel;
private String username;
@JsonProperty("icon_emoji")
private String iconEmoji;
@Builder
public SlackMessageDTO(String text, String channel, String username, String iconEmoji) {
this.text = text;
this.channel = channel;
this.username = username;
this.iconEmoji = iconEmoji;
}
}

17
sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/infrastructure/client/MessageSender.java Normal file
View File

@ -0,0 +1,17 @@
package com.myoa.engineering.crawl.ppomppu.sender.infrastructure.client;
import com.myoa.engineering.crawl.ppomppu.sender.dto.MessageDTO;
import reactor.core.publisher.Mono;
/**
* MessageSender
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-11-14
*
*/
public interface MessageSender<T extends MessageDTO> {
Mono<String> sendMessage(T message);
}

88
sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/infrastructure/client/SlackMessageSender.java Normal file
View File

@ -0,0 +1,88 @@
package com.myoa.engineering.crawl.ppomppu.sender.infrastructure.client;
import org.springframework.http.HttpHeaders;
import org.springframework.http.MediaType;
import org.springframework.web.reactive.function.client.WebClient;
import org.springframework.web.reactive.function.client.WebClientRequestException;
import com.myoa.engineering.crawl.ppomppu.sender.dto.SlackMessageDTO;
import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebClientFilterFactory;
import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebFluxExchangeStragiesFactory;
import lombok.extern.slf4j.Slf4j;
import reactor.core.publisher.Mono;
import reactor.core.scheduler.Schedulers;
/**
* SlackMessageSender
*
* @author Shin Woo-jin (woozu.shin@kakaoent.com)
* @since 2021-09-08
*/
@Slf4j
public class SlackMessageSender implements MessageSender<SlackMessageDTO> {
private static final String SLACK_API_URL = "https://slack.com/api";
private final WebClient webClient;
private final String apiSecret;
public SlackMessageSender(String apiSecret) {
this.webClient = WebClient.builder()
.baseUrl(SLACK_API_URL)
.defaultHeader(HttpHeaders.AUTHORIZATION, "Bearer " + apiSecret)
.exchangeStrategies(WebFluxExchangeStragiesFactory.ofDefault())
.defaultHeader(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_UTF8_VALUE)
.defaultHeader(HttpHeaders.ACCEPT_CHARSET, "UTF-8")
.filter(WebClientFilterFactory.logRequest())
.filter(WebClientFilterFactory.logResponse())
.build();
this.apiSecret = apiSecret;
}
@Override
public Mono<String> sendMessage(SlackMessageDTO message) {
return webClient.post()
.uri("/chat.postMessage")
.bodyValue(message)
.exchangeToMono(e -> e.bodyToMono(String.class))
.publishOn(Schedulers.boundedElastic())
.onErrorResume(WebClientRequestException.class, t -> {
log.info("Exception occured, ignoring. : {}", t.getClass().getSimpleName());
return Mono.empty();
})
.doOnNext(e -> log.info("response: {}", e));
}
}
/*
import requests
import json
# woozuyeni_watchtower
SLACK_TOKEN="xoxb-2688454277126-2695026012277-K2Ib13lKokmTiBSnSMrc0Bp2"
CHANNEL = "notify_watchtower"
CHANNEL = "common"
POST_API = "https://slack.com/api/chat.postMessage"
HEADER = "Authorization: Bearer"
headers = {
"Content-Type" : "application/json",
"Authorization" : f"Bearer {SLACK_TOKEN}"
}
data = {}
data["text"] = "몽몽! :monge_big:"
data["channel"] = CHANNEL
data["username"] = "몽이봇"
data["icon_emoji"] = ":monge_big:"
# requests.post(POST_API, data=json.dumps(data), headers=headers)
attachments = [{"title": "Cat", "image_url": image_url}]
image_url = "https://myoa-universe.com/assets/love1.png"
*/

11
sender/src/main/resources/application-development.yml Normal file
View File

@ -0,0 +1,11 @@
spring:
config:
activate:
on-profile: development
import:
- classpath:/development/webclient.yml
- "configserver:http://192.168.0.100:11080"
# import: optional:configserver:http://localhost:11080 # can be start up even config server was not found.
profiles:
group:
development: development, slackapi-development

6
sender/src/main/resources/application-production.yml Normal file
View File

@ -0,0 +1,6 @@
spring:
config:
activate:
on-profile: production
# import:
# - classpath:/production/webclient.yml

19
sender/src/main/resources/application.yml
View File

@ -0,0 +1,19 @@
spring:
application:
# name: ppn-sender
name: app1
main:
allow-bean-definition-overriding: true
profiles:
active: development
freemarker:
enabled: false
server:
port: 20080
management:
endpoints:
web:
exposure:
include: refresh

5
sender/src/main/resources/development/webclient.yml Normal file
View File

@ -0,0 +1,5 @@
webclient:
base-url: http://localhost:20081
units:
- unit-name: processor-api
base-url: http://localhost:20081

1
support/build.gradle
View File

@ -3,6 +3,7 @@ dependencies {
compileOnly 'org.projectlombok:lombok'
// https://projectreactor.io/docs/core/release/reference/#debug-activate
implementation 'org.springframework.boot:spring-boot-starter-webflux'
annotationProcessor 'org.springframework.boot:spring-boot-configuration-processor'
annotationProcessor 'org.projectlombok:lombok'

52
support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/dto/code/PpomppuBoardName.java
View File

@ -12,40 +12,40 @@ import lombok.Getter;
@Getter
@AllArgsConstructor
public enum PpomppuBoardName {
PPOMPPU_DOMESTIC_ALL("zboard/zboard.php?id=ppomppu", "전체", true),
PPOMPPU_DOMESTIC_ETC("zboard/zboard.php?id=ppomppu&category=1", "기타", true),
PPOMPPU_DOMESTIC_COMPUTER("zboard/zboard.php?id=ppomppu&category=4", "컴퓨터", true),
PPOMPPU_DOMESTIC_DIGITAL("zboard/zboard.php?id=ppomppu&category=5", "디지털", true),
PPOMPPU_DOMESTIC_FOOD("zboard/zboard.php?id=ppomppu&category=6", "식품/건강", true),
PPOMPPU_DOMESTIC_BOOK("zboard/zboard.php?id=ppomppu&category=8", "서적", true),
PPOMPPU_DOMESTIC_APPLIANCES("zboard/zboard.php?id=ppomppu&category=9", "가전/가구", true),
PPOMPPU_DOMESTIC_PARENTING("zboard/zboard.php?id=ppomppu&category=10", "육아", true),
PPOMPPU_DOMESTIC_GIFTCARD("zboard/zboard.php?id=ppomppu&category=11", "상품권", true),
PPOMPPU_DOMESTIC_CLOTHES("zboard/zboard.php?id=ppomppu&category=12", "의류/잡화", true),
PPOMPPU_DOMESTIC_COSMETIC("zboard/zboard.php?id=ppomppu&category=13", "화장품", true),
PPOMPPU_DOMESTIC_OUTDOOR("zboard/zboard.php?id=ppomppu&category=15", "등산/캠핑", true),
PPOMPPU_OVERSEA_ALL("zboard/zboard.php?id=ppomppu4", "전체", true),
PPOMPPU_OVERSEA_ETC("zboard/zboard.php?id=ppomppu4&category=1", "기타", true),
PPOMPPU_OVERSEA_APPLIANCES("zboard/zboard.php?id=ppomppu4&category=7", "가전", true),
PPOMPPU_OVERSEA_TVAV("zboard/zboard.php?id=ppomppu4&category=8", "TV/영상", true),
PPOMPPU_OVERSEA_COMPUTER("zboard/zboard.php?id=ppomppu4&category=3", "컴퓨터", true),
PPOMPPU_OVERSEA_DIGITAL("zboard/zboard.php?id=ppomppu4&category=4", "디지털", true),
PPOMPPU_OVERSEA_MOBILEACCESSORY("zboard/zboard.php?id=ppomppu4&category=9", "액세서리", true),
PPOMPPU_OVERSEA_CLOTHES("zboard/zboard.php?id=ppomppu4&category=5", "의류/잡화", true),
PPOMPPU_OVERSEA_WATCH("zboard/zboard.php?id=ppomppu4&category=2", "시계", true),
PPOMPPU_OVERSEA_SHOES("zboard/zboard.php?id=ppomppu4&category=11", "신발", true),
PPOMPPU_OVERSEA_FOOD("zboard/zboard.php?id=ppomppu4&category=10", "식품/건강", true),
PPOMPPU_OVERSEA_PARENTING("zboard/zboard.php?id=ppomppu4&category=6", "육아", true),
PPOMPPU_DOMESTIC_ALL("/zboard/zboard.php?id=ppomppu", "전체", true),
PPOMPPU_DOMESTIC_ETC("/zboard/zboard.php?id=ppomppu&category=1", "기타", true),
PPOMPPU_DOMESTIC_COMPUTER("/zboard/zboard.php?id=ppomppu&category=4", "컴퓨터", true),
PPOMPPU_DOMESTIC_DIGITAL("/zboard/zboard.php?id=ppomppu&category=5", "디지털", true),
PPOMPPU_DOMESTIC_FOOD("/zboard/zboard.php?id=ppomppu&category=6", "식품/건강", true),
PPOMPPU_DOMESTIC_BOOK("/zboard/zboard.php?id=ppomppu&category=8", "서적", true),
PPOMPPU_DOMESTIC_APPLIANCES("/zboard/zboard.php?id=ppomppu&category=9", "가전/가구", true),
PPOMPPU_DOMESTIC_PARENTING("/zboard/zboard.php?id=ppomppu&category=10", "육아", true),
PPOMPPU_DOMESTIC_GIFTCARD("/zboard/zboard.php?id=ppomppu&category=11", "상품권", true),
PPOMPPU_DOMESTIC_CLOTHES("/zboard/zboard.php?id=ppomppu&category=12", "의류/잡화", true),
PPOMPPU_DOMESTIC_COSMETIC("/zboard/zboard.php?id=ppomppu&category=13", "화장품", true),
PPOMPPU_DOMESTIC_OUTDOOR("/zboard/zboard.php?id=ppomppu&category=15", "등산/캠핑", true),
PPOMPPU_OVERSEA_ALL("/zboard/zboard.php?id=ppomppu4", "전체", true),
PPOMPPU_OVERSEA_ETC("/zboard/zboard.php?id=ppomppu4&category=1", "기타", true),
PPOMPPU_OVERSEA_APPLIANCES("/zboard/zboard.php?id=ppomppu4&category=7", "가전", true),
PPOMPPU_OVERSEA_TVAV("/zboard/zboard.php?id=ppomppu4&category=8", "TV/영상", true),
PPOMPPU_OVERSEA_COMPUTER("/zboard/zboard.php?id=ppomppu4&category=3", "컴퓨터", true),
PPOMPPU_OVERSEA_DIGITAL("/zboard/zboard.php?id=ppomppu4&category=4", "디지털", true),
PPOMPPU_OVERSEA_MOBILEACCESSORY("/zboard/zboard.php?id=ppomppu4&category=9", "액세서리", true),
PPOMPPU_OVERSEA_CLOTHES("/zboard/zboard.php?id=ppomppu4&category=5", "의류/잡화", true),
PPOMPPU_OVERSEA_WATCH("/zboard/zboard.php?id=ppomppu4&category=2", "시계", true),
PPOMPPU_OVERSEA_SHOES("/zboard/zboard.php?id=ppomppu4&category=11", "신발", true),
PPOMPPU_OVERSEA_FOOD("/zboard/zboard.php?id=ppomppu4&category=10", "식품/건강", true),
PPOMPPU_OVERSEA_PARENTING("/zboard/zboard.php?id=ppomppu4&category=6", "육아", true),
;
private String resourcePath;
private String menuName;
private boolean crawlWithDefaultTimer;
public static final String PPOMPPU_URL = "https://www.ppomppu.co.kr/";
public static final String PPOMPPU_URL = "https://www.ppomppu.co.kr";
public static String ofViewPageUrl(String articleUrl) {
return PPOMPPU_URL + "zboard/" + articleUrl;
return PPOMPPU_URL + "/zboard" + articleUrl;
}
}

2
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/util/ObjectMapperFactory.java → support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/util/ObjectMapperFactory.java
View File

@ -1,4 +1,4 @@
package com.myoa.engineering.crawl.ppomppu.processor.util;
package com.myoa.engineering.crawl.ppomppu.support.util;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.core.JsonParser.Feature;

8
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/factory/WebClientFilterFactory.java → support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/webclient/factory/WebClientFilterFactory.java
View File

@ -1,4 +1,4 @@
package com.myoa.engineering.crawl.ppomppu.processor.configuration.factory;
package com.myoa.engineering.crawl.ppomppu.support.webclient.factory;
import lombok.extern.slf4j.Slf4j;
import org.springframework.web.reactive.function.client.ClientRequest;
@ -27,8 +27,8 @@ public final class WebClientFilterFactory {
private static Mono<ClientRequest> writeRequest(ClientRequest clientRequest) {
try {
log.info("[WEBCLIENT REQUEST] uri : {} method : {} headers : {}",
clientRequest.url(), clientRequest.method(), clientRequest.headers());
log.info("[WEBCLIENT REQUEST] uri : {} method : {} headers : {}, body: {}",
clientRequest.url(), clientRequest.method(), clientRequest.headers(), clientRequest.body());
} catch (Exception e) {
log.error("[WEBCLIENT REQUEST] write request failed", e);
}
@ -37,7 +37,7 @@ public final class WebClientFilterFactory {
private static Mono<ClientResponse> writeResponse(ClientResponse clientResponse) {
try {
log.info("[WEBCLIENT REQUEST] statusCode : {} headers : {}",
log.info("[WEBCLIENT RESPONSE] statusCode : {} headers : {}",
clientResponse.rawStatusCode(), clientResponse.headers().asHttpHeaders());
} catch (Exception e) {
log.error("[WEBCLIENT RESPONSE] write response failed", e);

5
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/factory/WebFluxExchangeStragiesFactory.java → support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/webclient/factory/WebFluxExchangeStragiesFactory.java
View File

@ -1,7 +1,8 @@
package com.myoa.engineering.crawl.ppomppu.processor.configuration.factory;
package com.myoa.engineering.crawl.ppomppu.support.webclient.factory;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.myoa.engineering.crawl.ppomppu.processor.util.ObjectMapperFactory;
import com.myoa.engineering.crawl.ppomppu.support.util.ObjectMapperFactory;
import org.springframework.http.codec.json.Jackson2JsonDecoder;
import org.springframework.http.codec.json.Jackson2JsonEncoder;
import org.springframework.util.MimeTypeUtils;