From 5b4b44f093cfe26109b6d22a52b89e3c56d77154 Mon Sep 17 00:00:00 2001 From: "woo-jin.shin" Date: Wed, 17 Nov 2021 09:39:49 +0900 Subject: [PATCH 1/6] [PPN-211113] Implement spring config client --- build.gradle | 11 +++ processor/build.gradle | 3 + .../client/PpomppuBoardFeedRetriever.java | 6 +- .../resources/application-development.yml | 8 +- processor/src/main/resources/application.yml | 8 ++ receiver/src/main/resources/application.yml | 2 + sender/build.gradle | 23 +++++ .../ppomppu/sender/SenderApplication.java | 8 ++ .../sender/controller/TestAPIController.java | 37 ++++++++ .../crawl/ppomppu/sender/dto/MessageDTO.java | 12 +++ .../ppomppu/sender/dto/SlackMessageDTO.java | 36 ++++++++ .../infrastructure/client/MessageSender.java | 17 ++++ .../client/SlackMessageSender.java | 88 +++++++++++++++++++ .../resources/application-development.yml | 11 +++ .../main/resources/application-production.yml | 6 ++ sender/src/main/resources/application.yml | 19 ++++ .../main/resources/development/webclient.yml | 5 ++ support/build.gradle | 1 + .../support/dto/code/PpomppuBoardName.java | 52 +++++------ .../support}/util/ObjectMapperFactory.java | 2 +- .../factory/WebClientFilterFactory.java | 8 +- .../WebFluxExchangeStragiesFactory.java | 5 +- 22 files changed, 332 insertions(+), 36 deletions(-) create mode 100644 sender/build.gradle create mode 100644 sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/controller/TestAPIController.java create mode 100644 sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/dto/MessageDTO.java create mode 100644 sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/dto/SlackMessageDTO.java create mode 100644 sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/infrastructure/client/MessageSender.java create mode 100644 sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/infrastructure/client/SlackMessageSender.java create mode 100644 sender/src/main/resources/application-development.yml create mode 100644 sender/src/main/resources/application-production.yml create mode 100644 sender/src/main/resources/development/webclient.yml rename {processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor => support/src/main/java/com/myoa/engineering/crawl/ppomppu/support}/util/ObjectMapperFactory.java (98%) rename {processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration => support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/webclient}/factory/WebClientFilterFactory.java (87%) rename {processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration => support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/webclient}/factory/WebFluxExchangeStragiesFactory.java (94%) diff --git a/build.gradle b/build.gradle index 265f4ba..c09e9f9 100644 --- a/build.gradle +++ b/build.gradle @@ -32,6 +32,17 @@ allprojects { } } + ext { + set('springCloudVersion', "2020.0.4") + } + + dependencyManagement { + imports { + mavenBom "org.springframework.cloud:spring-cloud-dependencies:${springCloudVersion}" + } + } + + /* task initSourceFolders { sourceSets*.java.srcDirs*.each { if( !it.exists() ) { diff --git a/processor/build.gradle b/processor/build.gradle index ffc9e1d..eb68c4c 100644 --- a/processor/build.gradle +++ b/processor/build.gradle @@ -1,3 +1,4 @@ + dependencies { developmentOnly 'org.springframework.boot:spring-boot-devtools' runtimeOnly 'com.h2database:h2' @@ -9,6 +10,8 @@ dependencies { implementation 'org.springframework.boot:spring-boot-starter-webflux' implementation 'org.springframework.boot:spring-boot-starter-data-jpa' implementation 'org.springframework.boot:spring-boot-starter-jdbc' + implementation 'org.springframework.cloud:spring-cloud-starter-config' + implementation 'org.springframework.boot:spring-boot-starter-actuator' implementation 'com.rometools:rome:1.16.0' implementation 'org.jsoup:jsoup:1.14.2' implementation 'com.h2database:h2:1.4.200' diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/client/PpomppuBoardFeedRetriever.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/client/PpomppuBoardFeedRetriever.java index a635288..9347c63 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/client/PpomppuBoardFeedRetriever.java +++ b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/client/PpomppuBoardFeedRetriever.java @@ -1,9 +1,11 @@ package com.myoa.engineering.crawl.ppomppu.processor.infrastructure.client; -import com.myoa.engineering.crawl.ppomppu.processor.configuration.factory.WebClientFilterFactory; -import com.myoa.engineering.crawl.ppomppu.processor.configuration.factory.WebFluxExchangeStragiesFactory; +import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebClientFilterFactory; +import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebFluxExchangeStragiesFactory; import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName; import lombok.extern.slf4j.Slf4j; + +import org.springframework.core.ParameterizedTypeReference; import org.springframework.stereotype.Component; import org.springframework.web.reactive.function.client.WebClient; import org.springframework.web.reactive.function.client.WebClientRequestException; diff --git a/processor/src/main/resources/application-development.yml b/processor/src/main/resources/application-development.yml index 6d98894..88741dc 100644 --- a/processor/src/main/resources/application-development.yml +++ b/processor/src/main/resources/application-development.yml @@ -3,6 +3,12 @@ spring: activate: on-profile: development import: + - "configserver:http://192.168.0.100:11080" - classpath:/development/webclient.yml - classpath:/development/temppassword.yml - - classpath:/development/database.yml \ No newline at end of file + - classpath:/development/database.yml + profiles: + group: + development: development, datasource-development + + # import: optional:configserver:http://localhost:11080 # can be start up even config server was not found. \ No newline at end of file diff --git a/processor/src/main/resources/application.yml b/processor/src/main/resources/application.yml index aa61bfb..613bb66 100644 --- a/processor/src/main/resources/application.yml +++ b/processor/src/main/resources/application.yml @@ -1,4 +1,6 @@ spring: + application: + name: ppn-processor main: allow-bean-definition-overriding: true profiles: @@ -12,3 +14,9 @@ server: error: whitelabel: enabled: false + +management: + endpoints: + web: + exposure: + include: refresh diff --git a/receiver/src/main/resources/application.yml b/receiver/src/main/resources/application.yml index 782cd5a..89b6a30 100644 --- a/receiver/src/main/resources/application.yml +++ b/receiver/src/main/resources/application.yml @@ -1,4 +1,6 @@ spring: + application: + name: ppn-receiver main: allow-bean-definition-overriding: true profiles: diff --git a/sender/build.gradle b/sender/build.gradle new file mode 100644 index 0000000..3181ae3 --- /dev/null +++ b/sender/build.gradle @@ -0,0 +1,23 @@ +dependencies { + developmentOnly 'org.springframework.boot:spring-boot-devtools' + runtimeOnly 'mysql:mysql-connector-java' + compileOnly 'org.projectlombok:lombok' + + implementation project(':support') + // https://projectreactor.io/docs/core/release/reference/#debug-activate + implementation 'org.springframework.boot:spring-boot-starter-webflux' + implementation 'org.springframework.boot:spring-boot-starter-actuator' + implementation 'org.springframework.cloud:spring-cloud-starter-config' + + annotationProcessor 'org.springframework.boot:spring-boot-configuration-processor' + annotationProcessor 'org.projectlombok:lombok' + + testImplementation 'org.springframework.boot:spring-boot-starter-test' +} + +test { + useJUnitPlatform() + testLogging { + events "passed", "skipped", "failed" + } +} \ No newline at end of file diff --git a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/SenderApplication.java b/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/SenderApplication.java index 5f41116..e091164 100644 --- a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/SenderApplication.java +++ b/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/SenderApplication.java @@ -1,10 +1,18 @@ package com.myoa.engineering.crawl.ppomppu.sender; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; + /** * SenderApplication * @author Shin Woo-jin (woo-jin.shin@linecorp.com) * @since 2021-08-20 * */ +@SpringBootApplication public class SenderApplication { + + public static void main(String[] args) { + SpringApplication.run(SenderApplication.class, args); + } } diff --git a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/controller/TestAPIController.java b/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/controller/TestAPIController.java new file mode 100644 index 0000000..fad3683 --- /dev/null +++ b/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/controller/TestAPIController.java @@ -0,0 +1,37 @@ +package com.myoa.engineering.crawl.ppomppu.sender.controller; + +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import com.myoa.engineering.crawl.ppomppu.sender.dto.SlackMessageDTO; +import com.myoa.engineering.crawl.ppomppu.sender.infrastructure.client.SlackMessageSender; + +import reactor.core.publisher.Mono; + +/** + * TestAPIController + * @author Shin Woo-jin (woo-jin.shin@linecorp.com) + * @since 2021-11-15 + * + */ +@RestController +@RequestMapping("/api/v1") +public class TestAPIController { + + private final SlackMessageSender sender; + + public TestAPIController() { + this.sender = new SlackMessageSender("xoxb-2688454277126-2695026012277-K2Ib13lKokmTiBSnSMrc0Bp2"); + } + + @GetMapping("/test") + public Mono test() { + return sender.sendMessage(SlackMessageDTO.builder() + .text("test!") + .iconEmoji("monge") + .channel("notify_shopping") + .username("shopping notifier") + .build()); + } +} diff --git a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/dto/MessageDTO.java b/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/dto/MessageDTO.java new file mode 100644 index 0000000..db45e43 --- /dev/null +++ b/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/dto/MessageDTO.java @@ -0,0 +1,12 @@ +package com.myoa.engineering.crawl.ppomppu.sender.dto; + +import java.io.Serializable; + +/** + * MessageDTO + * @author Shin Woo-jin (woo-jin.shin@linecorp.com) + * @since 2021-11-14 + * + */ +public interface MessageDTO extends Serializable { +} diff --git a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/dto/SlackMessageDTO.java b/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/dto/SlackMessageDTO.java new file mode 100644 index 0000000..c02cb71 --- /dev/null +++ b/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/dto/SlackMessageDTO.java @@ -0,0 +1,36 @@ +package com.myoa.engineering.crawl.ppomppu.sender.dto; + +import com.fasterxml.jackson.annotation.JsonProperty; + +import lombok.Builder; +import lombok.Getter; +import lombok.NoArgsConstructor; + +/** + * SlackMessageDTO + * @author Shin Woo-jin (woo-jin.shin@linecorp.com) + * @since 2021-11-14 + * + */ +@Getter +@NoArgsConstructor +public class SlackMessageDTO implements MessageDTO { + + private final static long serialVersionUID = 4737608709660494713L; + + private String text; + private String channel; + private String username; + + @JsonProperty("icon_emoji") + private String iconEmoji; + + @Builder + public SlackMessageDTO(String text, String channel, String username, String iconEmoji) { + this.text = text; + this.channel = channel; + this.username = username; + this.iconEmoji = iconEmoji; + } + +} diff --git a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/infrastructure/client/MessageSender.java b/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/infrastructure/client/MessageSender.java new file mode 100644 index 0000000..4153bb4 --- /dev/null +++ b/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/infrastructure/client/MessageSender.java @@ -0,0 +1,17 @@ +package com.myoa.engineering.crawl.ppomppu.sender.infrastructure.client; + +import com.myoa.engineering.crawl.ppomppu.sender.dto.MessageDTO; + +import reactor.core.publisher.Mono; + +/** + * MessageSender + * @author Shin Woo-jin (woo-jin.shin@linecorp.com) + * @since 2021-11-14 + * + */ +public interface MessageSender { + + Mono sendMessage(T message); + +} diff --git a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/infrastructure/client/SlackMessageSender.java b/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/infrastructure/client/SlackMessageSender.java new file mode 100644 index 0000000..ea0732b --- /dev/null +++ b/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/infrastructure/client/SlackMessageSender.java @@ -0,0 +1,88 @@ +package com.myoa.engineering.crawl.ppomppu.sender.infrastructure.client; + +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; +import org.springframework.web.reactive.function.client.WebClient; +import org.springframework.web.reactive.function.client.WebClientRequestException; + +import com.myoa.engineering.crawl.ppomppu.sender.dto.SlackMessageDTO; +import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebClientFilterFactory; +import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebFluxExchangeStragiesFactory; + +import lombok.extern.slf4j.Slf4j; +import reactor.core.publisher.Mono; +import reactor.core.scheduler.Schedulers; + +/** + * SlackMessageSender + * + * @author Shin Woo-jin (woozu.shin@kakaoent.com) + * @since 2021-09-08 + */ +@Slf4j +public class SlackMessageSender implements MessageSender { + + private static final String SLACK_API_URL = "https://slack.com/api"; + + private final WebClient webClient; + private final String apiSecret; + + public SlackMessageSender(String apiSecret) { + this.webClient = WebClient.builder() + .baseUrl(SLACK_API_URL) + .defaultHeader(HttpHeaders.AUTHORIZATION, "Bearer " + apiSecret) + .exchangeStrategies(WebFluxExchangeStragiesFactory.ofDefault()) + .defaultHeader(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_UTF8_VALUE) + .defaultHeader(HttpHeaders.ACCEPT_CHARSET, "UTF-8") + .filter(WebClientFilterFactory.logRequest()) + .filter(WebClientFilterFactory.logResponse()) + .build(); + this.apiSecret = apiSecret; + } + + @Override + public Mono sendMessage(SlackMessageDTO message) { + return webClient.post() + .uri("/chat.postMessage") + .bodyValue(message) + .exchangeToMono(e -> e.bodyToMono(String.class)) + .publishOn(Schedulers.boundedElastic()) + .onErrorResume(WebClientRequestException.class, t -> { + log.info("Exception occured, ignoring. : {}", t.getClass().getSimpleName()); + return Mono.empty(); + }) + .doOnNext(e -> log.info("response: {}", e)); + } + +} + +/* +import requests +import json + +# woozuyeni_watchtower +SLACK_TOKEN="xoxb-2688454277126-2695026012277-K2Ib13lKokmTiBSnSMrc0Bp2" +CHANNEL = "notify_watchtower" +CHANNEL = "common" + + +POST_API = "https://slack.com/api/chat.postMessage" +HEADER = "Authorization: Bearer" + +headers = { + "Content-Type" : "application/json", + "Authorization" : f"Bearer {SLACK_TOKEN}" +} + +data = {} +data["text"] = "몽몽! :monge_big:" +data["channel"] = CHANNEL +data["username"] = "몽이봇" +data["icon_emoji"] = ":monge_big:" + +# requests.post(POST_API, data=json.dumps(data), headers=headers) + + +attachments = [{"title": "Cat", "image_url": image_url}] +image_url = "https://myoa-universe.com/assets/love1.png" + */ \ No newline at end of file diff --git a/sender/src/main/resources/application-development.yml b/sender/src/main/resources/application-development.yml new file mode 100644 index 0000000..7fc61ce --- /dev/null +++ b/sender/src/main/resources/application-development.yml @@ -0,0 +1,11 @@ +spring: + config: + activate: + on-profile: development + import: + - classpath:/development/webclient.yml + - "configserver:http://192.168.0.100:11080" + # import: optional:configserver:http://localhost:11080 # can be start up even config server was not found. + profiles: + group: + development: development, slackapi-development diff --git a/sender/src/main/resources/application-production.yml b/sender/src/main/resources/application-production.yml new file mode 100644 index 0000000..5028c79 --- /dev/null +++ b/sender/src/main/resources/application-production.yml @@ -0,0 +1,6 @@ +spring: + config: + activate: + on-profile: production +# import: +# - classpath:/production/webclient.yml \ No newline at end of file diff --git a/sender/src/main/resources/application.yml b/sender/src/main/resources/application.yml index e69de29..5097f71 100644 --- a/sender/src/main/resources/application.yml +++ b/sender/src/main/resources/application.yml @@ -0,0 +1,19 @@ +spring: + application: +# name: ppn-sender + name: app1 + main: + allow-bean-definition-overriding: true + profiles: + active: development + freemarker: + enabled: false + +server: + port: 20080 + +management: + endpoints: + web: + exposure: + include: refresh diff --git a/sender/src/main/resources/development/webclient.yml b/sender/src/main/resources/development/webclient.yml new file mode 100644 index 0000000..abe8b3b --- /dev/null +++ b/sender/src/main/resources/development/webclient.yml @@ -0,0 +1,5 @@ +webclient: + base-url: http://localhost:20081 + units: + - unit-name: processor-api + base-url: http://localhost:20081 \ No newline at end of file diff --git a/support/build.gradle b/support/build.gradle index 4d76cf7..ae063c9 100644 --- a/support/build.gradle +++ b/support/build.gradle @@ -3,6 +3,7 @@ dependencies { compileOnly 'org.projectlombok:lombok' // https://projectreactor.io/docs/core/release/reference/#debug-activate + implementation 'org.springframework.boot:spring-boot-starter-webflux' annotationProcessor 'org.springframework.boot:spring-boot-configuration-processor' annotationProcessor 'org.projectlombok:lombok' diff --git a/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/dto/code/PpomppuBoardName.java b/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/dto/code/PpomppuBoardName.java index 1fad0b9..aa15fc2 100644 --- a/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/dto/code/PpomppuBoardName.java +++ b/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/dto/code/PpomppuBoardName.java @@ -12,40 +12,40 @@ import lombok.Getter; @Getter @AllArgsConstructor public enum PpomppuBoardName { - PPOMPPU_DOMESTIC_ALL("zboard/zboard.php?id=ppomppu", "전체", true), - PPOMPPU_DOMESTIC_ETC("zboard/zboard.php?id=ppomppu&category=1", "기타", true), - PPOMPPU_DOMESTIC_COMPUTER("zboard/zboard.php?id=ppomppu&category=4", "컴퓨터", true), - PPOMPPU_DOMESTIC_DIGITAL("zboard/zboard.php?id=ppomppu&category=5", "디지털", true), - PPOMPPU_DOMESTIC_FOOD("zboard/zboard.php?id=ppomppu&category=6", "식품/건강", true), - PPOMPPU_DOMESTIC_BOOK("zboard/zboard.php?id=ppomppu&category=8", "서적", true), - PPOMPPU_DOMESTIC_APPLIANCES("zboard/zboard.php?id=ppomppu&category=9", "가전/가구", true), - PPOMPPU_DOMESTIC_PARENTING("zboard/zboard.php?id=ppomppu&category=10", "육아", true), - PPOMPPU_DOMESTIC_GIFTCARD("zboard/zboard.php?id=ppomppu&category=11", "상품권", true), - PPOMPPU_DOMESTIC_CLOTHES("zboard/zboard.php?id=ppomppu&category=12", "의류/잡화", true), - PPOMPPU_DOMESTIC_COSMETIC("zboard/zboard.php?id=ppomppu&category=13", "화장품", true), - PPOMPPU_DOMESTIC_OUTDOOR("zboard/zboard.php?id=ppomppu&category=15", "등산/캠핑", true), - PPOMPPU_OVERSEA_ALL("zboard/zboard.php?id=ppomppu4", "전체", true), - PPOMPPU_OVERSEA_ETC("zboard/zboard.php?id=ppomppu4&category=1", "기타", true), - PPOMPPU_OVERSEA_APPLIANCES("zboard/zboard.php?id=ppomppu4&category=7", "가전", true), - PPOMPPU_OVERSEA_TVAV("zboard/zboard.php?id=ppomppu4&category=8", "TV/영상", true), - PPOMPPU_OVERSEA_COMPUTER("zboard/zboard.php?id=ppomppu4&category=3", "컴퓨터", true), - PPOMPPU_OVERSEA_DIGITAL("zboard/zboard.php?id=ppomppu4&category=4", "디지털", true), - PPOMPPU_OVERSEA_MOBILEACCESSORY("zboard/zboard.php?id=ppomppu4&category=9", "액세서리", true), - PPOMPPU_OVERSEA_CLOTHES("zboard/zboard.php?id=ppomppu4&category=5", "의류/잡화", true), - PPOMPPU_OVERSEA_WATCH("zboard/zboard.php?id=ppomppu4&category=2", "시계", true), - PPOMPPU_OVERSEA_SHOES("zboard/zboard.php?id=ppomppu4&category=11", "신발", true), - PPOMPPU_OVERSEA_FOOD("zboard/zboard.php?id=ppomppu4&category=10", "식품/건강", true), - PPOMPPU_OVERSEA_PARENTING("zboard/zboard.php?id=ppomppu4&category=6", "육아", true), + PPOMPPU_DOMESTIC_ALL("/zboard/zboard.php?id=ppomppu", "전체", true), + PPOMPPU_DOMESTIC_ETC("/zboard/zboard.php?id=ppomppu&category=1", "기타", true), + PPOMPPU_DOMESTIC_COMPUTER("/zboard/zboard.php?id=ppomppu&category=4", "컴퓨터", true), + PPOMPPU_DOMESTIC_DIGITAL("/zboard/zboard.php?id=ppomppu&category=5", "디지털", true), + PPOMPPU_DOMESTIC_FOOD("/zboard/zboard.php?id=ppomppu&category=6", "식품/건강", true), + PPOMPPU_DOMESTIC_BOOK("/zboard/zboard.php?id=ppomppu&category=8", "서적", true), + PPOMPPU_DOMESTIC_APPLIANCES("/zboard/zboard.php?id=ppomppu&category=9", "가전/가구", true), + PPOMPPU_DOMESTIC_PARENTING("/zboard/zboard.php?id=ppomppu&category=10", "육아", true), + PPOMPPU_DOMESTIC_GIFTCARD("/zboard/zboard.php?id=ppomppu&category=11", "상품권", true), + PPOMPPU_DOMESTIC_CLOTHES("/zboard/zboard.php?id=ppomppu&category=12", "의류/잡화", true), + PPOMPPU_DOMESTIC_COSMETIC("/zboard/zboard.php?id=ppomppu&category=13", "화장품", true), + PPOMPPU_DOMESTIC_OUTDOOR("/zboard/zboard.php?id=ppomppu&category=15", "등산/캠핑", true), + PPOMPPU_OVERSEA_ALL("/zboard/zboard.php?id=ppomppu4", "전체", true), + PPOMPPU_OVERSEA_ETC("/zboard/zboard.php?id=ppomppu4&category=1", "기타", true), + PPOMPPU_OVERSEA_APPLIANCES("/zboard/zboard.php?id=ppomppu4&category=7", "가전", true), + PPOMPPU_OVERSEA_TVAV("/zboard/zboard.php?id=ppomppu4&category=8", "TV/영상", true), + PPOMPPU_OVERSEA_COMPUTER("/zboard/zboard.php?id=ppomppu4&category=3", "컴퓨터", true), + PPOMPPU_OVERSEA_DIGITAL("/zboard/zboard.php?id=ppomppu4&category=4", "디지털", true), + PPOMPPU_OVERSEA_MOBILEACCESSORY("/zboard/zboard.php?id=ppomppu4&category=9", "액세서리", true), + PPOMPPU_OVERSEA_CLOTHES("/zboard/zboard.php?id=ppomppu4&category=5", "의류/잡화", true), + PPOMPPU_OVERSEA_WATCH("/zboard/zboard.php?id=ppomppu4&category=2", "시계", true), + PPOMPPU_OVERSEA_SHOES("/zboard/zboard.php?id=ppomppu4&category=11", "신발", true), + PPOMPPU_OVERSEA_FOOD("/zboard/zboard.php?id=ppomppu4&category=10", "식품/건강", true), + PPOMPPU_OVERSEA_PARENTING("/zboard/zboard.php?id=ppomppu4&category=6", "육아", true), ; private String resourcePath; private String menuName; private boolean crawlWithDefaultTimer; - public static final String PPOMPPU_URL = "https://www.ppomppu.co.kr/"; + public static final String PPOMPPU_URL = "https://www.ppomppu.co.kr"; public static String ofViewPageUrl(String articleUrl) { - return PPOMPPU_URL + "zboard/" + articleUrl; + return PPOMPPU_URL + "/zboard" + articleUrl; } } diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/util/ObjectMapperFactory.java b/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/util/ObjectMapperFactory.java similarity index 98% rename from processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/util/ObjectMapperFactory.java rename to support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/util/ObjectMapperFactory.java index 811c76f..fcad823 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/util/ObjectMapperFactory.java +++ b/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/util/ObjectMapperFactory.java @@ -1,4 +1,4 @@ -package com.myoa.engineering.crawl.ppomppu.processor.util; +package com.myoa.engineering.crawl.ppomppu.support.util; import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.core.JsonParser.Feature; diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/factory/WebClientFilterFactory.java b/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/webclient/factory/WebClientFilterFactory.java similarity index 87% rename from processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/factory/WebClientFilterFactory.java rename to support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/webclient/factory/WebClientFilterFactory.java index dd74293..89d1ec9 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/factory/WebClientFilterFactory.java +++ b/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/webclient/factory/WebClientFilterFactory.java @@ -1,4 +1,4 @@ -package com.myoa.engineering.crawl.ppomppu.processor.configuration.factory; +package com.myoa.engineering.crawl.ppomppu.support.webclient.factory; import lombok.extern.slf4j.Slf4j; import org.springframework.web.reactive.function.client.ClientRequest; @@ -27,8 +27,8 @@ public final class WebClientFilterFactory { private static Mono writeRequest(ClientRequest clientRequest) { try { - log.info("[WEBCLIENT REQUEST] uri : {} method : {} headers : {}", - clientRequest.url(), clientRequest.method(), clientRequest.headers()); + log.info("[WEBCLIENT REQUEST] uri : {} method : {} headers : {}, body: {}", + clientRequest.url(), clientRequest.method(), clientRequest.headers(), clientRequest.body()); } catch (Exception e) { log.error("[WEBCLIENT REQUEST] write request failed", e); } @@ -37,7 +37,7 @@ public final class WebClientFilterFactory { private static Mono writeResponse(ClientResponse clientResponse) { try { - log.info("[WEBCLIENT REQUEST] statusCode : {} headers : {}", + log.info("[WEBCLIENT RESPONSE] statusCode : {} headers : {}", clientResponse.rawStatusCode(), clientResponse.headers().asHttpHeaders()); } catch (Exception e) { log.error("[WEBCLIENT RESPONSE] write response failed", e); diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/factory/WebFluxExchangeStragiesFactory.java b/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/webclient/factory/WebFluxExchangeStragiesFactory.java similarity index 94% rename from processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/factory/WebFluxExchangeStragiesFactory.java rename to support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/webclient/factory/WebFluxExchangeStragiesFactory.java index ebd9c68..6558638 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/factory/WebFluxExchangeStragiesFactory.java +++ b/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/webclient/factory/WebFluxExchangeStragiesFactory.java @@ -1,7 +1,8 @@ -package com.myoa.engineering.crawl.ppomppu.processor.configuration.factory; +package com.myoa.engineering.crawl.ppomppu.support.webclient.factory; import com.fasterxml.jackson.databind.ObjectMapper; -import com.myoa.engineering.crawl.ppomppu.processor.util.ObjectMapperFactory; +import com.myoa.engineering.crawl.ppomppu.support.util.ObjectMapperFactory; + import org.springframework.http.codec.json.Jackson2JsonDecoder; import org.springframework.http.codec.json.Jackson2JsonEncoder; import org.springframework.util.MimeTypeUtils; From 24a848dc9f24044911df541bb46c9ca7ef930f1d Mon Sep 17 00:00:00 2001 From: "woo-jin.shin" Date: Wed, 17 Nov 2021 23:49:30 +0900 Subject: [PATCH 2/6] [PPN-211113] Re-structure properties --- .../resources/application-development.yml | 5 ++-- .../main/resources/application-production.yml | 10 ++++++++ processor/src/main/resources/application.yml | 4 ++-- .../resources/application-development.yml | 1 - receiver/src/main/resources/application.yml | 7 ++++-- .../src/main/resources/logback-spring.xml | 11 +++++++++ .../logback/component/logback-file.xml | 23 +++++++++++++++++++ .../resources/logback/logback-development.xml | 19 +++++++++++++++ .../resources/logback/logback-production.xml | 19 +++++++++++++++ .../main/resources/production/production.yml | 5 ---- .../main/resources/production/webclient.yml | 5 ++++ .../resources/application-development.yml | 3 +++ sender/src/main/resources/application.yml | 10 ++++---- .../main/resources/development/webclient.yml | 5 ---- sender/src/main/resources/logback-spring.xml | 11 +++++++++ .../logback/component/logback-file.xml | 23 +++++++++++++++++++ .../resources/logback/logback-development.xml | 19 +++++++++++++++ .../resources/logback/logback-production.xml | 19 +++++++++++++++ 18 files changed, 178 insertions(+), 21 deletions(-) create mode 100644 processor/src/main/resources/application-production.yml create mode 100644 receiver/src/main/resources/logback-spring.xml create mode 100644 receiver/src/main/resources/logback/component/logback-file.xml create mode 100644 receiver/src/main/resources/logback/logback-development.xml create mode 100644 receiver/src/main/resources/logback/logback-production.xml delete mode 100644 receiver/src/main/resources/production/production.yml create mode 100644 receiver/src/main/resources/production/webclient.yml delete mode 100644 sender/src/main/resources/development/webclient.yml create mode 100644 sender/src/main/resources/logback-spring.xml create mode 100644 sender/src/main/resources/logback/component/logback-file.xml create mode 100644 sender/src/main/resources/logback/logback-development.xml create mode 100644 sender/src/main/resources/logback/logback-production.xml diff --git a/processor/src/main/resources/application-development.yml b/processor/src/main/resources/application-development.yml index 88741dc..6341010 100644 --- a/processor/src/main/resources/application-development.yml +++ b/processor/src/main/resources/application-development.yml @@ -5,10 +5,11 @@ spring: import: - "configserver:http://192.168.0.100:11080" - classpath:/development/webclient.yml - - classpath:/development/temppassword.yml - - classpath:/development/database.yml profiles: group: development: development, datasource-development +server: + port: 20081 + # import: optional:configserver:http://localhost:11080 # can be start up even config server was not found. \ No newline at end of file diff --git a/processor/src/main/resources/application-production.yml b/processor/src/main/resources/application-production.yml new file mode 100644 index 0000000..892c27d --- /dev/null +++ b/processor/src/main/resources/application-production.yml @@ -0,0 +1,10 @@ +spring: + config: + activate: + on-profile: development + import: + - "configserver:http://192.168.0.100:11080" + - classpath:/development/webclient.yml + profiles: + group: + development: development, datasource-development diff --git a/processor/src/main/resources/application.yml b/processor/src/main/resources/application.yml index 613bb66..b8b84f6 100644 --- a/processor/src/main/resources/application.yml +++ b/processor/src/main/resources/application.yml @@ -4,13 +4,13 @@ spring: main: allow-bean-definition-overriding: true profiles: - active: development + active: ${SPRING_ACTIVE_PROFILE:development} freemarker: enabled: false server: - port: 20081 + port: 20080 error: whitelabel: enabled: false diff --git a/receiver/src/main/resources/application-development.yml b/receiver/src/main/resources/application-development.yml index 34f30c9..70ac133 100644 --- a/receiver/src/main/resources/application-development.yml +++ b/receiver/src/main/resources/application-development.yml @@ -4,4 +4,3 @@ spring: on-profile: development import: - classpath:/development/webclient.yml - - classpath:/development/temppassword.yml diff --git a/receiver/src/main/resources/application.yml b/receiver/src/main/resources/application.yml index 89b6a30..61ebab9 100644 --- a/receiver/src/main/resources/application.yml +++ b/receiver/src/main/resources/application.yml @@ -4,9 +4,12 @@ spring: main: allow-bean-definition-overriding: true profiles: - active: development + active: ${SPRING_ACTIVE_PROFILE:development} freemarker: enabled: false server: - port: 20080 \ No newline at end of file + port: 20080 + error: + whitelabel: + enabled: false \ No newline at end of file diff --git a/receiver/src/main/resources/logback-spring.xml b/receiver/src/main/resources/logback-spring.xml new file mode 100644 index 0000000..f0fcb6c --- /dev/null +++ b/receiver/src/main/resources/logback-spring.xml @@ -0,0 +1,11 @@ + + + + + + + + + + + \ No newline at end of file diff --git a/receiver/src/main/resources/logback/component/logback-file.xml b/receiver/src/main/resources/logback/component/logback-file.xml new file mode 100644 index 0000000..65e54db --- /dev/null +++ b/receiver/src/main/resources/logback/component/logback-file.xml @@ -0,0 +1,23 @@ + + + + + ${DIRECTORY}/${LOG_FILE_BASE}_log + + ${DIRECTORY}/${LOG_FILE_BASE}_log.%d{yyyyMMdd}.%i + + 1000MB + + 60 + + + ${FILE_LOG_PATTERN} + ${IMMEDIATE_FLUSH} + + + + 1024 + + + diff --git a/receiver/src/main/resources/logback/logback-development.xml b/receiver/src/main/resources/logback/logback-development.xml new file mode 100644 index 0000000..458e3d8 --- /dev/null +++ b/receiver/src/main/resources/logback/logback-development.xml @@ -0,0 +1,19 @@ + + + + + + + + + + + + + + + + + + + diff --git a/receiver/src/main/resources/logback/logback-production.xml b/receiver/src/main/resources/logback/logback-production.xml new file mode 100644 index 0000000..f824e41 --- /dev/null +++ b/receiver/src/main/resources/logback/logback-production.xml @@ -0,0 +1,19 @@ + + + + + + + + + + + + + + + + + + + diff --git a/receiver/src/main/resources/production/production.yml b/receiver/src/main/resources/production/production.yml deleted file mode 100644 index 8bc6223..0000000 --- a/receiver/src/main/resources/production/production.yml +++ /dev/null @@ -1,5 +0,0 @@ -webclient: - base-url: http://ppomppu_notifier_processor:20080 - units: - - unit-name: processor-api - base-url: http://ppomppu_notifier_processor:20080 diff --git a/receiver/src/main/resources/production/webclient.yml b/receiver/src/main/resources/production/webclient.yml new file mode 100644 index 0000000..18322d9 --- /dev/null +++ b/receiver/src/main/resources/production/webclient.yml @@ -0,0 +1,5 @@ +webclient: + base-url: http://ppn_processor:20080 + units: + - unit-name: processor-api + base-url: http://ppn_processor:20080 diff --git a/sender/src/main/resources/application-development.yml b/sender/src/main/resources/application-development.yml index 7fc61ce..b821007 100644 --- a/sender/src/main/resources/application-development.yml +++ b/sender/src/main/resources/application-development.yml @@ -9,3 +9,6 @@ spring: profiles: group: development: development, slackapi-development + +server: + port: 20081 diff --git a/sender/src/main/resources/application.yml b/sender/src/main/resources/application.yml index 5097f71..cb42b9d 100644 --- a/sender/src/main/resources/application.yml +++ b/sender/src/main/resources/application.yml @@ -1,19 +1,21 @@ spring: application: -# name: ppn-sender - name: app1 + name: ppn-sender main: allow-bean-definition-overriding: true profiles: - active: development + active: ${SPRING_ACTIVE_PROFILE:development} freemarker: enabled: false server: port: 20080 + error: + whitelabel: + enabled: false management: endpoints: web: exposure: - include: refresh + include: refresh, env diff --git a/sender/src/main/resources/development/webclient.yml b/sender/src/main/resources/development/webclient.yml deleted file mode 100644 index abe8b3b..0000000 --- a/sender/src/main/resources/development/webclient.yml +++ /dev/null @@ -1,5 +0,0 @@ -webclient: - base-url: http://localhost:20081 - units: - - unit-name: processor-api - base-url: http://localhost:20081 \ No newline at end of file diff --git a/sender/src/main/resources/logback-spring.xml b/sender/src/main/resources/logback-spring.xml new file mode 100644 index 0000000..f0fcb6c --- /dev/null +++ b/sender/src/main/resources/logback-spring.xml @@ -0,0 +1,11 @@ + + + + + + + + + + + \ No newline at end of file diff --git a/sender/src/main/resources/logback/component/logback-file.xml b/sender/src/main/resources/logback/component/logback-file.xml new file mode 100644 index 0000000..65e54db --- /dev/null +++ b/sender/src/main/resources/logback/component/logback-file.xml @@ -0,0 +1,23 @@ + + + + + ${DIRECTORY}/${LOG_FILE_BASE}_log + + ${DIRECTORY}/${LOG_FILE_BASE}_log.%d{yyyyMMdd}.%i + + 1000MB + + 60 + + + ${FILE_LOG_PATTERN} + ${IMMEDIATE_FLUSH} + + + + 1024 + + + diff --git a/sender/src/main/resources/logback/logback-development.xml b/sender/src/main/resources/logback/logback-development.xml new file mode 100644 index 0000000..458e3d8 --- /dev/null +++ b/sender/src/main/resources/logback/logback-development.xml @@ -0,0 +1,19 @@ + + + + + + + + + + + + + + + + + + + diff --git a/sender/src/main/resources/logback/logback-production.xml b/sender/src/main/resources/logback/logback-production.xml new file mode 100644 index 0000000..f824e41 --- /dev/null +++ b/sender/src/main/resources/logback/logback-production.xml @@ -0,0 +1,19 @@ + + + + + + + + + + + + + + + + + + + From bbf4affc160903d04b5fc60cd61ae359ecbc6c3a Mon Sep 17 00:00:00 2001 From: "woo-jin.shin" Date: Thu, 18 Nov 2021 00:38:16 +0900 Subject: [PATCH 3/6] [PPN-211113] Implement ConfigurationProperties --- .../processor/ProcessorApplication.java | 4 ++ .../constant/WebClientPropertiesUnitName.java | 19 ++++++++ .../client/SenderAPIClient.java | 36 +++++++++++++++ .../main/resources/development/webclient.yml | 4 +- .../main/resources/production/webclient.yml | 5 ++- receiver/build.gradle | 1 + .../ppomppu/receiver/ReceiverApplication.java | 5 ++- .../TelegramBotConfiguration.java | 7 ++- .../properties/TelegramBotProperties.java | 33 +++++++++++--- .../constant/WebClientPropertiesUnitName.java | 19 ++++++++ .../client/ProcessorAPIWebClient.java | 17 ++++--- .../resources/application-development.yml | 1 + .../main/resources/application-production.yml | 3 +- .../main/resources/development/webclient.yml | 4 +- .../main/resources/production/webclient.yml | 4 +- .../ppomppu/sender/SenderApplication.java | 4 ++ .../client/SlackMessageSender.java | 2 - support/build.gradle | 5 +-- ...PpomppuNotifierWebClientConfiguration.java | 19 ++++++++ .../properties/WebClientProperties.java | 44 +++++++++++++++++++ 20 files changed, 206 insertions(+), 30 deletions(-) create mode 100644 processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/constant/WebClientPropertiesUnitName.java create mode 100644 processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/client/SenderAPIClient.java create mode 100644 receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/dto/constant/WebClientPropertiesUnitName.java create mode 100644 support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/webclient/PpomppuNotifierWebClientConfiguration.java create mode 100644 support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/webclient/properties/WebClientProperties.java diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/ProcessorApplication.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/ProcessorApplication.java index b310e75..eb9f40c 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/ProcessorApplication.java +++ b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/ProcessorApplication.java @@ -2,6 +2,9 @@ package com.myoa.engineering.crawl.ppomppu.processor; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.context.annotation.Import; + +import com.myoa.engineering.crawl.ppomppu.support.webclient.PpomppuNotifierWebClientConfiguration; /** * ProcessorApplication @@ -9,6 +12,7 @@ import org.springframework.boot.autoconfigure.SpringBootApplication; * @since 2021-08-20 * */ +@Import({ PpomppuNotifierWebClientConfiguration.class }) @SpringBootApplication public class ProcessorApplication { diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/constant/WebClientPropertiesUnitName.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/constant/WebClientPropertiesUnitName.java new file mode 100644 index 0000000..3383138 --- /dev/null +++ b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/constant/WebClientPropertiesUnitName.java @@ -0,0 +1,19 @@ +package com.myoa.engineering.crawl.ppomppu.processor.dto.constant; + +import lombok.AllArgsConstructor; +import lombok.Getter; + +/** + * WebClientPropertiesUnitName + * @author Shin Woo-jin (woo-jin.shin@linecorp.com) + * @since 2021-11-18 + * + */ +@Getter +@AllArgsConstructor +public enum WebClientPropertiesUnitName { + PPOMPPU_NOTIFIER_SENDER_API("ppn-sender-api"), + ; + + private String unitName; +} diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/client/SenderAPIClient.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/client/SenderAPIClient.java new file mode 100644 index 0000000..e9de2a8 --- /dev/null +++ b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/client/SenderAPIClient.java @@ -0,0 +1,36 @@ +package com.myoa.engineering.crawl.ppomppu.processor.infrastructure.client; + +import org.springframework.stereotype.Component; +import org.springframework.web.reactive.function.client.WebClient; + +import com.myoa.engineering.crawl.ppomppu.processor.dto.constant.WebClientPropertiesUnitName; +import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebClientFilterFactory; +import com.myoa.engineering.crawl.ppomppu.support.webclient.properties.WebClientProperties; +import com.myoa.engineering.crawl.ppomppu.support.webclient.properties.WebClientProperties.WebClientPropertiesUnit; + +import lombok.extern.slf4j.Slf4j; + +/** + * PpomppuNotifierSenderAPIClient + * @author Shin Woo-jin (woo-jin.shin@linecorp.com) + * @since 2021-11-17 + * + */ +@Slf4j +@Component +public class SenderAPIClient { + + private final WebClient webClient; + + public SenderAPIClient(WebClientProperties webClientProperties) { + WebClientPropertiesUnit webClientPropertiesUnit = + webClientProperties.find(WebClientPropertiesUnitName.PPOMPPU_NOTIFIER_SENDER_API.getUnitName()); + this.webClient = WebClient.builder() + .baseUrl(webClientPropertiesUnit.getBaseUrl()) + .filter(WebClientFilterFactory.logRequest()) + .filter(WebClientFilterFactory.logResponse()) + .build(); + } +} + +// TODO webclient properties \ No newline at end of file diff --git a/processor/src/main/resources/development/webclient.yml b/processor/src/main/resources/development/webclient.yml index dc5e59f..d600658 100644 --- a/processor/src/main/resources/development/webclient.yml +++ b/processor/src/main/resources/development/webclient.yml @@ -1,5 +1,5 @@ webclient: - some: test + init: true units: - - unit-name: processor-api + - unit-name: ppn-sender-api base-url: http://localhost:20081 \ No newline at end of file diff --git a/processor/src/main/resources/production/webclient.yml b/processor/src/main/resources/production/webclient.yml index 4bb1d1d..862c2b9 100644 --- a/processor/src/main/resources/production/webclient.yml +++ b/processor/src/main/resources/production/webclient.yml @@ -1,4 +1,5 @@ webclient: + init: true units: - - unit-name: processor-api - base-url: http://soundhoundfound-processor:20080 \ No newline at end of file + - unit-name: ppn-sender-api + base-url: http://ppn_sender:20081 \ No newline at end of file diff --git a/receiver/build.gradle b/receiver/build.gradle index f56745d..f7771b1 100644 --- a/receiver/build.gradle +++ b/receiver/build.gradle @@ -8,6 +8,7 @@ dependencies { // https://projectreactor.io/docs/core/release/reference/#debug-activate implementation 'org.springframework.boot:spring-boot-starter-webflux' implementation 'org.springframework.boot:spring-boot-configuration-processor' + implementation 'org.springframework.cloud:spring-cloud-starter-config' implementation 'org.telegram:telegrambots:5.3.0' annotationProcessor 'org.springframework.boot:spring-boot-configuration-processor' diff --git a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/ReceiverApplication.java b/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/ReceiverApplication.java index a50c70b..339a01d 100644 --- a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/ReceiverApplication.java +++ b/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/ReceiverApplication.java @@ -3,8 +3,10 @@ package com.myoa.engineering.crawl.ppomppu.receiver; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Import; import com.myoa.engineering.crawl.ppomppu.receiver.configuration.properties.TelegramBotProperties; +import com.myoa.engineering.crawl.ppomppu.support.webclient.PpomppuNotifierWebClientConfiguration; /** * ReceiverApplication @@ -12,8 +14,9 @@ import com.myoa.engineering.crawl.ppomppu.receiver.configuration.properties.Tele * @since 2021-08-20 * */ +@Import({ PpomppuNotifierWebClientConfiguration.class}) @SpringBootApplication -@EnableConfigurationProperties({ TelegramBotProperties.class }) +// @EnableConfigurationProperties({ TelegramBotProperties.class }) public class ReceiverApplication { public static void main(String[] args) { diff --git a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/configuration/TelegramBotConfiguration.java b/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/configuration/TelegramBotConfiguration.java index 31ab08b..dd120dc 100644 --- a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/configuration/TelegramBotConfiguration.java +++ b/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/configuration/TelegramBotConfiguration.java @@ -9,6 +9,7 @@ import org.telegram.telegrambots.meta.exceptions.TelegramApiException; import org.telegram.telegrambots.updatesreceivers.DefaultBotSession; import com.myoa.engineering.crawl.ppomppu.receiver.configuration.properties.TelegramBotProperties; +import com.myoa.engineering.crawl.ppomppu.receiver.configuration.properties.TelegramBotProperties.TelegramBotPropertiesUnit; import com.myoa.engineering.crawl.ppomppu.receiver.dispatch.MessageDispatcher; import com.myoa.engineering.crawl.ppomppu.receiver.handler.message.MessageHandler; @@ -21,6 +22,7 @@ import com.myoa.engineering.crawl.ppomppu.receiver.handler.message.MessageHandle @Configuration public class TelegramBotConfiguration { + private static final String BOT_PROPERTIES_UNIT_NAME = "ppomppu_notify_bot"; @Bean public TelegramBotsApi telegramBotsApi(MessageDispatcher messageDispatcher) throws TelegramApiException { TelegramBotsApi api = new TelegramBotsApi(DefaultBotSession.class); @@ -30,7 +32,8 @@ public class TelegramBotConfiguration { @Bean public MessageDispatcher messageDispatcher(List messageHandlers, - TelegramBotProperties botProperties) { - return new MessageDispatcher(messageHandlers, botProperties.getName(), botProperties.getToken()); + TelegramBotProperties telegramBotProperties) { + TelegramBotPropertiesUnit propertiesUnit = telegramBotProperties.find(BOT_PROPERTIES_UNIT_NAME); + return new MessageDispatcher(messageHandlers, propertiesUnit.getName(), propertiesUnit.getToken()); } } diff --git a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/configuration/properties/TelegramBotProperties.java b/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/configuration/properties/TelegramBotProperties.java index f524798..e438889 100644 --- a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/configuration/properties/TelegramBotProperties.java +++ b/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/configuration/properties/TelegramBotProperties.java @@ -1,9 +1,17 @@ package com.myoa.engineering.crawl.ppomppu.receiver.configuration.properties; +import java.util.ArrayList; +import java.util.List; + import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.boot.context.properties.ConstructorBinding; +import org.springframework.stereotype.Component; +import lombok.AllArgsConstructor; +import lombok.Data; import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; /** * TelegramBotProperties @@ -11,15 +19,26 @@ import lombok.Getter; * @since 2021-09-05 * */ +@Component +@Setter @Getter -@ConstructorBinding -@ConfigurationProperties(prefix = "telegram.bot") +@ConfigurationProperties(prefix = "infra.telegram.bot") public class TelegramBotProperties { - private final String name; - private final String token; - public TelegramBotProperties(final String name, final String token) { - this.name = name; - this.token = token; + private List units = new ArrayList<>(); + + @Data + public static class TelegramBotPropertiesUnit { + private String unitName; + private String name; + private String token; + } + + public TelegramBotPropertiesUnit find(String unitName) { + return units.stream() + .filter(e -> e.getUnitName().equals(unitName)) + .findFirst() + .orElseThrow( + () -> new IllegalArgumentException(this.getClass().getName() + ": unitName Not found. " + unitName)); } } diff --git a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/dto/constant/WebClientPropertiesUnitName.java b/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/dto/constant/WebClientPropertiesUnitName.java new file mode 100644 index 0000000..bdab14d --- /dev/null +++ b/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/dto/constant/WebClientPropertiesUnitName.java @@ -0,0 +1,19 @@ +package com.myoa.engineering.crawl.ppomppu.receiver.dto.constant; + +import lombok.AllArgsConstructor; +import lombok.Getter; + +/** + * WebClientPropertiesUnitName + * @author Shin Woo-jin (woo-jin.shin@linecorp.com) + * @since 2021-11-18 + * + */ +@Getter +@AllArgsConstructor +public enum WebClientPropertiesUnitName { + PPOMPPU_NOTIFIER_PROCESSOR_API("ppn-processor-api"), + ; + + private String unitName; +} diff --git a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/infrastructure/client/ProcessorAPIWebClient.java b/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/infrastructure/client/ProcessorAPIWebClient.java index 87b763e..2dbca2a 100644 --- a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/infrastructure/client/ProcessorAPIWebClient.java +++ b/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/infrastructure/client/ProcessorAPIWebClient.java @@ -1,12 +1,15 @@ package com.myoa.engineering.crawl.ppomppu.receiver.infrastructure.client; -import org.springframework.beans.factory.annotation.Value; import org.springframework.core.ParameterizedTypeReference; import org.springframework.stereotype.Component; import org.springframework.web.reactive.function.client.WebClient; import org.springframework.web.reactive.function.client.WebClientRequestException; +import com.myoa.engineering.crawl.ppomppu.receiver.dto.constant.WebClientPropertiesUnitName; import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName; +import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebClientFilterFactory; +import com.myoa.engineering.crawl.ppomppu.support.webclient.properties.WebClientProperties; +import com.myoa.engineering.crawl.ppomppu.support.webclient.properties.WebClientProperties.WebClientPropertiesUnit; import lombok.extern.slf4j.Slf4j; import reactor.core.publisher.Mono; @@ -18,10 +21,14 @@ public class ProcessorAPIWebClient { private final WebClient webClient; - public ProcessorAPIWebClient(WebClient.Builder webClientBuilder, - @Value("${webclient.base-url}") String baseUrl) { - this.webClient = webClientBuilder.baseUrl(baseUrl) - .build(); + public ProcessorAPIWebClient(WebClientProperties webClientProperties) { + WebClientPropertiesUnit webClientPropertiesUnit = + webClientProperties.find(WebClientPropertiesUnitName.PPOMPPU_NOTIFIER_PROCESSOR_API.getUnitName()); + this.webClient = WebClient.builder() + .baseUrl(webClientPropertiesUnit.getBaseUrl()) + .filter(WebClientFilterFactory.logRequest()) + .filter(WebClientFilterFactory.logResponse()) + .build(); } public Mono emitParseEvent(PpomppuBoardName boardName) { diff --git a/receiver/src/main/resources/application-development.yml b/receiver/src/main/resources/application-development.yml index 70ac133..ad67d50 100644 --- a/receiver/src/main/resources/application-development.yml +++ b/receiver/src/main/resources/application-development.yml @@ -4,3 +4,4 @@ spring: on-profile: development import: - classpath:/development/webclient.yml + - "configserver:http://192.168.0.100:11080" \ No newline at end of file diff --git a/receiver/src/main/resources/application-production.yml b/receiver/src/main/resources/application-production.yml index 7c1069c..831bcb9 100644 --- a/receiver/src/main/resources/application-production.yml +++ b/receiver/src/main/resources/application-production.yml @@ -3,4 +3,5 @@ spring: activate: on-profile: production import: - - classpath:/production/webclient.yml \ No newline at end of file + - classpath:/production/webclient.yml + - "configserver:http://192.168.0.100:11080" \ No newline at end of file diff --git a/receiver/src/main/resources/development/webclient.yml b/receiver/src/main/resources/development/webclient.yml index abe8b3b..9fbfe3a 100644 --- a/receiver/src/main/resources/development/webclient.yml +++ b/receiver/src/main/resources/development/webclient.yml @@ -1,5 +1,5 @@ webclient: - base-url: http://localhost:20081 + init: true units: - - unit-name: processor-api + - unit-name: ppn-processor-api base-url: http://localhost:20081 \ No newline at end of file diff --git a/receiver/src/main/resources/production/webclient.yml b/receiver/src/main/resources/production/webclient.yml index 18322d9..aa3fb1b 100644 --- a/receiver/src/main/resources/production/webclient.yml +++ b/receiver/src/main/resources/production/webclient.yml @@ -1,5 +1,5 @@ webclient: - base-url: http://ppn_processor:20080 + init: true units: - - unit-name: processor-api + - unit-name: ppn-processor-api base-url: http://ppn_processor:20080 diff --git a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/SenderApplication.java b/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/SenderApplication.java index e091164..20f8126 100644 --- a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/SenderApplication.java +++ b/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/SenderApplication.java @@ -2,6 +2,9 @@ package com.myoa.engineering.crawl.ppomppu.sender; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.context.annotation.Import; + +import com.myoa.engineering.crawl.ppomppu.support.webclient.PpomppuNotifierWebClientConfiguration; /** * SenderApplication @@ -9,6 +12,7 @@ import org.springframework.boot.autoconfigure.SpringBootApplication; * @since 2021-08-20 * */ +@Import({ PpomppuNotifierWebClientConfiguration.class }) @SpringBootApplication public class SenderApplication { diff --git a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/infrastructure/client/SlackMessageSender.java b/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/infrastructure/client/SlackMessageSender.java index ea0732b..1eef297 100644 --- a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/infrastructure/client/SlackMessageSender.java +++ b/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/infrastructure/client/SlackMessageSender.java @@ -25,7 +25,6 @@ public class SlackMessageSender implements MessageSender { private static final String SLACK_API_URL = "https://slack.com/api"; private final WebClient webClient; - private final String apiSecret; public SlackMessageSender(String apiSecret) { this.webClient = WebClient.builder() @@ -37,7 +36,6 @@ public class SlackMessageSender implements MessageSender { .filter(WebClientFilterFactory.logRequest()) .filter(WebClientFilterFactory.logResponse()) .build(); - this.apiSecret = apiSecret; } @Override diff --git a/support/build.gradle b/support/build.gradle index ae063c9..42415c5 100644 --- a/support/build.gradle +++ b/support/build.gradle @@ -1,12 +1,9 @@ dependencies { - runtimeOnly 'mysql:mysql-connector-java' compileOnly 'org.projectlombok:lombok' // https://projectreactor.io/docs/core/release/reference/#debug-activate - implementation 'org.springframework.boot:spring-boot-starter-webflux' - - annotationProcessor 'org.springframework.boot:spring-boot-configuration-processor' annotationProcessor 'org.projectlombok:lombok' + implementation 'org.springframework.boot:spring-boot-starter-webflux' } diff --git a/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/webclient/PpomppuNotifierWebClientConfiguration.java b/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/webclient/PpomppuNotifierWebClientConfiguration.java new file mode 100644 index 0000000..b907331 --- /dev/null +++ b/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/webclient/PpomppuNotifierWebClientConfiguration.java @@ -0,0 +1,19 @@ +package com.myoa.engineering.crawl.ppomppu.support.webclient; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Configuration; + +/** + * PpomppuNotifierWebClientConfiguration + * @author Shin Woo-jin (woo-jin.shin@linecorp.com) + * @since 2021-11-18 + * + */ +@Configuration +@ConditionalOnProperty(value = "webclient.init", havingValue = "true") +@ComponentScan(basePackageClasses = WebClientBaseScan.class) +public class PpomppuNotifierWebClientConfiguration { + + +} diff --git a/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/webclient/properties/WebClientProperties.java b/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/webclient/properties/WebClientProperties.java new file mode 100644 index 0000000..64b89f8 --- /dev/null +++ b/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/webclient/properties/WebClientProperties.java @@ -0,0 +1,44 @@ +package com.myoa.engineering.crawl.ppomppu.support.webclient.properties; + +import java.util.ArrayList; +import java.util.List; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.lang.NonNull; +import org.springframework.stereotype.Component; + +import lombok.Data; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +/** + * WebClientPropertiesUnit + * @author Shin Woo-jin (woo-jin.shin@linecorp.com) + * @since 2021-11-18 + * + */ +@NoArgsConstructor +@Setter +@Getter +@Component +@ConfigurationProperties(prefix = "webclient") +public class WebClientProperties { + + private List units = new ArrayList<>(); + + @Data + public static class WebClientPropertiesUnit { + private String unitName; + private String baseUrl; + // TODO headers + } + + public WebClientPropertiesUnit find(@NonNull String unitName) { + return units.stream() + .filter(x -> x.getUnitName().equals(unitName)) + .findFirst() + .orElseThrow( + () -> new IllegalArgumentException("Not found properties unit. unitName : " + unitName)); + } +} From 15052270373b4f6c48818cfb39a2ed573d1d8a5a Mon Sep 17 00:00:00 2001 From: "woo-jin.shin" Date: Sun, 21 Nov 2021 13:30:44 +0900 Subject: [PATCH 4/6] [PPN-211113] Modify application properties --- .../configuration/H2ConsoleConfiguration.java | 2 +- .../PpomppuDatasourceConfiguration.java | 7 +- .../properties/DatasourceProperties.java | 16 ++-- .../properties/HibernateProperties.java | 3 + .../controller/CrawlAPIController.java | 31 ++++--- .../processor/dto/PpomppuArticleParser.java | 79 ++++++++++++++++ .../dto/PpomppuArticleTransformer.java | 91 +++++++------------ ...lient.java => MessageSenderAPIClient.java} | 22 ++++- .../client/PpomppuBoardFeedRetriever.java | 4 +- .../service/MessageSenderService.java | 37 ++++++++ .../processor/service/PpomppuFeedService.java | 12 +-- .../resources/application-development.yml | 4 +- .../main/resources/application-production.yml | 6 +- processor/src/main/resources/application.yml | 4 +- .../MessageSenderAPIController.java | 31 +++++++ .../resources/application-development.yml | 6 +- .../main/resources/application-production.yml | 4 +- sender/src/main/resources/application.yml | 8 ++ .../ppomppu/support/dto/SimpleMessageDTO.java | 37 ++++++++ 19 files changed, 296 insertions(+), 108 deletions(-) create mode 100644 processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/PpomppuArticleParser.java rename processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/client/{SenderAPIClient.java => MessageSenderAPIClient.java} (57%) create mode 100644 processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/MessageSenderService.java create mode 100644 sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/controller/MessageSenderAPIController.java create mode 100644 support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/dto/SimpleMessageDTO.java diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/H2ConsoleConfiguration.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/H2ConsoleConfiguration.java index d0d1a9b..993d7f8 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/H2ConsoleConfiguration.java +++ b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/H2ConsoleConfiguration.java @@ -11,7 +11,7 @@ import org.springframework.context.event.ContextRefreshedEvent; import org.springframework.context.event.EventListener; @Slf4j -@Profile("development") +@Profile("datasource-development") @Configuration public class H2ConsoleConfiguration { diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/PpomppuDatasourceConfiguration.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/PpomppuDatasourceConfiguration.java index 64c0a52..b78c475 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/PpomppuDatasourceConfiguration.java +++ b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/PpomppuDatasourceConfiguration.java @@ -32,7 +32,7 @@ import org.springframework.transaction.PlatformTransactionManager; ) public class PpomppuDatasourceConfiguration { - private static final String DATA_SOURCE_UNIT_NAME = "ppomppu"; + private static final String DATA_SOURCE_UNIT_NAME = "ppn_mysql"; private final DatasourceProperties dataSourceProeprties; private final HikariProperties hikariProperties; @@ -52,8 +52,9 @@ public class PpomppuDatasourceConfiguration { final HikariConfig hikariConfig = new HikariConfig(); hikariConfig.setJdbcUrl(dataSourcePropertiesUnit.toCompletedJdbcUrl()); - hikariConfig.setUsername("sa"); - hikariConfig.setPassword("sa"); + hikariConfig.setDriverClassName(dataSourcePropertiesUnit.getDriverClassName()); + hikariConfig.setUsername(dataSourcePropertiesUnit.getUsername()); + hikariConfig.setPassword(dataSourcePropertiesUnit.getPassword()); hikariConfig.setAutoCommit(hikariProperties.getAutoCommit()); hikariConfig.setMaximumPoolSize(hikariProperties.getMaximumPoolSize()); hikariConfig.setMinimumIdle(hikariProperties.getMinimumIdle()); diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/properties/DatasourceProperties.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/properties/DatasourceProperties.java index 34c0ac5..351eb13 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/properties/DatasourceProperties.java +++ b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/properties/DatasourceProperties.java @@ -2,6 +2,7 @@ package com.myoa.engineering.crawl.ppomppu.processor.configuration.properties; import com.myoa.engineering.crawl.ppomppu.support.util.ObjectUtil; import java.util.List; + import lombok.Getter; import lombok.Setter; import org.springframework.boot.context.properties.ConfigurationProperties; @@ -10,7 +11,7 @@ import org.springframework.stereotype.Component; @Component @Setter @Getter -@ConfigurationProperties(prefix = "datasource") +@ConfigurationProperties(prefix = "infra.database") public class DatasourceProperties { private List units; @@ -22,14 +23,17 @@ public class DatasourceProperties { private String unitName; private String schemaName; private String connectionParameters; - private String dbConnectionUrl; - private Boolean simpleConnectionUrl; + private String datasourceUrl; + private Boolean isSimpleConnectionUrl; + private String username; + private String password; + private String driverClassName; public String toCompletedJdbcUrl() { - if (ObjectUtil.isEmpty(simpleConnectionUrl) || simpleConnectionUrl == false) { - return String.format("%s/%s?%s", dbConnectionUrl, schemaName, connectionParameters); + if (ObjectUtil.isEmpty(isSimpleConnectionUrl) || isSimpleConnectionUrl == false) { + return String.format("%s/%s?%s", datasourceUrl, schemaName, connectionParameters); } - return dbConnectionUrl; + return datasourceUrl; } } diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/properties/HibernateProperties.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/properties/HibernateProperties.java index 8de1649..4543d8a 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/properties/HibernateProperties.java +++ b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/properties/HibernateProperties.java @@ -3,6 +3,9 @@ package com.myoa.engineering.crawl.ppomppu.processor.configuration.properties; import java.util.List; import lombok.Getter; import lombok.Setter; + +import org.hibernate.dialect.MySQL8Dialect; +import org.hibernate.dialect.MySQLDialect; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.stereotype.Component; diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/controller/CrawlAPIController.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/controller/CrawlAPIController.java index 3592278..1e72f78 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/controller/CrawlAPIController.java +++ b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/controller/CrawlAPIController.java @@ -1,17 +1,21 @@ package com.myoa.engineering.crawl.ppomppu.processor.controller; -import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle; -import com.myoa.engineering.crawl.ppomppu.processor.dto.FeedParsedResult; -import com.myoa.engineering.crawl.ppomppu.processor.service.PpomppuArticleService; -import com.myoa.engineering.crawl.ppomppu.processor.service.PpomppuFeedService; -import com.myoa.engineering.crawl.ppomppu.support.dto.APIResponse; -import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName; import java.util.List; -import lombok.extern.slf4j.Slf4j; + import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; + +import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle; +import com.myoa.engineering.crawl.ppomppu.processor.dto.FeedParsedResult; +import com.myoa.engineering.crawl.ppomppu.processor.service.MessageSenderService; +import com.myoa.engineering.crawl.ppomppu.processor.service.PpomppuArticleService; +import com.myoa.engineering.crawl.ppomppu.processor.service.PpomppuFeedService; +import com.myoa.engineering.crawl.ppomppu.support.dto.APIResponse; +import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName; + +import lombok.extern.slf4j.Slf4j; import reactor.core.publisher.Mono; /** @@ -27,20 +31,25 @@ public class CrawlAPIController { private final PpomppuFeedService ppomppuRSSFeedService; private final PpomppuArticleService ppomppuArticleService; + private final MessageSenderService messageSenderService; public CrawlAPIController(PpomppuFeedService ppomppuRSSFeedService, - PpomppuArticleService ppomppuArticleService) { + PpomppuArticleService ppomppuArticleService, + MessageSenderService messageSenderService) { this.ppomppuRSSFeedService = ppomppuRSSFeedService; this.ppomppuArticleService = ppomppuArticleService; + this.messageSenderService = messageSenderService; } @PostMapping("/boards/{boardName}") public Mono> crawlBoard(@PathVariable("boardName") PpomppuBoardName boardName) { log.info("got request... {}", boardName); FeedParsedResult result = FeedParsedResult.of(boardName); - Mono> articles = ppomppuRSSFeedService.getArticles(boardName) - .doOnNext(e -> ppomppuArticleService.filterOnlyNewArticles(boardName, e)) - .doOnNext(e -> ppomppuArticleService.save(boardName, e)); + Mono> articles = + ppomppuRSSFeedService.getArticles(boardName) + .doOnNext(e -> ppomppuArticleService.filterOnlyNewArticles(boardName, e)) + .doOnNext(e -> ppomppuArticleService.save(boardName, e)) + .doOnNext(messageSenderService::sendMessageToSlack); return articles.then(Mono.just(APIResponse.success(result.done()))); } diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/PpomppuArticleParser.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/PpomppuArticleParser.java new file mode 100644 index 0000000..20dc7d0 --- /dev/null +++ b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/PpomppuArticleParser.java @@ -0,0 +1,79 @@ +package com.myoa.engineering.crawl.ppomppu.processor.dto; + +import java.time.Instant; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; + +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; + +import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle; +import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName; + +/** + * PpomppuArticleTransformer + * + * @author Shin Woo-jin (woozu.shin@kakaoent.com) + * @since 2021-09-08 + */ +public final class PpomppuArticleParser { + + private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yy.MM.dd HH:mm:ss") + .withZone(ZoneId.of("Asia/Seoul")); + + private PpomppuArticleParser() {} + + public static PpomppuArticle toArticle(Elements articleElement) { + final long articleId = PpomppuArticleParser.parseArticleId(articleElement.get(0)); + final String title = PpomppuArticleParser.parseTitle(articleElement.get(2)); + final String articleUrl = PpomppuArticleParser.parseArticleUrl(articleElement.get(2)); + final int recommended = PpomppuArticleParser.parseRecommended(articleElement.get(6)); + final int hit = PpomppuArticleParser.parseHit(articleElement.get(7)); + final Instant registeredAt = PpomppuArticleParser.parseRegisteredAt(articleElement.get(5)); + + return PpomppuArticle.builder() + .articleId(articleId) + .title(title) + .articleUrl(articleUrl) + .recommended(recommended) + .hit(hit) + .registeredAt(registeredAt) + .build(); + } + + public static Long parseArticleId(Element td) { + return Long.parseLong(td.text().trim()); + } + + public static String parseTitle(Element td) { + return td.text(); + } + + public static String parseArticleUrl(Element td) { + return PpomppuBoardName.ofViewPageUrl(td.getElementsByTag("a").attr("href")); + } + + public static Integer parseRecommended(Element td) { + final String voteString = td.text(); + final int recommended; + + if (voteString.isEmpty()) { + recommended = 0; + } else { + final int voteUp = Integer.parseInt(td.text().split(" - ")[0]); + final int voteDown = Integer.parseInt(td.text().split(" - ")[1]); + recommended = voteUp - voteDown; + } + return recommended; + } + + public static Integer parseHit(Element td) { + return Integer.parseInt(td.text()); + } + + public static Instant parseRegisteredAt(Element td) { + final String registeredAtString = td.attr("title"); + return DATE_TIME_FORMATTER.parse(registeredAtString, Instant::from); + } + +} diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/PpomppuArticleTransformer.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/PpomppuArticleTransformer.java index 18bd30f..95f4e8a 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/PpomppuArticleTransformer.java +++ b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/PpomppuArticleTransformer.java @@ -1,77 +1,50 @@ package com.myoa.engineering.crawl.ppomppu.processor.dto; -import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle; -import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName; import java.time.Instant; import java.time.ZoneId; import java.time.format.DateTimeFormatter; -import org.jsoup.nodes.Element; -import org.jsoup.select.Elements; +import java.util.List; +import java.util.function.Function; +import java.util.stream.Collectors; + +import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle; +import com.myoa.engineering.crawl.ppomppu.support.dto.SimpleMessageDTO; /** * PpomppuArticleTransformer + * @author Shin Woo-jin (woo-jin.shin@linecorp.com) + * @since 2021-11-21 * - * @author Shin Woo-jin (woozu.shin@kakaoent.com) - * @since 2021-09-08 */ public final class PpomppuArticleTransformer { - - private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yy.MM.dd HH:mm:ss") - .withZone(ZoneId.of("Asia/Seoul")); - private PpomppuArticleTransformer() {} - public static PpomppuArticle toArticle(Elements articleElement) { - final long articleId = PpomppuArticleTransformer.toArticleId(articleElement.get(0)); - final String title = PpomppuArticleTransformer.toTitle(articleElement.get(2)); - final String articleUrl = PpomppuArticleTransformer.toArticleUrl(articleElement.get(2)); - final int recommended = PpomppuArticleTransformer.toRecommended(articleElement.get(6)); - final int hit = PpomppuArticleTransformer.toHit(articleElement.get(7)); - final Instant registeredAt = PpomppuArticleTransformer.toRegisteredAt(articleElement.get(5)); + private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss") + .withZone(ZoneId.of("Asia/Seoul")); - return PpomppuArticle.builder() - .articleId(articleId) - .title(title) - .articleUrl(articleUrl) - .recommended(recommended) - .hit(hit) - .registeredAt(registeredAt) - .build(); + public static final Function TRANSFORM_TO_MESSAGE_DTO = entity -> + SimpleMessageDTO.builder() + .requestedAt(Instant.now()) + .publishedAt(entity.getRegisteredAt()) + .title(String.format("[%s] %s", entity.getBoardName().getMenuName(), entity.getTitle())) + .body(entity.getArticleUrl()) + .build(); + + // https://stackoverflow.com/questions/24882927/using-streams-to-convert-a-list-of-objects-into-a-string-obtained-from-the-tostr + public static SimpleMessageDTO transform(List articles) { + Instant requestedAt = Instant.now(); + String body = articles.stream() + .map(PpomppuArticleTransformer::convertToInlineMessage) + .collect(Collectors.joining("\n\n")); + return SimpleMessageDTO.builder() + .requestedAt(requestedAt) + .title(DATE_TIME_FORMATTER.format(requestedAt)) + .body(body) + .build(); } - public static Long toArticleId(Element td) { - return Long.parseLong(td.text().trim()); + public static String convertToInlineMessage(PpomppuArticle article) { + return String.format("[%s] %s\n%s", + article.getBoardName().getMenuName(), article.getTitle(), article.getArticleUrl()); } - - public static String toTitle(Element td) { - return td.text(); - } - - public static String toArticleUrl(Element td) { - return PpomppuBoardName.ofViewPageUrl(td.getElementsByTag("a").attr("href")); - } - - public static Integer toRecommended(Element td) { - final String voteString = td.text(); - final int recommended; - - if (voteString.isEmpty()) { - recommended = 0; - } else { - final int voteUp = Integer.parseInt(td.text().split(" - ")[0]); - final int voteDown = Integer.parseInt(td.text().split(" - ")[1]); - recommended = voteUp - voteDown; - } - return recommended; - } - - public static Integer toHit(Element td) { - return Integer.parseInt(td.text()); - } - - public static Instant toRegisteredAt(Element td) { - final String registeredAtString = td.attr("title"); - return DATE_TIME_FORMATTER.parse(registeredAtString, Instant::from); - } - } diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/client/SenderAPIClient.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/client/MessageSenderAPIClient.java similarity index 57% rename from processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/client/SenderAPIClient.java rename to processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/client/MessageSenderAPIClient.java index e9de2a8..1fd80ae 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/client/SenderAPIClient.java +++ b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/client/MessageSenderAPIClient.java @@ -2,13 +2,17 @@ package com.myoa.engineering.crawl.ppomppu.processor.infrastructure.client; import org.springframework.stereotype.Component; import org.springframework.web.reactive.function.client.WebClient; +import org.springframework.web.reactive.function.client.WebClientRequestException; import com.myoa.engineering.crawl.ppomppu.processor.dto.constant.WebClientPropertiesUnitName; +import com.myoa.engineering.crawl.ppomppu.support.dto.SimpleMessageDTO; import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebClientFilterFactory; import com.myoa.engineering.crawl.ppomppu.support.webclient.properties.WebClientProperties; import com.myoa.engineering.crawl.ppomppu.support.webclient.properties.WebClientProperties.WebClientPropertiesUnit; import lombok.extern.slf4j.Slf4j; +import reactor.core.publisher.Mono; +import reactor.core.scheduler.Schedulers; /** * PpomppuNotifierSenderAPIClient @@ -18,11 +22,11 @@ import lombok.extern.slf4j.Slf4j; */ @Slf4j @Component -public class SenderAPIClient { +public class MessageSenderAPIClient { private final WebClient webClient; - public SenderAPIClient(WebClientProperties webClientProperties) { + public MessageSenderAPIClient(WebClientProperties webClientProperties) { WebClientPropertiesUnit webClientPropertiesUnit = webClientProperties.find(WebClientPropertiesUnitName.PPOMPPU_NOTIFIER_SENDER_API.getUnitName()); this.webClient = WebClient.builder() @@ -31,6 +35,16 @@ public class SenderAPIClient { .filter(WebClientFilterFactory.logResponse()) .build(); } -} -// TODO webclient properties \ No newline at end of file + public Mono sendMessageToSlack(SimpleMessageDTO dto) { + return webClient.post() + .uri("/api/v1/messages/sendMessage/messengers/slack") + .bodyValue(dto) + .exchangeToMono(e -> e.bodyToMono(String.class)) + .publishOn(Schedulers.boundedElastic()) + .onErrorResume(WebClientRequestException.class, t -> { + log.info("Exception occured, ignoring. : {}", t.getClass().getSimpleName()); + return Mono.empty(); + }); + } +} \ No newline at end of file diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/client/PpomppuBoardFeedRetriever.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/client/PpomppuBoardFeedRetriever.java index 9347c63..9db4dab 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/client/PpomppuBoardFeedRetriever.java +++ b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/client/PpomppuBoardFeedRetriever.java @@ -40,8 +40,8 @@ public class PpomppuBoardFeedRetriever { .onErrorResume(WebClientRequestException.class, t -> { log.info("Exception occured, ignoring. : {}", t.getClass().getSimpleName()); return Mono.empty(); - }) - .doOnNext(e -> log.info("[getHtml] {}", e)); + }); + // .doOnNext(e -> log.info("[getHtml] {}", e)); } } diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/MessageSenderService.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/MessageSenderService.java new file mode 100644 index 0000000..60f1d0d --- /dev/null +++ b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/MessageSenderService.java @@ -0,0 +1,37 @@ +package com.myoa.engineering.crawl.ppomppu.processor.service; + +import java.util.List; + +import org.springframework.stereotype.Service; + +import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle; +import com.myoa.engineering.crawl.ppomppu.processor.dto.PpomppuArticleTransformer; +import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.client.MessageSenderAPIClient; + +import lombok.extern.slf4j.Slf4j; + +/** + * MessageSenderService + * @author Shin Woo-jin (woo-jin.shin@linecorp.com) + * @since 2021-11-21 + * + */ +@Slf4j +@Service +public class MessageSenderService { + + private final MessageSenderAPIClient messageSenderAPIClient; + + public MessageSenderService(MessageSenderAPIClient messageSenderAPIClient) { + this.messageSenderAPIClient = messageSenderAPIClient; + } + + public void sendMessageToSlack(PpomppuArticle article) { + messageSenderAPIClient.sendMessageToSlack(PpomppuArticleTransformer.TRANSFORM_TO_MESSAGE_DTO.apply(article)); + } + + public void sendMessageToSlack(List articles) { + messageSenderAPIClient.sendMessageToSlack(PpomppuArticleTransformer.transform(articles)); + } + +} diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/PpomppuFeedService.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/PpomppuFeedService.java index 461bd2f..b8a3568 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/PpomppuFeedService.java +++ b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/PpomppuFeedService.java @@ -1,7 +1,7 @@ package com.myoa.engineering.crawl.ppomppu.processor.service; import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle; -import com.myoa.engineering.crawl.ppomppu.processor.dto.PpomppuArticleTransformer; +import com.myoa.engineering.crawl.ppomppu.processor.dto.PpomppuArticleParser; import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.client.PpomppuBoardFeedRetriever; import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName; import java.util.List; @@ -30,11 +30,11 @@ public class PpomppuFeedService { public Mono> getArticles(PpomppuBoardName boardName) { final Mono html = ppomppuBoardFeedRetriever.getHtml(boardName.getResourcePath()); - final Mono tbody = extractTbodyFromHtml(html) - .doOnNext(e -> log.info("pre tbody - {}", e.html())); + final Mono tbody = extractTbodyFromHtml(html); + // .doOnNext(e -> log.info("pre tbody - {}", e.html())); return extractArticlesFromTbody(tbody).map(this::convertFromElement) .map(e -> e.updateBoardName(boardName)) - .doOnNext(e -> log.info("parsed Result: {}", e)) + // .doOnNext(e -> log.info("parsed Result: {}", e)) .collectList(); } @@ -42,7 +42,7 @@ public class PpomppuFeedService { return html.map(Jsoup::parse) .mapNotNull(e -> e.getElementById("revolution_main_table")) .map(e -> e.getElementsByTag("tbody")) - .doOnNext(e -> log.info("tbody - {}", e.html())) + // .doOnNext(e -> log.info("tbody - {}", e.html())) .map(e -> e.stream() .findFirst() .orElseThrow(() -> new IndexOutOfBoundsException("no tbody"))); @@ -54,6 +54,6 @@ public class PpomppuFeedService { } private PpomppuArticle convertFromElement(Element element) { - return PpomppuArticleTransformer.toArticle(element.getElementsByTag("td")); + return PpomppuArticleParser.toArticle(element.getElementsByTag("td")); } } diff --git a/processor/src/main/resources/application-development.yml b/processor/src/main/resources/application-development.yml index 6341010..8ceb30d 100644 --- a/processor/src/main/resources/application-development.yml +++ b/processor/src/main/resources/application-development.yml @@ -5,9 +5,7 @@ spring: import: - "configserver:http://192.168.0.100:11080" - classpath:/development/webclient.yml - profiles: - group: - development: development, datasource-development + server: port: 20081 diff --git a/processor/src/main/resources/application-production.yml b/processor/src/main/resources/application-production.yml index 892c27d..aa661d6 100644 --- a/processor/src/main/resources/application-production.yml +++ b/processor/src/main/resources/application-production.yml @@ -1,10 +1,6 @@ spring: config: activate: - on-profile: development + on-profile: production import: - "configserver:http://192.168.0.100:11080" - - classpath:/development/webclient.yml - profiles: - group: - development: development, datasource-development diff --git a/processor/src/main/resources/application.yml b/processor/src/main/resources/application.yml index b8b84f6..0cc3fd9 100644 --- a/processor/src/main/resources/application.yml +++ b/processor/src/main/resources/application.yml @@ -5,10 +5,12 @@ spring: allow-bean-definition-overriding: true profiles: active: ${SPRING_ACTIVE_PROFILE:development} + group: + development: "development,datasource-development" + production: "production, datasource-production" freemarker: enabled: false - server: port: 20080 error: diff --git a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/controller/MessageSenderAPIController.java b/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/controller/MessageSenderAPIController.java new file mode 100644 index 0000000..a049fe1 --- /dev/null +++ b/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/controller/MessageSenderAPIController.java @@ -0,0 +1,31 @@ +package com.myoa.engineering.crawl.ppomppu.sender.controller; + +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import com.myoa.engineering.crawl.ppomppu.support.dto.APIResponse; +import com.myoa.engineering.crawl.ppomppu.support.dto.SimpleMessageDTO; + +import lombok.extern.slf4j.Slf4j; +import reactor.core.publisher.Mono; + +/** + * MessageSenderAPIController + * @author Shin Woo-jin (woo-jin.shin@linecorp.com) + * @since 2021-11-21 + * + */ +@Slf4j +@RestController +@RequestMapping("/api/v1") +public class MessageSenderAPIController { + + @PostMapping("/messages/sendMessage/messengers/slack") + public Mono> sendMessageToSlack(SimpleMessageDTO dto) { + + log.info("received : {}, \nbody: {}", dto.getTitle(), dto.getBody()); + // TODO transform + return Mono.just(APIResponse.success(dto)); + } +} diff --git a/sender/src/main/resources/application-development.yml b/sender/src/main/resources/application-development.yml index b821007..da50d27 100644 --- a/sender/src/main/resources/application-development.yml +++ b/sender/src/main/resources/application-development.yml @@ -3,12 +3,8 @@ spring: activate: on-profile: development import: - - classpath:/development/webclient.yml - "configserver:http://192.168.0.100:11080" # import: optional:configserver:http://localhost:11080 # can be start up even config server was not found. - profiles: - group: - development: development, slackapi-development server: - port: 20081 + port: 20082 diff --git a/sender/src/main/resources/application-production.yml b/sender/src/main/resources/application-production.yml index 5028c79..aa661d6 100644 --- a/sender/src/main/resources/application-production.yml +++ b/sender/src/main/resources/application-production.yml @@ -2,5 +2,5 @@ spring: config: activate: on-profile: production -# import: -# - classpath:/production/webclient.yml \ No newline at end of file + import: + - "configserver:http://192.168.0.100:11080" diff --git a/sender/src/main/resources/application.yml b/sender/src/main/resources/application.yml index cb42b9d..ee15561 100644 --- a/sender/src/main/resources/application.yml +++ b/sender/src/main/resources/application.yml @@ -5,9 +5,17 @@ spring: allow-bean-definition-overriding: true profiles: active: ${SPRING_ACTIVE_PROFILE:development} + group: + development: development, slackapi-development + production: production, slackapi-production freemarker: enabled: false + jpa: + hibernate: + ddl-auto: create-drop + generate-ddl: true + server: port: 20080 error: diff --git a/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/dto/SimpleMessageDTO.java b/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/dto/SimpleMessageDTO.java new file mode 100644 index 0000000..1a2c026 --- /dev/null +++ b/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/dto/SimpleMessageDTO.java @@ -0,0 +1,37 @@ + package com.myoa.engineering.crawl.ppomppu.support.dto; + +import java.io.Serializable; +import java.time.Instant; + +import lombok.Builder; +import lombok.Getter; +import lombok.NoArgsConstructor; + +/** + * SimpleMessageDTO + * @author Shin Woo-jin (woo-jin.shin@linecorp.com) + * @since 2021-11-21 + * + */ +@Getter +@NoArgsConstructor +public class SimpleMessageDTO implements Serializable { + + private static final long serialVersionUID = 2203955567672404428L; + + private String title; + private String body; + private String url; + private Instant publishedAt; + private Instant requestedAt; + + @Builder + public SimpleMessageDTO(String title, String body, String url, Instant publishedAt, Instant requestedAt) { + this.title = title; + this.body = body; + this.url = url; + this.publishedAt = publishedAt; + this.requestedAt = requestedAt; + } + +} From c97c8dc01fff0e52794ded9ceada00084e20e228 Mon Sep 17 00:00:00 2001 From: "woozu.shin" Date: Sun, 21 Nov 2021 20:53:20 +0900 Subject: [PATCH 5/6] Implement MessageSenderService --- .../configuration/H2ConsoleConfiguration.java | 2 +- .../controller/CrawlAPIController.java | 4 +- .../client/MessageSenderAPIClient.java | 21 ++++++---- .../service/MessageSenderService.java | 9 +++-- .../service/PpomppuArticleService.java | 4 +- .../src/main/resources/application-local.yml | 12 ++++++ processor/src/main/resources/application.yml | 3 +- .../main/resources/development/webclient.yml | 2 +- .../src/main/resources/local/webclient.yml | 5 +++ .../src/main/resources/logback-spring.xml | 4 ++ .../main/resources/production/webclient.yml | 2 +- .../src/main/resources/application-local.yml | 7 ++++ receiver/src/main/resources/application.yml | 2 +- .../src/main/resources/local/webclient.yml | 5 +++ .../src/main/resources/logback-spring.xml | 4 ++ .../properties/SlackSecretProperties.java | 34 ++++++++++++++++ .../MessageSenderAPIController.java | 19 +++++---- .../sender/controller/TestAPIController.java | 20 +++------- .../ppomppu/sender/dto/SlackMessageDTO.java | 3 ++ .../MongeShoppingBotSlackMessageSender.java | 39 +++++++++++++++++++ .../client/SlackMessageSender.java | 31 --------------- .../src/main/resources/application-local.yml | 10 +++++ sender/src/main/resources/application.yml | 3 +- sender/src/main/resources/logback-spring.xml | 4 ++ 24 files changed, 175 insertions(+), 74 deletions(-) create mode 100644 processor/src/main/resources/application-local.yml create mode 100644 processor/src/main/resources/local/webclient.yml create mode 100644 receiver/src/main/resources/application-local.yml create mode 100644 receiver/src/main/resources/local/webclient.yml create mode 100644 sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/configuration/properties/SlackSecretProperties.java create mode 100644 sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/infrastructure/client/MongeShoppingBotSlackMessageSender.java create mode 100644 sender/src/main/resources/application-local.yml diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/H2ConsoleConfiguration.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/H2ConsoleConfiguration.java index 993d7f8..3fd4bd3 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/H2ConsoleConfiguration.java +++ b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/H2ConsoleConfiguration.java @@ -11,7 +11,7 @@ import org.springframework.context.event.ContextRefreshedEvent; import org.springframework.context.event.EventListener; @Slf4j -@Profile("datasource-development") +@Profile({"datasource-local", "datasource-development"}) @Configuration public class H2ConsoleConfiguration { diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/controller/CrawlAPIController.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/controller/CrawlAPIController.java index 1e72f78..c520aad 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/controller/CrawlAPIController.java +++ b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/controller/CrawlAPIController.java @@ -48,8 +48,8 @@ public class CrawlAPIController { Mono> articles = ppomppuRSSFeedService.getArticles(boardName) .doOnNext(e -> ppomppuArticleService.filterOnlyNewArticles(boardName, e)) - .doOnNext(e -> ppomppuArticleService.save(boardName, e)) - .doOnNext(messageSenderService::sendMessageToSlack); + .doOnNext(e -> messageSenderService.sendMessageToSlack(e)) + .doOnNext(e -> ppomppuArticleService.save(boardName, e)); return articles.then(Mono.just(APIResponse.success(result.done()))); } diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/client/MessageSenderAPIClient.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/client/MessageSenderAPIClient.java index 1fd80ae..5110c3e 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/client/MessageSenderAPIClient.java +++ b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/client/MessageSenderAPIClient.java @@ -1,24 +1,26 @@ package com.myoa.engineering.crawl.ppomppu.processor.infrastructure.client; -import org.springframework.stereotype.Component; -import org.springframework.web.reactive.function.client.WebClient; -import org.springframework.web.reactive.function.client.WebClientRequestException; - import com.myoa.engineering.crawl.ppomppu.processor.dto.constant.WebClientPropertiesUnitName; import com.myoa.engineering.crawl.ppomppu.support.dto.SimpleMessageDTO; import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebClientFilterFactory; +import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebFluxExchangeStragiesFactory; import com.myoa.engineering.crawl.ppomppu.support.webclient.properties.WebClientProperties; import com.myoa.engineering.crawl.ppomppu.support.webclient.properties.WebClientProperties.WebClientPropertiesUnit; - import lombok.extern.slf4j.Slf4j; +import org.springframework.core.ParameterizedTypeReference; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; +import org.springframework.stereotype.Component; +import org.springframework.web.reactive.function.client.WebClient; +import org.springframework.web.reactive.function.client.WebClientRequestException; import reactor.core.publisher.Mono; import reactor.core.scheduler.Schedulers; /** * PpomppuNotifierSenderAPIClient + * * @author Shin Woo-jin (woo-jin.shin@linecorp.com) * @since 2021-11-17 - * */ @Slf4j @Component @@ -31,6 +33,8 @@ public class MessageSenderAPIClient { webClientProperties.find(WebClientPropertiesUnitName.PPOMPPU_NOTIFIER_SENDER_API.getUnitName()); this.webClient = WebClient.builder() .baseUrl(webClientPropertiesUnit.getBaseUrl()) + .exchangeStrategies(WebFluxExchangeStragiesFactory.ofDefault()) + .defaultHeader(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_VALUE) .filter(WebClientFilterFactory.logRequest()) .filter(WebClientFilterFactory.logResponse()) .build(); @@ -40,11 +44,12 @@ public class MessageSenderAPIClient { return webClient.post() .uri("/api/v1/messages/sendMessage/messengers/slack") .bodyValue(dto) - .exchangeToMono(e -> e.bodyToMono(String.class)) + .exchangeToMono(e -> e.bodyToMono(new ParameterizedTypeReference() {})) .publishOn(Schedulers.boundedElastic()) .onErrorResume(WebClientRequestException.class, t -> { log.info("Exception occured, ignoring. : {}", t.getClass().getSimpleName()); return Mono.empty(); - }); + }) + .doOnNext(e -> log.info("response: {} ", e)); } } \ No newline at end of file diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/MessageSenderService.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/MessageSenderService.java index 60f1d0d..2a6d0ac 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/MessageSenderService.java +++ b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/MessageSenderService.java @@ -9,6 +9,7 @@ import com.myoa.engineering.crawl.ppomppu.processor.dto.PpomppuArticleTransforme import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.client.MessageSenderAPIClient; import lombok.extern.slf4j.Slf4j; +import reactor.core.publisher.Mono; /** * MessageSenderService @@ -26,12 +27,12 @@ public class MessageSenderService { this.messageSenderAPIClient = messageSenderAPIClient; } - public void sendMessageToSlack(PpomppuArticle article) { - messageSenderAPIClient.sendMessageToSlack(PpomppuArticleTransformer.TRANSFORM_TO_MESSAGE_DTO.apply(article)); + public Mono sendMessageToSlack(PpomppuArticle article) { + return messageSenderAPIClient.sendMessageToSlack(PpomppuArticleTransformer.TRANSFORM_TO_MESSAGE_DTO.apply(article)); } - public void sendMessageToSlack(List articles) { - messageSenderAPIClient.sendMessageToSlack(PpomppuArticleTransformer.transform(articles)); + public Mono sendMessageToSlack(List articles) { + return messageSenderAPIClient.sendMessageToSlack(PpomppuArticleTransformer.transform(articles)); } } diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/PpomppuArticleService.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/PpomppuArticleService.java index 632e3c3..e810c56 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/PpomppuArticleService.java +++ b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/PpomppuArticleService.java @@ -38,7 +38,7 @@ public class PpomppuArticleService { } @Transactional - public void save(PpomppuBoardName boardName, List articles) { + public List save(PpomppuBoardName boardName, List articles) { Long latestArticleId = articles.stream() .map(PpomppuArticle::getArticleId) .max(Long::compareTo) @@ -54,6 +54,6 @@ public class PpomppuArticleService { latestArticleId))); // save real articles. - ppomppuArticleRepository.saveAll(articles); + return ppomppuArticleRepository.saveAll(articles); } } diff --git a/processor/src/main/resources/application-local.yml b/processor/src/main/resources/application-local.yml new file mode 100644 index 0000000..c2198e0 --- /dev/null +++ b/processor/src/main/resources/application-local.yml @@ -0,0 +1,12 @@ +spring: + config: + activate: + on-profile: local + import: + - "configserver:http://localhost:20085" + - classpath:/local/webclient.yml + +server: + port: 20081 + + # import: optional:configserver:http://localhost:11080 # can be start up even config server was not found. \ No newline at end of file diff --git a/processor/src/main/resources/application.yml b/processor/src/main/resources/application.yml index 0cc3fd9..d136cf7 100644 --- a/processor/src/main/resources/application.yml +++ b/processor/src/main/resources/application.yml @@ -4,8 +4,9 @@ spring: main: allow-bean-definition-overriding: true profiles: - active: ${SPRING_ACTIVE_PROFILE:development} + active: ${SPRING_ACTIVE_PROFILE:local} group: + local: "local,datasource-local" development: "development,datasource-development" production: "production, datasource-production" freemarker: diff --git a/processor/src/main/resources/development/webclient.yml b/processor/src/main/resources/development/webclient.yml index d600658..06ec4a9 100644 --- a/processor/src/main/resources/development/webclient.yml +++ b/processor/src/main/resources/development/webclient.yml @@ -2,4 +2,4 @@ webclient: init: true units: - unit-name: ppn-sender-api - base-url: http://localhost:20081 \ No newline at end of file + base-url: http://localhost:20082 \ No newline at end of file diff --git a/processor/src/main/resources/local/webclient.yml b/processor/src/main/resources/local/webclient.yml new file mode 100644 index 0000000..06ec4a9 --- /dev/null +++ b/processor/src/main/resources/local/webclient.yml @@ -0,0 +1,5 @@ +webclient: + init: true + units: + - unit-name: ppn-sender-api + base-url: http://localhost:20082 \ No newline at end of file diff --git a/processor/src/main/resources/logback-spring.xml b/processor/src/main/resources/logback-spring.xml index f0fcb6c..907a4f5 100644 --- a/processor/src/main/resources/logback-spring.xml +++ b/processor/src/main/resources/logback-spring.xml @@ -1,6 +1,10 @@ + + + + diff --git a/processor/src/main/resources/production/webclient.yml b/processor/src/main/resources/production/webclient.yml index 862c2b9..373ba18 100644 --- a/processor/src/main/resources/production/webclient.yml +++ b/processor/src/main/resources/production/webclient.yml @@ -2,4 +2,4 @@ webclient: init: true units: - unit-name: ppn-sender-api - base-url: http://ppn_sender:20081 \ No newline at end of file + base-url: http://ppn_sender:20082 \ No newline at end of file diff --git a/receiver/src/main/resources/application-local.yml b/receiver/src/main/resources/application-local.yml new file mode 100644 index 0000000..297ac57 --- /dev/null +++ b/receiver/src/main/resources/application-local.yml @@ -0,0 +1,7 @@ +spring: + config: + activate: + on-profile: local + import: + - classpath:/local/webclient.yml + - "configserver:http://localhost:20085" \ No newline at end of file diff --git a/receiver/src/main/resources/application.yml b/receiver/src/main/resources/application.yml index 61ebab9..a89a769 100644 --- a/receiver/src/main/resources/application.yml +++ b/receiver/src/main/resources/application.yml @@ -4,7 +4,7 @@ spring: main: allow-bean-definition-overriding: true profiles: - active: ${SPRING_ACTIVE_PROFILE:development} + active: ${SPRING_ACTIVE_PROFILE:local} freemarker: enabled: false diff --git a/receiver/src/main/resources/local/webclient.yml b/receiver/src/main/resources/local/webclient.yml new file mode 100644 index 0000000..9fbfe3a --- /dev/null +++ b/receiver/src/main/resources/local/webclient.yml @@ -0,0 +1,5 @@ +webclient: + init: true + units: + - unit-name: ppn-processor-api + base-url: http://localhost:20081 \ No newline at end of file diff --git a/receiver/src/main/resources/logback-spring.xml b/receiver/src/main/resources/logback-spring.xml index f0fcb6c..907a4f5 100644 --- a/receiver/src/main/resources/logback-spring.xml +++ b/receiver/src/main/resources/logback-spring.xml @@ -1,6 +1,10 @@ + + + + diff --git a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/configuration/properties/SlackSecretProperties.java b/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/configuration/properties/SlackSecretProperties.java new file mode 100644 index 0000000..d47983c --- /dev/null +++ b/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/configuration/properties/SlackSecretProperties.java @@ -0,0 +1,34 @@ +package com.myoa.engineering.crawl.ppomppu.sender.configuration.properties; + +import java.util.List; +import lombok.Data; +import lombok.Getter; +import lombok.Setter; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +@Getter +@Setter +@Component +@ConfigurationProperties("infra.slack.bot") +public class SlackSecretProperties { + + private List units; + + @Data + public static class SlackSecretPropertiesUnit { + + private String botName; + private String username; + private String iconEmoji; + private String channel; + private String token; + } + + public SlackSecretPropertiesUnit find(String botUnitName) { + return units.stream() + .filter(e -> e.getBotName().equals(botUnitName)) + .findFirst() + .orElseThrow(() -> new IllegalArgumentException("not found bot unit name : " + botUnitName)); + } +} diff --git a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/controller/MessageSenderAPIController.java b/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/controller/MessageSenderAPIController.java index a049fe1..373e5bf 100644 --- a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/controller/MessageSenderAPIController.java +++ b/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/controller/MessageSenderAPIController.java @@ -1,13 +1,12 @@ package com.myoa.engineering.crawl.ppomppu.sender.controller; +import com.myoa.engineering.crawl.ppomppu.sender.infrastructure.client.MongeShoppingBotSlackMessageSender; +import com.myoa.engineering.crawl.ppomppu.support.dto.APIResponse; +import com.myoa.engineering.crawl.ppomppu.support.dto.SimpleMessageDTO; +import lombok.extern.slf4j.Slf4j; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; - -import com.myoa.engineering.crawl.ppomppu.support.dto.APIResponse; -import com.myoa.engineering.crawl.ppomppu.support.dto.SimpleMessageDTO; - -import lombok.extern.slf4j.Slf4j; import reactor.core.publisher.Mono; /** @@ -21,11 +20,17 @@ import reactor.core.publisher.Mono; @RequestMapping("/api/v1") public class MessageSenderAPIController { + private final MongeShoppingBotSlackMessageSender sender; + + public MessageSenderAPIController(MongeShoppingBotSlackMessageSender sender) { + this.sender = sender; + } + @PostMapping("/messages/sendMessage/messengers/slack") public Mono> sendMessageToSlack(SimpleMessageDTO dto) { log.info("received : {}, \nbody: {}", dto.getTitle(), dto.getBody()); - // TODO transform - return Mono.just(APIResponse.success(dto)); + return sender.sendMessage(sender.ofMessage(dto.getBody())) + .then(Mono.just(APIResponse.success(dto))); } } diff --git a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/controller/TestAPIController.java b/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/controller/TestAPIController.java index fad3683..96fcc55 100644 --- a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/controller/TestAPIController.java +++ b/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/controller/TestAPIController.java @@ -1,37 +1,29 @@ package com.myoa.engineering.crawl.ppomppu.sender.controller; +import com.myoa.engineering.crawl.ppomppu.sender.infrastructure.client.MongeShoppingBotSlackMessageSender; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; - -import com.myoa.engineering.crawl.ppomppu.sender.dto.SlackMessageDTO; -import com.myoa.engineering.crawl.ppomppu.sender.infrastructure.client.SlackMessageSender; - import reactor.core.publisher.Mono; /** * TestAPIController + * * @author Shin Woo-jin (woo-jin.shin@linecorp.com) * @since 2021-11-15 - * */ @RestController @RequestMapping("/api/v1") public class TestAPIController { - private final SlackMessageSender sender; + private final MongeShoppingBotSlackMessageSender sender; - public TestAPIController() { - this.sender = new SlackMessageSender("xoxb-2688454277126-2695026012277-K2Ib13lKokmTiBSnSMrc0Bp2"); + public TestAPIController(MongeShoppingBotSlackMessageSender sender) { + this.sender = sender; } @GetMapping("/test") public Mono test() { - return sender.sendMessage(SlackMessageDTO.builder() - .text("test!") - .iconEmoji("monge") - .channel("notify_shopping") - .username("shopping notifier") - .build()); + return sender.sendMessage(sender.ofMessage("testtesttest!!!")); } } diff --git a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/dto/SlackMessageDTO.java b/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/dto/SlackMessageDTO.java index c02cb71..faf7ef5 100644 --- a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/dto/SlackMessageDTO.java +++ b/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/dto/SlackMessageDTO.java @@ -33,4 +33,7 @@ public class SlackMessageDTO implements MessageDTO { this.iconEmoji = iconEmoji; } + public void applyText(String text) { + this.text = text; + } } diff --git a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/infrastructure/client/MongeShoppingBotSlackMessageSender.java b/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/infrastructure/client/MongeShoppingBotSlackMessageSender.java new file mode 100644 index 0000000..8ab5b00 --- /dev/null +++ b/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/infrastructure/client/MongeShoppingBotSlackMessageSender.java @@ -0,0 +1,39 @@ +package com.myoa.engineering.crawl.ppomppu.sender.infrastructure.client; + +import com.myoa.engineering.crawl.ppomppu.sender.configuration.properties.SlackSecretProperties; +import com.myoa.engineering.crawl.ppomppu.sender.configuration.properties.SlackSecretProperties.SlackSecretPropertiesUnit; +import com.myoa.engineering.crawl.ppomppu.sender.dto.SlackMessageDTO; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; + +@Slf4j +@Component +public class MongeShoppingBotSlackMessageSender extends SlackMessageSender { + + private static final String SLACK_SECRET_UNIT_NAME = "monge_shopping_bot"; + + private final SlackSecretPropertiesUnit slackProperties; +// private final SlackMessageSender slackMessageSender; + + public MongeShoppingBotSlackMessageSender(SlackSecretProperties slackSecretProperties) { + super(slackSecretProperties.find(SLACK_SECRET_UNIT_NAME).getToken()); + this.slackProperties = slackSecretProperties.find(SLACK_SECRET_UNIT_NAME); + } + + public SlackMessageDTO ofMessageTemplate() { + return SlackMessageDTO.builder() + .channel(slackProperties.getChannel()) + .iconEmoji(slackProperties.getIconEmoji()) + .username(slackProperties.getUsername()) + .build(); + } + + public SlackMessageDTO ofMessage(String text) { + return SlackMessageDTO.builder() + .channel(slackProperties.getChannel()) + .iconEmoji(slackProperties.getIconEmoji()) + .username(slackProperties.getUsername()) + .text(text) + .build(); + } +} diff --git a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/infrastructure/client/SlackMessageSender.java b/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/infrastructure/client/SlackMessageSender.java index 1eef297..5f349bd 100644 --- a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/infrastructure/client/SlackMessageSender.java +++ b/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/infrastructure/client/SlackMessageSender.java @@ -53,34 +53,3 @@ public class SlackMessageSender implements MessageSender { } } - -/* -import requests -import json - -# woozuyeni_watchtower -SLACK_TOKEN="xoxb-2688454277126-2695026012277-K2Ib13lKokmTiBSnSMrc0Bp2" -CHANNEL = "notify_watchtower" -CHANNEL = "common" - - -POST_API = "https://slack.com/api/chat.postMessage" -HEADER = "Authorization: Bearer" - -headers = { - "Content-Type" : "application/json", - "Authorization" : f"Bearer {SLACK_TOKEN}" -} - -data = {} -data["text"] = "몽몽! :monge_big:" -data["channel"] = CHANNEL -data["username"] = "몽이봇" -data["icon_emoji"] = ":monge_big:" - -# requests.post(POST_API, data=json.dumps(data), headers=headers) - - -attachments = [{"title": "Cat", "image_url": image_url}] -image_url = "https://myoa-universe.com/assets/love1.png" - */ \ No newline at end of file diff --git a/sender/src/main/resources/application-local.yml b/sender/src/main/resources/application-local.yml new file mode 100644 index 0000000..5c6bfbb --- /dev/null +++ b/sender/src/main/resources/application-local.yml @@ -0,0 +1,10 @@ +spring: + config: + activate: + on-profile: local + import: + - "configserver:http://localhost:20085" + # import: optional:configserver:http://localhost:11080 # can be start up even config server was not found. + +server: + port: 20082 diff --git a/sender/src/main/resources/application.yml b/sender/src/main/resources/application.yml index ee15561..670365b 100644 --- a/sender/src/main/resources/application.yml +++ b/sender/src/main/resources/application.yml @@ -4,8 +4,9 @@ spring: main: allow-bean-definition-overriding: true profiles: - active: ${SPRING_ACTIVE_PROFILE:development} + active: ${SPRING_ACTIVE_PROFILE:local} group: + local: local, slackapi-local development: development, slackapi-development production: production, slackapi-production freemarker: diff --git a/sender/src/main/resources/logback-spring.xml b/sender/src/main/resources/logback-spring.xml index f0fcb6c..32276f3 100644 --- a/sender/src/main/resources/logback-spring.xml +++ b/sender/src/main/resources/logback-spring.xml @@ -1,5 +1,9 @@ + + + + From a0c3962e0dcefe92eeec58f09b4e87b0a10df730 Mon Sep 17 00:00:00 2001 From: "woo-jin.shin" Date: Sun, 21 Nov 2021 22:36:30 +0900 Subject: [PATCH 6/6] [PPN-211113] Fix bug --- .../controller/CrawlAPIController.java | 17 +++++----- .../processor/dto/PpomppuArticleParser.java | 2 +- .../dto/PpomppuArticleTransformer.java | 7 +++-- .../client/MessageSenderAPIClient.java | 21 +++++++------ .../service/PpomppuArticleService.java | 21 ++++++++----- .../receiver/scheduler/ParseEventEmitter.java | 14 +++------ .../MessageSenderAPIController.java | 5 ++- .../sender/controller/TestAPIController.java | 4 +++ .../MongeShoppingBotSlackMessageSender.java | 1 - .../client/SlackMessageSender.java | 4 +-- sender/src/main/resources/application.yml | 5 --- ...t_message_tempalte(block_kit_builder).json | 31 +++++++++++++++++++ .../support/dto/code/PpomppuBoardName.java | 6 ++-- 13 files changed, 85 insertions(+), 53 deletions(-) create mode 100644 support/next_message_tempalte(block_kit_builder).json diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/controller/CrawlAPIController.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/controller/CrawlAPIController.java index c520aad..2e92503 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/controller/CrawlAPIController.java +++ b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/controller/CrawlAPIController.java @@ -1,13 +1,11 @@ package com.myoa.engineering.crawl.ppomppu.processor.controller; -import java.util.List; - +import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; -import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle; import com.myoa.engineering.crawl.ppomppu.processor.dto.FeedParsedResult; import com.myoa.engineering.crawl.ppomppu.processor.service.MessageSenderService; import com.myoa.engineering.crawl.ppomppu.processor.service.PpomppuArticleService; @@ -45,13 +43,14 @@ public class CrawlAPIController { public Mono> crawlBoard(@PathVariable("boardName") PpomppuBoardName boardName) { log.info("got request... {}", boardName); FeedParsedResult result = FeedParsedResult.of(boardName); - Mono> articles = - ppomppuRSSFeedService.getArticles(boardName) - .doOnNext(e -> ppomppuArticleService.filterOnlyNewArticles(boardName, e)) - .doOnNext(e -> messageSenderService.sendMessageToSlack(e)) - .doOnNext(e -> ppomppuArticleService.save(boardName, e)); - return articles.then(Mono.just(APIResponse.success(result.done()))); + Mono publishedMessages = + ppomppuRSSFeedService.getArticles(boardName) + .map(e -> ppomppuArticleService.filterOnlyNewArticles(boardName, e)) + .map(e -> ppomppuArticleService.save(boardName, e)) + .flatMap(messageSenderService::sendMessageToSlack); + + return publishedMessages.then(Mono.just(APIResponse.success(result.done()))); } } diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/PpomppuArticleParser.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/PpomppuArticleParser.java index 20dc7d0..4387642 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/PpomppuArticleParser.java +++ b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/PpomppuArticleParser.java @@ -46,7 +46,7 @@ public final class PpomppuArticleParser { } public static String parseTitle(Element td) { - return td.text(); + return td.getElementsByTag("a").text(); // TODO cdn image extracting } public static String parseArticleUrl(Element td) { diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/PpomppuArticleTransformer.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/PpomppuArticleTransformer.java index 95f4e8a..ede7623 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/PpomppuArticleTransformer.java +++ b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/PpomppuArticleTransformer.java @@ -17,8 +17,11 @@ import com.myoa.engineering.crawl.ppomppu.support.dto.SimpleMessageDTO; * */ public final class PpomppuArticleTransformer { + private PpomppuArticleTransformer() {} + private static final String MESSAGE_FORMAT_V1 = "%s)) `%s` <%s:LINK>"; + private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss") .withZone(ZoneId.of("Asia/Seoul")); @@ -26,7 +29,7 @@ public final class PpomppuArticleTransformer { SimpleMessageDTO.builder() .requestedAt(Instant.now()) .publishedAt(entity.getRegisteredAt()) - .title(String.format("[%s] %s", entity.getBoardName().getMenuName(), entity.getTitle())) + .title(String.format(MESSAGE_FORMAT_V1, entity.getBoardName().getMenuName(), entity.getTitle())) .body(entity.getArticleUrl()) .build(); @@ -44,7 +47,7 @@ public final class PpomppuArticleTransformer { } public static String convertToInlineMessage(PpomppuArticle article) { - return String.format("[%s] %s\n%s", + return String.format(MESSAGE_FORMAT_V1, article.getBoardName().getMenuName(), article.getTitle(), article.getArticleUrl()); } } diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/client/MessageSenderAPIClient.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/client/MessageSenderAPIClient.java index 5110c3e..155214f 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/client/MessageSenderAPIClient.java +++ b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/client/MessageSenderAPIClient.java @@ -1,18 +1,20 @@ package com.myoa.engineering.crawl.ppomppu.processor.infrastructure.client; +import org.springframework.core.ParameterizedTypeReference; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; +import org.springframework.stereotype.Component; +import org.springframework.web.reactive.function.client.WebClient; +import org.springframework.web.reactive.function.client.WebClientRequestException; + import com.myoa.engineering.crawl.ppomppu.processor.dto.constant.WebClientPropertiesUnitName; import com.myoa.engineering.crawl.ppomppu.support.dto.SimpleMessageDTO; import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebClientFilterFactory; import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebFluxExchangeStragiesFactory; import com.myoa.engineering.crawl.ppomppu.support.webclient.properties.WebClientProperties; import com.myoa.engineering.crawl.ppomppu.support.webclient.properties.WebClientProperties.WebClientPropertiesUnit; + import lombok.extern.slf4j.Slf4j; -import org.springframework.core.ParameterizedTypeReference; -import org.springframework.http.HttpHeaders; -import org.springframework.http.MediaType; -import org.springframework.stereotype.Component; -import org.springframework.web.reactive.function.client.WebClient; -import org.springframework.web.reactive.function.client.WebClientRequestException; import reactor.core.publisher.Mono; import reactor.core.scheduler.Schedulers; @@ -35,8 +37,8 @@ public class MessageSenderAPIClient { .baseUrl(webClientPropertiesUnit.getBaseUrl()) .exchangeStrategies(WebFluxExchangeStragiesFactory.ofDefault()) .defaultHeader(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_VALUE) - .filter(WebClientFilterFactory.logRequest()) - .filter(WebClientFilterFactory.logResponse()) +// .filter(WebClientFilterFactory.logRequest()) +// .filter(WebClientFilterFactory.logResponse()) .build(); } @@ -49,7 +51,6 @@ public class MessageSenderAPIClient { .onErrorResume(WebClientRequestException.class, t -> { log.info("Exception occured, ignoring. : {}", t.getClass().getSimpleName()); return Mono.empty(); - }) - .doOnNext(e -> log.info("response: {} ", e)); + }); } } \ No newline at end of file diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/PpomppuArticleService.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/PpomppuArticleService.java index e810c56..ad210bb 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/PpomppuArticleService.java +++ b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/PpomppuArticleService.java @@ -1,16 +1,19 @@ package com.myoa.engineering.crawl.ppomppu.processor.service; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle; import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuBoardFeedStatus; import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository.PpomppuArticleRepository; import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository.PpomppuBoardFeedStatusRepository; import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName; -import java.util.List; -import java.util.Optional; -import java.util.stream.Collectors; + import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; @Slf4j @Service @@ -32,6 +35,7 @@ public class PpomppuArticleService { Long latestArticleId = boardFeedStatus.map(PpomppuBoardFeedStatus::getLatestParsedArticleId) .orElse(0L); + log.info("latestArticleId : {}", latestArticleId); return articles.stream() .filter(e -> e.getArticleId().compareTo(latestArticleId) > 0) .collect(Collectors.toList()); @@ -46,9 +50,12 @@ public class PpomppuArticleService { // save PpomppuBoardFeedStatus Optional boardFeedStatus = ppomppuBoardFeedStatusRepository.findByBoardName(boardName); + log.info("boardName: {}, isPresent?: {}", boardName, boardFeedStatus.isPresent()); boardFeedStatus.ifPresentOrElse(e -> { - e.updateArticleId(latestArticleId); - ppomppuBoardFeedStatusRepository.save(e); + if (latestArticleId.longValue() > 0L) { + e.updateArticleId(latestArticleId); + ppomppuBoardFeedStatusRepository.save(e); + } }, () -> ppomppuBoardFeedStatusRepository.save(PpomppuBoardFeedStatus.of(boardName, latestArticleId))); diff --git a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/scheduler/ParseEventEmitter.java b/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/scheduler/ParseEventEmitter.java index 4a0cc9b..bf94c88 100644 --- a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/scheduler/ParseEventEmitter.java +++ b/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/scheduler/ParseEventEmitter.java @@ -26,15 +26,11 @@ public class ParseEventEmitter { this.processorAPIService = processorAPIService; } - @Scheduled(fixedRate = 60 * 1000L) - public void emitDomesticBoard() { - log.info("[emitDomesticBoard] trigger fired!"); - processorAPIService.emitParseEvent(PpomppuBoardName.PPOMPPU_DOMESTIC_ETC).block(); - } - @Scheduled(fixedRate = 300 * 1000L) - public void emitOverseaBoard() { - log.info("[emitOverseaBoard] trigger fired!"); - processorAPIService.emitParseEvent(PpomppuBoardName.PPOMPPU_OVERSEA_ETC).block(); + public void emitBoards() { + log.info("[emitDomesticBoard] trigger fired!"); + for (PpomppuBoardName boardName : PpomppuBoardName.values()) { + processorAPIService.emitParseEvent(boardName).block(); + } } } diff --git a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/controller/MessageSenderAPIController.java b/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/controller/MessageSenderAPIController.java index 373e5bf..479d101 100644 --- a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/controller/MessageSenderAPIController.java +++ b/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/controller/MessageSenderAPIController.java @@ -5,6 +5,7 @@ import com.myoa.engineering.crawl.ppomppu.support.dto.APIResponse; import com.myoa.engineering.crawl.ppomppu.support.dto.SimpleMessageDTO; import lombok.extern.slf4j.Slf4j; import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; import reactor.core.publisher.Mono; @@ -27,9 +28,7 @@ public class MessageSenderAPIController { } @PostMapping("/messages/sendMessage/messengers/slack") - public Mono> sendMessageToSlack(SimpleMessageDTO dto) { - - log.info("received : {}, \nbody: {}", dto.getTitle(), dto.getBody()); + public Mono> sendMessageToSlack(@RequestBody SimpleMessageDTO dto) { return sender.sendMessage(sender.ofMessage(dto.getBody())) .then(Mono.just(APIResponse.success(dto))); } diff --git a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/controller/TestAPIController.java b/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/controller/TestAPIController.java index 96fcc55..28192e1 100644 --- a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/controller/TestAPIController.java +++ b/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/controller/TestAPIController.java @@ -4,6 +4,8 @@ import com.myoa.engineering.crawl.ppomppu.sender.infrastructure.client.MongeShop import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; + +import lombok.extern.slf4j.Slf4j; import reactor.core.publisher.Mono; /** @@ -12,6 +14,7 @@ import reactor.core.publisher.Mono; * @author Shin Woo-jin (woo-jin.shin@linecorp.com) * @since 2021-11-15 */ +@Slf4j @RestController @RequestMapping("/api/v1") public class TestAPIController { @@ -24,6 +27,7 @@ public class TestAPIController { @GetMapping("/test") public Mono test() { + log.info("received!!!"); return sender.sendMessage(sender.ofMessage("testtesttest!!!")); } } diff --git a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/infrastructure/client/MongeShoppingBotSlackMessageSender.java b/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/infrastructure/client/MongeShoppingBotSlackMessageSender.java index 8ab5b00..6f5b352 100644 --- a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/infrastructure/client/MongeShoppingBotSlackMessageSender.java +++ b/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/infrastructure/client/MongeShoppingBotSlackMessageSender.java @@ -13,7 +13,6 @@ public class MongeShoppingBotSlackMessageSender extends SlackMessageSender { private static final String SLACK_SECRET_UNIT_NAME = "monge_shopping_bot"; private final SlackSecretPropertiesUnit slackProperties; -// private final SlackMessageSender slackMessageSender; public MongeShoppingBotSlackMessageSender(SlackSecretProperties slackSecretProperties) { super(slackSecretProperties.find(SLACK_SECRET_UNIT_NAME).getToken()); diff --git a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/infrastructure/client/SlackMessageSender.java b/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/infrastructure/client/SlackMessageSender.java index 5f349bd..c0f2efb 100644 --- a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/infrastructure/client/SlackMessageSender.java +++ b/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/infrastructure/client/SlackMessageSender.java @@ -33,7 +33,6 @@ public class SlackMessageSender implements MessageSender { .exchangeStrategies(WebFluxExchangeStragiesFactory.ofDefault()) .defaultHeader(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_UTF8_VALUE) .defaultHeader(HttpHeaders.ACCEPT_CHARSET, "UTF-8") - .filter(WebClientFilterFactory.logRequest()) .filter(WebClientFilterFactory.logResponse()) .build(); } @@ -48,8 +47,7 @@ public class SlackMessageSender implements MessageSender { .onErrorResume(WebClientRequestException.class, t -> { log.info("Exception occured, ignoring. : {}", t.getClass().getSimpleName()); return Mono.empty(); - }) - .doOnNext(e -> log.info("response: {}", e)); + }); } } diff --git a/sender/src/main/resources/application.yml b/sender/src/main/resources/application.yml index 670365b..4903239 100644 --- a/sender/src/main/resources/application.yml +++ b/sender/src/main/resources/application.yml @@ -12,11 +12,6 @@ spring: freemarker: enabled: false - jpa: - hibernate: - ddl-auto: create-drop - generate-ddl: true - server: port: 20080 error: diff --git a/support/next_message_tempalte(block_kit_builder).json b/support/next_message_tempalte(block_kit_builder).json new file mode 100644 index 0000000..eae1772 --- /dev/null +++ b/support/next_message_tempalte(block_kit_builder).json @@ -0,0 +1,31 @@ +{ + "blocks": [ + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "기타)) `[쿠팡] 타푸코 이중 진공 스텐 세라믹 코팅 텀블러 (화이트만 11,130원/로켓와우무료) 4 [기타]`" + }, + "accessory": { + "type": "image", + "image_url": "cdn.ppomppu.co.kr/zboard/data3/2021/1121/m_20211121184835_zfyahnow.png", + "alt_text": "alt text for image" + } + }, + { + "type": "divider" + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "기타)) `[롯데온] 1매 149원 올국산 KF94새부리마스크 대형 200매 (29,920원/무료배송) 58 [기타]`" + }, + "accessory": { + "type": "image", + "image_url": "https://s3-media3.fl.yelpcdn.com/bphoto/c7ed05m9lC2EmA3Aruue7A/o.jpg", + "alt_text": "alt text for image" + } + } + ] +} \ No newline at end of file diff --git a/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/dto/code/PpomppuBoardName.java b/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/dto/code/PpomppuBoardName.java index aa15fc2..d66bdc0 100644 --- a/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/dto/code/PpomppuBoardName.java +++ b/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/dto/code/PpomppuBoardName.java @@ -12,7 +12,7 @@ import lombok.Getter; @Getter @AllArgsConstructor public enum PpomppuBoardName { - PPOMPPU_DOMESTIC_ALL("/zboard/zboard.php?id=ppomppu", "전체", true), + PPOMPPU_DOMESTIC_ALL("/zboard/zboard.php?id=ppomppu", "전체", false), PPOMPPU_DOMESTIC_ETC("/zboard/zboard.php?id=ppomppu&category=1", "기타", true), PPOMPPU_DOMESTIC_COMPUTER("/zboard/zboard.php?id=ppomppu&category=4", "컴퓨터", true), PPOMPPU_DOMESTIC_DIGITAL("/zboard/zboard.php?id=ppomppu&category=5", "디지털", true), @@ -24,7 +24,7 @@ public enum PpomppuBoardName { PPOMPPU_DOMESTIC_CLOTHES("/zboard/zboard.php?id=ppomppu&category=12", "의류/잡화", true), PPOMPPU_DOMESTIC_COSMETIC("/zboard/zboard.php?id=ppomppu&category=13", "화장품", true), PPOMPPU_DOMESTIC_OUTDOOR("/zboard/zboard.php?id=ppomppu&category=15", "등산/캠핑", true), - PPOMPPU_OVERSEA_ALL("/zboard/zboard.php?id=ppomppu4", "전체", true), + PPOMPPU_OVERSEA_ALL("/zboard/zboard.php?id=ppomppu4", "전체", false), PPOMPPU_OVERSEA_ETC("/zboard/zboard.php?id=ppomppu4&category=1", "기타", true), PPOMPPU_OVERSEA_APPLIANCES("/zboard/zboard.php?id=ppomppu4&category=7", "가전", true), PPOMPPU_OVERSEA_TVAV("/zboard/zboard.php?id=ppomppu4&category=8", "TV/영상", true), @@ -45,7 +45,7 @@ public enum PpomppuBoardName { public static final String PPOMPPU_URL = "https://www.ppomppu.co.kr"; public static String ofViewPageUrl(String articleUrl) { - return PPOMPPU_URL + "/zboard" + articleUrl; + return PPOMPPU_URL + "/zboard/" + articleUrl; } }