[PPN-211113] Implement spring config client

build.gradle

@@ -32,6 +32,17 @@ allprojects {
         }
     }
 
+    ext {
+        set('springCloudVersion', "2020.0.4")
+    }
+
+    dependencyManagement {
+        imports {
+            mavenBom "org.springframework.cloud:spring-cloud-dependencies:${springCloudVersion}"
+        }
+    }
+
+
 /*    task initSourceFolders {
         sourceSets*.java.srcDirs*.each {
             if( !it.exists() ) {

processor/build.gradle

@@ -1,3 +1,4 @@
+
 dependencies {
     developmentOnly 'org.springframework.boot:spring-boot-devtools'
     runtimeOnly 'com.h2database:h2'
@@ -9,6 +10,8 @@ dependencies {
     implementation 'org.springframework.boot:spring-boot-starter-webflux'
     implementation 'org.springframework.boot:spring-boot-starter-data-jpa'
     implementation 'org.springframework.boot:spring-boot-starter-jdbc'
+    implementation 'org.springframework.cloud:spring-cloud-starter-config'
+    implementation 'org.springframework.boot:spring-boot-starter-actuator'
     implementation 'com.rometools:rome:1.16.0'
     implementation 'org.jsoup:jsoup:1.14.2'
     implementation 'com.h2database:h2:1.4.200'

processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/client/PpomppuBoardFeedRetriever.java

@@ -1,9 +1,11 @@
 package com.myoa.engineering.crawl.ppomppu.processor.infrastructure.client;
 
-import com.myoa.engineering.crawl.ppomppu.processor.configuration.factory.WebClientFilterFactory;
+import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebClientFilterFactory;
-import com.myoa.engineering.crawl.ppomppu.processor.configuration.factory.WebFluxExchangeStragiesFactory;
+import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebFluxExchangeStragiesFactory;
 import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
 import lombok.extern.slf4j.Slf4j;
+
+import org.springframework.core.ParameterizedTypeReference;
 import org.springframework.stereotype.Component;
 import org.springframework.web.reactive.function.client.WebClient;
 import org.springframework.web.reactive.function.client.WebClientRequestException;

processor/src/main/resources/application-development.yml

@@ -3,6 +3,12 @@ spring:
     activate:
       on-profile: development
     import:
+      - "configserver:http://192.168.0.100:11080"
       - classpath:/development/webclient.yml
       - classpath:/development/temppassword.yml
       - classpath:/development/database.yml
+  profiles:
+    group:
+      development: development, datasource-development
+
+    # import: optional:configserver:http://localhost:11080 # can be start up even config server was not found.

processor/src/main/resources/application.yml

@@ -1,4 +1,6 @@
 spring:
+  application:
+    name: ppn-processor
   main:
     allow-bean-definition-overriding: true
   profiles:
@@ -12,3 +14,9 @@ server:
   error:
     whitelabel:
       enabled: false
+
+management:
+  endpoints:
+    web:
+      exposure:
+        include: refresh

receiver/src/main/resources/application.yml

@@ -1,4 +1,6 @@
 spring:
+  application:
+    name: ppn-receiver
   main:
     allow-bean-definition-overriding: true
   profiles:

sender/build.gradle

@@ -0,0 +1,23 @@
+dependencies {
+    developmentOnly 'org.springframework.boot:spring-boot-devtools'
+    runtimeOnly 'mysql:mysql-connector-java'
+    compileOnly 'org.projectlombok:lombok'
+
+    implementation project(':support')
+    // https://projectreactor.io/docs/core/release/reference/#debug-activate
+    implementation 'org.springframework.boot:spring-boot-starter-webflux'
+    implementation 'org.springframework.boot:spring-boot-starter-actuator'
+    implementation 'org.springframework.cloud:spring-cloud-starter-config'
+
+    annotationProcessor 'org.springframework.boot:spring-boot-configuration-processor'
+    annotationProcessor 'org.projectlombok:lombok'
+
+    testImplementation 'org.springframework.boot:spring-boot-starter-test'
+}
+
+test {
+    useJUnitPlatform()
+    testLogging {
+        events "passed", "skipped", "failed"
+    }
+}

sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/SenderApplication.java

@@ -1,10 +1,18 @@
 package com.myoa.engineering.crawl.ppomppu.sender;
 
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+
 /**
  * SenderApplication
  * @author Shin Woo-jin (woo-jin.shin@linecorp.com)
  * @since 2021-08-20
  *
  */
+@SpringBootApplication
 public class SenderApplication {
+
+    public static void main(String[] args) {
+        SpringApplication.run(SenderApplication.class, args);
+    }
 }

sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/controller/TestAPIController.java

@@ -0,0 +1,37 @@
+package com.myoa.engineering.crawl.ppomppu.sender.controller;
+
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+
+import com.myoa.engineering.crawl.ppomppu.sender.dto.SlackMessageDTO;
+import com.myoa.engineering.crawl.ppomppu.sender.infrastructure.client.SlackMessageSender;
+
+import reactor.core.publisher.Mono;
+
+/**
+ * TestAPIController
+ * @author Shin Woo-jin (woo-jin.shin@linecorp.com)
+ * @since 2021-11-15
+ *
+ */
+@RestController
+@RequestMapping("/api/v1")
+public class TestAPIController {
+
+    private final SlackMessageSender sender;
+
+    public TestAPIController() {
+        this.sender = new SlackMessageSender("xoxb-2688454277126-2695026012277-K2Ib13lKokmTiBSnSMrc0Bp2");
+    }
+
+    @GetMapping("/test")
+    public Mono<String> test() {
+        return sender.sendMessage(SlackMessageDTO.builder()
+                                                 .text("test!")
+                                                 .iconEmoji("monge")
+                                                 .channel("notify_shopping")
+                                                 .username("shopping notifier")
+                                                 .build());
+    }
+}

sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/dto/MessageDTO.java

@@ -0,0 +1,12 @@
+package com.myoa.engineering.crawl.ppomppu.sender.dto;
+
+import java.io.Serializable;
+
+/**
+ * MessageDTO
+ * @author Shin Woo-jin (woo-jin.shin@linecorp.com)
+ * @since 2021-11-14
+ *
+ */
+public interface MessageDTO extends Serializable {
+}

sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/dto/SlackMessageDTO.java

@@ -0,0 +1,36 @@
+package com.myoa.engineering.crawl.ppomppu.sender.dto;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import lombok.Builder;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+
+/**
+ * SlackMessageDTO
+ * @author Shin Woo-jin (woo-jin.shin@linecorp.com)
+ * @since 2021-11-14
+ *
+ */
+@Getter
+@NoArgsConstructor
+public class SlackMessageDTO implements MessageDTO {
+
+    private final static long serialVersionUID = 4737608709660494713L;
+
+    private String text;
+    private String channel;
+    private String username;
+
+    @JsonProperty("icon_emoji")
+    private String iconEmoji;
+
+    @Builder
+    public SlackMessageDTO(String text, String channel, String username, String iconEmoji) {
+        this.text = text;
+        this.channel = channel;
+        this.username = username;
+        this.iconEmoji = iconEmoji;
+    }
+
+}

sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/infrastructure/client/MessageSender.java

@@ -0,0 +1,17 @@
+package com.myoa.engineering.crawl.ppomppu.sender.infrastructure.client;
+
+import com.myoa.engineering.crawl.ppomppu.sender.dto.MessageDTO;
+
+import reactor.core.publisher.Mono;
+
+/**
+ * MessageSender
+ * @author Shin Woo-jin (woo-jin.shin@linecorp.com)
+ * @since 2021-11-14
+ *
+ */
+public interface MessageSender<T extends MessageDTO> {
+
+    Mono<String> sendMessage(T message);
+
+}

sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/infrastructure/client/SlackMessageSender.java

@@ -0,0 +1,88 @@
+package com.myoa.engineering.crawl.ppomppu.sender.infrastructure.client;
+
+import org.springframework.http.HttpHeaders;
+import org.springframework.http.MediaType;
+import org.springframework.web.reactive.function.client.WebClient;
+import org.springframework.web.reactive.function.client.WebClientRequestException;
+
+import com.myoa.engineering.crawl.ppomppu.sender.dto.SlackMessageDTO;
+import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebClientFilterFactory;
+import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebFluxExchangeStragiesFactory;
+
+import lombok.extern.slf4j.Slf4j;
+import reactor.core.publisher.Mono;
+import reactor.core.scheduler.Schedulers;
+
+/**
+ * SlackMessageSender
+ *
+ * @author Shin Woo-jin (woozu.shin@kakaoent.com)
+ * @since 2021-09-08
+ */
+@Slf4j
+public class SlackMessageSender implements MessageSender<SlackMessageDTO> {
+
+    private static final String SLACK_API_URL = "https://slack.com/api";
+
+    private final WebClient webClient;
+    private final String apiSecret;
+
+    public SlackMessageSender(String apiSecret) {
+        this.webClient = WebClient.builder()
+                                  .baseUrl(SLACK_API_URL)
+                                  .defaultHeader(HttpHeaders.AUTHORIZATION, "Bearer " + apiSecret)
+                                  .exchangeStrategies(WebFluxExchangeStragiesFactory.ofDefault())
+                                  .defaultHeader(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_UTF8_VALUE)
+                                  .defaultHeader(HttpHeaders.ACCEPT_CHARSET, "UTF-8")
+                                  .filter(WebClientFilterFactory.logRequest())
+                                  .filter(WebClientFilterFactory.logResponse())
+                                  .build();
+        this.apiSecret = apiSecret;
+    }
+
+    @Override
+    public Mono<String> sendMessage(SlackMessageDTO message) {
+        return webClient.post()
+                        .uri("/chat.postMessage")
+                        .bodyValue(message)
+                        .exchangeToMono(e -> e.bodyToMono(String.class))
+                        .publishOn(Schedulers.boundedElastic())
+                        .onErrorResume(WebClientRequestException.class, t -> {
+                            log.info("Exception occured, ignoring. : {}", t.getClass().getSimpleName());
+                            return Mono.empty();
+                        })
+                        .doOnNext(e -> log.info("response: {}", e));
+    }
+
+}
+
+/*
+import requests
+import json
+
+# woozuyeni_watchtower
+SLACK_TOKEN="xoxb-2688454277126-2695026012277-K2Ib13lKokmTiBSnSMrc0Bp2"
+CHANNEL = "notify_watchtower"
+CHANNEL = "common"
+
+
+POST_API = "https://slack.com/api/chat.postMessage"
+HEADER = "Authorization: Bearer"
+
+headers = {
+    "Content-Type" : "application/json",
+    "Authorization" : f"Bearer {SLACK_TOKEN}"
+}
+
+data = {}
+data["text"] = "몽몽! :monge_big:"
+data["channel"] = CHANNEL
+data["username"] = "몽이봇"
+data["icon_emoji"] = ":monge_big:"
+
+# requests.post(POST_API, data=json.dumps(data), headers=headers)
+
+
+attachments = [{"title": "Cat", "image_url": image_url}]
+image_url = "https://myoa-universe.com/assets/love1.png"
+ */

sender/src/main/resources/application-development.yml

@@ -0,0 +1,11 @@
+spring:
+  config:
+    activate:
+      on-profile: development
+    import:
+      - classpath:/development/webclient.yml
+      - "configserver:http://192.168.0.100:11080"
+    # import: optional:configserver:http://localhost:11080 # can be start up even config server was not found.
+  profiles:
+    group:
+      development: development, slackapi-development

sender/src/main/resources/application-production.yml

@@ -0,0 +1,6 @@
+spring:
+  config:
+    activate:
+      on-profile: production
+#    import:
+#      - classpath:/production/webclient.yml

sender/src/main/resources/application.yml

@@ -0,0 +1,19 @@
+spring:
+  application:
+#    name: ppn-sender
+    name: app1
+  main:
+    allow-bean-definition-overriding: true
+  profiles:
+    active: development
+  freemarker:
+    enabled: false
+
+server:
+  port: 20080
+
+management:
+  endpoints:
+    web:
+      exposure:
+        include: refresh

sender/src/main/resources/development/webclient.yml

@@ -0,0 +1,5 @@
+webclient:
+  base-url: http://localhost:20081
+  units:
+    - unit-name: processor-api
+      base-url: http://localhost:20081

support/build.gradle

@@ -3,6 +3,7 @@ dependencies {
     compileOnly 'org.projectlombok:lombok'
 
     // https://projectreactor.io/docs/core/release/reference/#debug-activate
+    implementation 'org.springframework.boot:spring-boot-starter-webflux'
 
     annotationProcessor 'org.springframework.boot:spring-boot-configuration-processor'
     annotationProcessor 'org.projectlombok:lombok'

support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/dto/code/PpomppuBoardName.java

@@ -12,40 +12,40 @@ import lombok.Getter;
 @Getter
 @AllArgsConstructor
 public enum PpomppuBoardName {
-    PPOMPPU_DOMESTIC_ALL("zboard/zboard.php?id=ppomppu", "전체", true),
+    PPOMPPU_DOMESTIC_ALL("/zboard/zboard.php?id=ppomppu", "전체", true),
-    PPOMPPU_DOMESTIC_ETC("zboard/zboard.php?id=ppomppu&category=1", "기타", true),
+    PPOMPPU_DOMESTIC_ETC("/zboard/zboard.php?id=ppomppu&category=1", "기타", true),
-    PPOMPPU_DOMESTIC_COMPUTER("zboard/zboard.php?id=ppomppu&category=4", "컴퓨터", true),
+    PPOMPPU_DOMESTIC_COMPUTER("/zboard/zboard.php?id=ppomppu&category=4", "컴퓨터", true),
-    PPOMPPU_DOMESTIC_DIGITAL("zboard/zboard.php?id=ppomppu&category=5", "디지털", true),
+    PPOMPPU_DOMESTIC_DIGITAL("/zboard/zboard.php?id=ppomppu&category=5", "디지털", true),
-    PPOMPPU_DOMESTIC_FOOD("zboard/zboard.php?id=ppomppu&category=6", "식품/건강", true),
+    PPOMPPU_DOMESTIC_FOOD("/zboard/zboard.php?id=ppomppu&category=6", "식품/건강", true),
-    PPOMPPU_DOMESTIC_BOOK("zboard/zboard.php?id=ppomppu&category=8", "서적", true),
+    PPOMPPU_DOMESTIC_BOOK("/zboard/zboard.php?id=ppomppu&category=8", "서적", true),
-    PPOMPPU_DOMESTIC_APPLIANCES("zboard/zboard.php?id=ppomppu&category=9", "가전/가구", true),
+    PPOMPPU_DOMESTIC_APPLIANCES("/zboard/zboard.php?id=ppomppu&category=9", "가전/가구", true),
-    PPOMPPU_DOMESTIC_PARENTING("zboard/zboard.php?id=ppomppu&category=10", "육아", true),
+    PPOMPPU_DOMESTIC_PARENTING("/zboard/zboard.php?id=ppomppu&category=10", "육아", true),
-    PPOMPPU_DOMESTIC_GIFTCARD("zboard/zboard.php?id=ppomppu&category=11", "상품권", true),
+    PPOMPPU_DOMESTIC_GIFTCARD("/zboard/zboard.php?id=ppomppu&category=11", "상품권", true),
-    PPOMPPU_DOMESTIC_CLOTHES("zboard/zboard.php?id=ppomppu&category=12", "의류/잡화", true),
+    PPOMPPU_DOMESTIC_CLOTHES("/zboard/zboard.php?id=ppomppu&category=12", "의류/잡화", true),
-    PPOMPPU_DOMESTIC_COSMETIC("zboard/zboard.php?id=ppomppu&category=13", "화장품", true),
+    PPOMPPU_DOMESTIC_COSMETIC("/zboard/zboard.php?id=ppomppu&category=13", "화장품", true),
-    PPOMPPU_DOMESTIC_OUTDOOR("zboard/zboard.php?id=ppomppu&category=15", "등산/캠핑", true),
+    PPOMPPU_DOMESTIC_OUTDOOR("/zboard/zboard.php?id=ppomppu&category=15", "등산/캠핑", true),
-    PPOMPPU_OVERSEA_ALL("zboard/zboard.php?id=ppomppu4", "전체", true),
+    PPOMPPU_OVERSEA_ALL("/zboard/zboard.php?id=ppomppu4", "전체", true),
-    PPOMPPU_OVERSEA_ETC("zboard/zboard.php?id=ppomppu4&category=1", "기타", true),
+    PPOMPPU_OVERSEA_ETC("/zboard/zboard.php?id=ppomppu4&category=1", "기타", true),
-    PPOMPPU_OVERSEA_APPLIANCES("zboard/zboard.php?id=ppomppu4&category=7", "가전", true),
+    PPOMPPU_OVERSEA_APPLIANCES("/zboard/zboard.php?id=ppomppu4&category=7", "가전", true),
-    PPOMPPU_OVERSEA_TVAV("zboard/zboard.php?id=ppomppu4&category=8", "TV/영상", true),
+    PPOMPPU_OVERSEA_TVAV("/zboard/zboard.php?id=ppomppu4&category=8", "TV/영상", true),
-    PPOMPPU_OVERSEA_COMPUTER("zboard/zboard.php?id=ppomppu4&category=3", "컴퓨터", true),
+    PPOMPPU_OVERSEA_COMPUTER("/zboard/zboard.php?id=ppomppu4&category=3", "컴퓨터", true),
-    PPOMPPU_OVERSEA_DIGITAL("zboard/zboard.php?id=ppomppu4&category=4", "디지털", true),
+    PPOMPPU_OVERSEA_DIGITAL("/zboard/zboard.php?id=ppomppu4&category=4", "디지털", true),
-    PPOMPPU_OVERSEA_MOBILEACCESSORY("zboard/zboard.php?id=ppomppu4&category=9", "액세서리", true),
+    PPOMPPU_OVERSEA_MOBILEACCESSORY("/zboard/zboard.php?id=ppomppu4&category=9", "액세서리", true),
-    PPOMPPU_OVERSEA_CLOTHES("zboard/zboard.php?id=ppomppu4&category=5", "의류/잡화", true),
+    PPOMPPU_OVERSEA_CLOTHES("/zboard/zboard.php?id=ppomppu4&category=5", "의류/잡화", true),
-    PPOMPPU_OVERSEA_WATCH("zboard/zboard.php?id=ppomppu4&category=2", "시계", true),
+    PPOMPPU_OVERSEA_WATCH("/zboard/zboard.php?id=ppomppu4&category=2", "시계", true),
-    PPOMPPU_OVERSEA_SHOES("zboard/zboard.php?id=ppomppu4&category=11", "신발", true),
+    PPOMPPU_OVERSEA_SHOES("/zboard/zboard.php?id=ppomppu4&category=11", "신발", true),
-    PPOMPPU_OVERSEA_FOOD("zboard/zboard.php?id=ppomppu4&category=10", "식품/건강", true),
+    PPOMPPU_OVERSEA_FOOD("/zboard/zboard.php?id=ppomppu4&category=10", "식품/건강", true),
-    PPOMPPU_OVERSEA_PARENTING("zboard/zboard.php?id=ppomppu4&category=6", "육아", true),
+    PPOMPPU_OVERSEA_PARENTING("/zboard/zboard.php?id=ppomppu4&category=6", "육아", true),
     ;
 
     private String resourcePath;
     private String menuName;
     private boolean crawlWithDefaultTimer;
 
-    public static final String PPOMPPU_URL = "https://www.ppomppu.co.kr/";
+    public static final String PPOMPPU_URL = "https://www.ppomppu.co.kr";
 
     public static String ofViewPageUrl(String articleUrl) {
-        return PPOMPPU_URL + "zboard/" + articleUrl;
+        return PPOMPPU_URL + "/zboard" + articleUrl;
     }
 
 }

support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/util/ObjectMapperFactory.java

@@ -1,4 +1,4 @@
-package com.myoa.engineering.crawl.ppomppu.processor.util;
+package com.myoa.engineering.crawl.ppomppu.support.util;
 
 import com.fasterxml.jackson.core.JsonGenerator;
 import com.fasterxml.jackson.core.JsonParser.Feature;

support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/webclient/factory/WebClientFilterFactory.java

@@ -1,4 +1,4 @@
-package com.myoa.engineering.crawl.ppomppu.processor.configuration.factory;
+package com.myoa.engineering.crawl.ppomppu.support.webclient.factory;
 
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.web.reactive.function.client.ClientRequest;
@@ -27,8 +27,8 @@ public final class WebClientFilterFactory {
 
     private static Mono<ClientRequest> writeRequest(ClientRequest clientRequest) {
         try {
-            log.info("[WEBCLIENT REQUEST] uri : {} method : {} headers : {}",
+            log.info("[WEBCLIENT REQUEST] uri : {} method : {} headers : {}, body: {}",
-                     clientRequest.url(), clientRequest.method(), clientRequest.headers());
+                     clientRequest.url(), clientRequest.method(), clientRequest.headers(), clientRequest.body());
         } catch (Exception e) {
             log.error("[WEBCLIENT REQUEST] write request failed", e);
         }
@@ -37,7 +37,7 @@ public final class WebClientFilterFactory {
 
     private static Mono<ClientResponse> writeResponse(ClientResponse clientResponse) {
         try {
-            log.info("[WEBCLIENT REQUEST] statusCode : {} headers : {}",
+            log.info("[WEBCLIENT RESPONSE] statusCode : {} headers : {}",
                      clientResponse.rawStatusCode(), clientResponse.headers().asHttpHeaders());
         } catch (Exception e) {
             log.error("[WEBCLIENT RESPONSE] write response failed", e);

support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/webclient/factory/WebFluxExchangeStragiesFactory.java

@@ -1,7 +1,8 @@
-package com.myoa.engineering.crawl.ppomppu.processor.configuration.factory;
+package com.myoa.engineering.crawl.ppomppu.support.webclient.factory;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
-import com.myoa.engineering.crawl.ppomppu.processor.util.ObjectMapperFactory;
+import com.myoa.engineering.crawl.ppomppu.support.util.ObjectMapperFactory;
+
 import org.springframework.http.codec.json.Jackson2JsonDecoder;
 import org.springframework.http.codec.json.Jackson2JsonEncoder;
 import org.springframework.util.MimeTypeUtils;