[NO-ISSUE] Prettify

shopping-crawler/build.gradle

@@ -9,6 +9,8 @@ dependencies {
     implementation("org.springframework.boot:spring-boot-starter-web") {
         exclude group: "org.springframework.boot", module: "spring-boot-starter-tomcat"
     }
+    implementation("org.springframework.boot:spring-boot-starter-webflux")
+
 
     implementation("org.springframework.boot:spring-boot-starter-undertow") {
         exclude group: "io.undertow", module: "undertow-websockets-jsr"
@@ -24,6 +26,7 @@ dependencies {
     implementation 'org.ahocorasick:ahocorasick:0.6.3'
     implementation "com.slack.api:slack-api-client:1.39.1"
 
+
 //    implementation "io.github.resilience4j:resilience4j-spring-boot3:2.2.0"
     implementation 'io.github.resilience4j:resilience4j-all:2.2.0'
     implementation "io.github.resilience4j:resilience4j-feign:2.2.0"

shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/feign/FmkoreaClientErrorDecoder.java

@@ -0,0 +1,40 @@
+package com.myoa.engineering.crawl.shopping.configuration.feign;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.myoa.engineering.crawl.shopping.dto.ExceptionMessage;
+import feign.Response;
+import feign.codec.ErrorDecoder;
+import io.undertow.util.BadRequestException;
+import javassist.NotFoundException;
+import lombok.extern.slf4j.Slf4j;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+@Slf4j
+public class FmkoreaClientErrorDecoder implements ErrorDecoder {
+
+    private final ErrorDecoder errorDecoder = new Default();
+
+    @Override
+    public Exception decode(String methodsKey, Response response) {
+        ExceptionMessage message = null;
+        try (InputStream bodyIs = response.body()
+                                          .asInputStream()) {
+            ObjectMapper mapper = new ObjectMapper();
+            message = mapper.readValue(bodyIs, ExceptionMessage.class);
+        } catch (IOException e) {
+            return new Exception(e.getMessage());
+        }
+        switch (response.status()) {
+            case 400:
+                return new BadRequestException(message.getMessage() != null ? message.getMessage() : "Bad Request");
+            case 404:
+                return new NotFoundException(message.getMessage() != null ? message.getMessage() : "Not found");
+            default:
+                return errorDecoder.decode(methodsKey, response);
+        }
+
+    }
+}
+

shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/feign/FmkoreaClientFeignConfiguration.java

@@ -1,19 +1,24 @@
 package com.myoa.engineering.crawl.shopping.configuration.feign;
 
 import feign.RequestInterceptor;
-import io.github.resilience4j.circuitbreaker.annotation.CircuitBreaker;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 
 @Configuration
 public class FmkoreaClientFeignConfiguration {
 
+
     @Bean
     public RequestInterceptor requestInterceptor() {
         // TODO ignore 4xx
         return requestTemplate -> new FakeUserAgentInterceptor().apply(requestTemplate);
     }
 
+    @Bean
+    public FmkoreaClientErrorDecoder errorDecoder() {
+        return new FmkoreaClientErrorDecoder();
+    }
+
 /*
     @Bean
     public FmkoreaBoardClient fmkoreaBoardClient(RateLimiterRegistry rateLimiterRegistry,

shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/controller/TestAPIController.java

@@ -1,7 +1,7 @@
 package com.myoa.engineering.crawl.shopping.controller;
 
 import com.myoa.engineering.crawl.shopping.crawlhandler.CrawlHandler;
-import com.myoa.engineering.crawl.shopping.infra.client.fmkorea.FmkoreaBoardClient;
+import com.myoa.engineering.crawl.shopping.infra.client.fmkorea.FmkoreaBoardClientV2;
 import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget;
 import com.slack.api.methods.MethodsClient;
 import com.slack.api.methods.SlackApiException;
@@ -23,14 +23,14 @@ public class TestAPIController {
 
     private final MethodsClient methodsClient;
     private final List<CrawlHandler> crawlHandlers;
-    private final FmkoreaBoardClient fmkoreaBoardClient;
+    private final FmkoreaBoardClientV2 fmkoreaBoardClientV2;
 
     public TestAPIController(MethodsClient methodsClient,
                              List<CrawlHandler> crawlHandlers,
-                             FmkoreaBoardClient fmkoreaBoardClient) {
+                             FmkoreaBoardClientV2 fmkoreaBoardClientV2) {
         this.methodsClient = methodsClient;
         this.crawlHandlers = crawlHandlers;
-        this.fmkoreaBoardClient = fmkoreaBoardClient;
+        this.fmkoreaBoardClientV2 = fmkoreaBoardClientV2;
     }
 
     @GetMapping("/triggers")
@@ -43,16 +43,9 @@ public class TestAPIController {
     @GetMapping("/ratelimiter")
     public void triggerExploit() {
         log.info("will be called page 1");
-        fmkoreaBoardClient.getBoardHtml("/index.php", generateRequestParams(1));
+        fmkoreaBoardClientV2.getBoardHtml(1, null);
         log.info("called page 1");
 
-//        log.info("will be called page 2");
-//        fmkoreaBoardClient.getBoardHtml("/index.php", generateRequestParams(2));
-//        log.info("called page 2");
-//
-//        log.info("will be called page 3");
-//        fmkoreaBoardClient.getBoardHtml("/index.php", generateRequestParams(3));
-//        log.info("called page 3");
     }
 
     private Map<String, String> generateRequestParams(int pageId) {

shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/crawlhandler/FmkoreaCookieService.java

@@ -0,0 +1,22 @@
+package com.myoa.engineering.crawl.shopping.crawlhandler;
+
+import com.myoa.engineering.crawl.shopping.infra.client.fmkorea.FmkoreaBoardClientV2;
+import org.springframework.cache.annotation.Cacheable;
+import org.springframework.stereotype.Service;
+
+
+@Service
+public class FmkoreaCookieService {
+
+    private final FmkoreaBoardClientV2 fmkoreaBoardClientV2;
+
+    public FmkoreaCookieService(FmkoreaBoardClientV2 fmkoreaBoardClientV2) {
+        this.fmkoreaBoardClientV2 = fmkoreaBoardClientV2;
+    }
+
+    @Cacheable(cacheNames = "crawltarget.fmkorea", key = "#root.methodName")
+    public String getCookie() {
+        String fakeHtml = fmkoreaBoardClientV2.getBoardHtml(1, null);
+        return FmkoreaFake430Resolver.resolveFake430(fakeHtml);
+    }
+}

shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/crawlhandler/FmkoreaCrawlHandler.java

@@ -2,7 +2,7 @@ package com.myoa.engineering.crawl.shopping.crawlhandler;
 
 import com.myoa.engineering.crawl.shopping.crawlhandler.parser.FmkoreaArticleParser;
 import com.myoa.engineering.crawl.shopping.domain.entity.v2.Article;
-import com.myoa.engineering.crawl.shopping.infra.client.fmkorea.FmkoreaBoardClient;
+import com.myoa.engineering.crawl.shopping.infra.client.fmkorea.FmkoreaBoardClientV2;
 import com.myoa.engineering.crawl.shopping.service.ArticleCommandService;
 import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget;
 import lombok.extern.slf4j.Slf4j;
@@ -17,15 +17,17 @@ import java.util.stream.Stream;
 @Component
 public class FmkoreaCrawlHandler implements CrawlHandler {
 
-    private final FmkoreaBoardClient fmkoreaBoardClient;
+    private final FmkoreaBoardClientV2 fmkoreaBoardClientV2;
     private final FmkoreaArticleParser fmkoreaArticleParser;
     private final ArticleCommandService articleCommandService;
+    private final FmkoreaCookieService fmkoreaCookieService;
 
-    public FmkoreaCrawlHandler(FmkoreaBoardClient fmkoreaBoardClient,
+    public FmkoreaCrawlHandler(FmkoreaBoardClientV2 fmkoreaBoardClientV2,
-                               FmkoreaArticleParser fmkoreaArticleParser, ArticleCommandService articleCommandService) {
+                               FmkoreaArticleParser fmkoreaArticleParser, ArticleCommandService articleCommandService, FmkoreaCookieService fmkoreaCookieService) {
-        this.fmkoreaBoardClient = fmkoreaBoardClient;
+        this.fmkoreaBoardClientV2 = fmkoreaBoardClientV2;
         this.fmkoreaArticleParser = fmkoreaArticleParser;
         this.articleCommandService = articleCommandService;
+        this.fmkoreaCookieService = fmkoreaCookieService;
     }
 
     @Override
@@ -35,15 +37,12 @@ public class FmkoreaCrawlHandler implements CrawlHandler {
 
     @Override
     public void handle() {
+        String cookie = fmkoreaCookieService.getCookie();
 
-        String fakeHtml = fmkoreaBoardClient.getBoardHtml("/index.php", generateRequestParams(1, null));
+        String boardHtmlPage1 = fmkoreaBoardClientV2.getBoardHtml(1, cookie);
-        String cookie = FmkoreaFake430Resolver.resolveFake430(fakeHtml);
-
-
-        String boardHtmlPage1 = fmkoreaBoardClient.getBoardHtml("/index.php", generateRequestParams(1, cookie));
         List<Article> parsedPage1 = fmkoreaArticleParser.parse(boardHtmlPage1);
 
-        String boardHtmlPage2 = fmkoreaBoardClient.getBoardHtml("/index.php", generateRequestParams(2, cookie));
+        String boardHtmlPage2 = fmkoreaBoardClientV2.getBoardHtml(2, cookie);
         List<Article> parsedPage2 = fmkoreaArticleParser.parse(boardHtmlPage2);
 
         List<Article> merged = Stream.of(parsedPage1, parsedPage2)

shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/model/v2/ArticleModel.java

@@ -23,6 +23,6 @@ public class ArticleModel {
     private ZonedDateTime registeredAt;
 
     public String convertArticletoMessage() {
-        return "- <" + this.getArticleUrl() + "|" + this.getTitle() + ">";
+        return "• <" + this.getArticleUrl() + "|" + this.getTitle() + ">";
     }
 }

shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/ExceptionMessage.java

@@ -0,0 +1,19 @@
+package com.myoa.engineering.crawl.shopping.dto;
+
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+
+@Getter
+@Setter
+@NoArgsConstructor
+@AllArgsConstructor
+public class ExceptionMessage {
+    private String timestamp;
+    private int status;
+    private String error;
+    private String message;
+    private String path;
+
+}

shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/event/handler/ArticleUpsertEventListener.java

@@ -9,8 +9,10 @@ import com.myoa.engineering.crawl.shopping.service.AppUserQueryService;
 import com.myoa.engineering.crawl.shopping.service.SubscribedKeywordCacheService;
 import com.myoa.engineering.crawl.shopping.service.slack.UserNotifyService;
 import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget;
+import com.myoa.engineering.crawl.shopping.util.SlackMessageUtils;
 import com.slack.api.methods.request.chat.ChatPostMessageRequest;
 import com.slack.api.methods.response.chat.ChatPostMessageResponse;
+import com.slack.api.model.block.Blocks;
 import org.springframework.context.event.EventListener;
 import org.springframework.stereotype.Component;
 
@@ -55,7 +57,7 @@ public class ArticleUpsertEventListener {
                     return subscribedKeywords.entrySet()
                                              .stream()
                                              .map(entry -> {
-                                                 List<ArticleModel> filtered = doAhoCorasick(articleMap.get(entry.getKey())).apply(entry.getValue());
+                                                 List<ArticleModel> filtered = doAhocorasick(articleMap.get(entry.getKey()), entry.getValue());
                                                  return UserNotifyModel.builder()
                                                                        .slackId(user.getSlackId())
                                                                        .articles(filtered)
@@ -68,11 +70,14 @@ public class ArticleUpsertEventListener {
 
     private Function<SubscribedKeywordAggregatedModel, List<ArticleModel>> doAhoCorasick(
             List<ArticleModel> articles) {
-        return userTrieModel -> filterAhocorasick(articles, userTrieModel);
+        return userTrieModel -> doAhocorasick(articles, userTrieModel);
     }
 
-    private List<ArticleModel> filterAhocorasick(List<ArticleModel> articles,
+    private List<ArticleModel> doAhocorasick(List<ArticleModel> articles,
-                                                 SubscribedKeywordAggregatedModel trieModel) {
+                                             SubscribedKeywordAggregatedModel trieModel) {
+        if (articles == null || articles.isEmpty()) {
+            return List.of();
+        }
         return articles.stream()
                        .filter(article -> !trieModel.getAhoCorasickTrie()
                                                     .parseText(article.getTitle())
@@ -81,12 +86,19 @@ public class ArticleUpsertEventListener {
     }
 
     private ChatPostMessageResponse notifyMessage(CrawlTarget crawlTarget, List<ArticleModel> articles) {
-        var sb = new StringBuilder();
+        String composited = articles.stream()
-        sb.append("[").append(crawlTarget.getAlias()).append("]\n");
+                                    .map(ArticleModel::convertArticletoMessage)
-        articles.forEach(article -> sb.append(article.convertArticletoMessage()).append("\n"));
+                                    .collect(Collectors.joining("\n"));
-        sb.append("-----------------------------------\n");
+
+        ChatPostMessageRequest request =
+                userNotifyService.generateMessage()
+                                 .blocks(Blocks.asBlocks(
+                                         SlackMessageUtils.ofHeader(crawlTarget.getAlias()),
+                                         SlackMessageUtils.ofSection(composited),
+                                         SlackMessageUtils.ofDivider()
+                                 ))
+                                 .build();
 
-        ChatPostMessageRequest request = userNotifyService.generateMessage(sb.toString()).build();
         return userNotifyService.notify(request);
     }
 
@@ -96,9 +108,12 @@ public class ArticleUpsertEventListener {
             return;
         }
 
-        ChatPostMessageRequest request = userNotifyService.generateMessage(userNotifyModel.toCompositedMessage())
+        ChatPostMessageRequest request =
-                                                          .threadTs(userNotifyModel.getChatPostMessageResponse().getTs())
+                userNotifyService.generateMessage()
-                                                          .build();
+                                 .blocks(Blocks.asBlocks(
+                                         SlackMessageUtils.ofSection(userNotifyModel.toCompositedMessage())
+                                 ))
+                                 .build();
         userNotifyService.notify(request);
     }
 

shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/fmkorea/FmkoreaBoardClient.java

@@ -1,21 +0,0 @@
-package com.myoa.engineering.crawl.shopping.infra.client.fmkorea;
-
-import com.myoa.engineering.crawl.shopping.configuration.feign.FmkoreaClientFeignConfiguration;
-import io.github.resilience4j.circuitbreaker.annotation.CircuitBreaker;
-import io.github.resilience4j.ratelimiter.annotation.RateLimiter;
-import org.springframework.cloud.openfeign.FeignClient;
-import org.springframework.cloud.openfeign.SpringQueryMap;
-import org.springframework.web.bind.annotation.GetMapping;
-import org.springframework.web.bind.annotation.PathVariable;
-
-import java.util.Map;
-
-@FeignClient(value = "fmkorea-board-client", url = "https://www.fmkorea.com", configuration = FmkoreaClientFeignConfiguration.class)
-public interface FmkoreaBoardClient {
-
-    @CircuitBreaker(name = "fmkoreaAvoid429")
-    @RateLimiter(name = "fmkoreaAvoid429")
-    @GetMapping("{boardLink}")
-    String getBoardHtml(@PathVariable("boardLink") String boardLink,
-                        @SpringQueryMap Map<String, String> params);
-}

shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/fmkorea/FmkoreaBoardClientV2.java

@@ -0,0 +1,34 @@
+package com.myoa.engineering.crawl.shopping.infra.client.fmkorea;
+
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.http.HttpStatus;
+import org.springframework.stereotype.Component;
+import org.springframework.web.reactive.function.client.WebClient;
+
+@Slf4j
+@Component
+public class FmkoreaBoardClientV2 {
+
+    private static final String URI_BOARD = "/index.php";
+
+    private final WebClient webClient;
+
+    public FmkoreaBoardClientV2() {
+        webClient = WebClient.builder()
+                             .codecs(configurer -> configurer.defaultCodecs()
+                                                             .maxInMemorySize(2 * 1024 * 1024))
+                             .baseUrl("https://www.fmkorea.com")
+                             .build();
+    }
+
+    public String getBoardHtml(Integer page, String cookie) {
+        return webClient.get()
+                        .uri(builder -> builder.path(URI_BOARD)
+                                               .queryParam("page", page)
+                                               .queryParam("mid", "hotdeal")
+                                               .build())
+                        .header("Cookie", cookie)
+                        .exchangeToMono(clientResponse -> clientResponse.bodyToMono(String.class))
+                        .block();
+    }
+}

shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/scheduler/ParseEventEmitter.java

@@ -2,6 +2,7 @@ package com.myoa.engineering.crawl.shopping.scheduler;
 
 import com.myoa.engineering.crawl.shopping.crawlhandler.CrawlHandler;
 import lombok.extern.slf4j.Slf4j;
+import org.springframework.context.annotation.Profile;
 import org.springframework.scheduling.annotation.EnableScheduling;
 import org.springframework.scheduling.annotation.Scheduled;
 import org.springframework.stereotype.Component;
@@ -11,6 +12,7 @@ import java.util.List;
 @Slf4j
 @Component
 @EnableScheduling
+@Profile("!local")
 public class ParseEventEmitter {
     private final List<CrawlHandler> crawlHandlers;
 

shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/slack/UserNotifyService.java

@@ -39,4 +39,10 @@ public class UserNotifyService {
                                      .username(slackSecretProperties.getUsername())
                                      .text(message);
     }
+
+    public ChatPostMessageRequest.ChatPostMessageRequestBuilder generateMessage() {
+        return ChatPostMessageRequest.builder()
+                                     .channel(slackSecretProperties.getChannel())
+                                     .username(slackSecretProperties.getUsername());
+    }
 }

shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/util/SlackMessageUtils.java

@@ -0,0 +1,32 @@
+package com.myoa.engineering.crawl.shopping.util;
+
+import com.slack.api.model.block.DividerBlock;
+import com.slack.api.model.block.HeaderBlock;
+import com.slack.api.model.block.SectionBlock;
+import com.slack.api.model.block.composition.PlainTextObject;
+
+public final class SlackMessageUtils {
+
+    private SlackMessageUtils() {
+    }
+
+    public static HeaderBlock ofHeader(String message) {
+        return HeaderBlock.builder()
+                          .text(PlainTextObject.builder()
+                                               .text(message)
+                                               .build())
+                          .build();
+    }
+
+    public static DividerBlock ofDivider() {
+        return DividerBlock.builder().build();
+    }
+
+    public static SectionBlock ofSection(String message) {
+        return SectionBlock.builder()
+                           .text(PlainTextObject.builder()
+                                                .text(message)
+                                                .build())
+                           .build();
+    }
+}

shopping-crawler/src/test/java/com/myoa/engineering/crawl/shopping/controller/TestAPIControllerTest.java

@@ -0,0 +1,63 @@
+package com.myoa.engineering.crawl.shopping.controller;
+
+import com.slack.api.methods.MethodsClient;
+import com.slack.api.methods.SlackApiException;
+import com.slack.api.methods.request.chat.ChatPostMessageRequest;
+import com.slack.api.model.block.DividerBlock;
+import com.slack.api.model.block.HeaderBlock;
+import com.slack.api.model.block.LayoutBlock;
+import com.slack.api.model.block.SectionBlock;
+import com.slack.api.model.block.composition.PlainTextObject;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.test.context.ActiveProfiles;
+import org.springframework.test.context.junit.jupiter.SpringExtension;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+@ActiveProfiles("local")
+@ExtendWith(SpringExtension.class)
+@SpringBootTest
+class TestAPIControllerTest {
+
+    @Autowired
+    MethodsClient methodsClient;
+
+    @Test
+    void test1() throws SlackApiException, IOException {
+        List<LayoutBlock> blocks = new ArrayList<>();
+        HeaderBlock asdf = HeaderBlock.builder()
+                                      .text(PlainTextObject.builder()
+                                                           .text("asdf")
+                                                           .build())
+                                      .build();
+
+        DividerBlock dividerBlock = DividerBlock.builder().build();
+        SectionBlock section = SectionBlock.builder()
+                                           .text(PlainTextObject.builder()
+                                                                .text("• asdf")
+                                                                .build())
+                                           .build();
+        blocks.add(asdf);
+        blocks.add(section);
+        blocks.add(section);
+        blocks.add(section);
+        blocks.add(dividerBlock);
+
+        ChatPostMessageRequest request = ChatPostMessageRequest.builder()
+                                                               .channel("notify_shopping")
+                                                               .blocks(blocks)
+                                                               .build();
+        methodsClient.chatPostMessage(request);
+
+    }
+
+    private void notifyMessage() {
+
+    }
+
+}

shopping-crawler/src/test/resources/logback-development.xml

@@ -1,12 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<included>
-  <!-- =========== property BETA ========= -->
-  <property name="DEFAULT_LEVEL" value="${DEFAULT_LEVEL_CONFIG:-INFO}"/>
-  <!-- =========== include appender =========== -->
-  <include resource="org/springframework/boot/logging/logback/defaults.xml"/>
-  <include resource="org/springframework/boot/logging/logback/console-appender.xml"/>
-  <!-- =========== root logger ============== -->
-  <root level="${DEFAULT_LEVEL}">
-    <appender-ref ref="CONSOLE"/>
-  </root>
-</included>

shopping-crawler/src/test/resources/logback-spring.xml

@@ -1,6 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<configuratiown>
-  <springProperty name="DEFAULT_LEVEL_CONFIG" source="log.defaultLevel"/>
-
-  <include resource="logback-development.xml"/>
-</configuratiown>