MyoaEngineering
/
ShoppingCrawler
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

[NO-ISSUE] Prettify

This commit is contained in:
woozu-shin 2024-05-15 23:25:55 +09:00
parent b83db38b61
commit ea6909201c
17 changed files with 270 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
shopping-crawler/build.gradle
View File

@ -9,6 +9,8 @@ dependencies {
implementation("org.springframework.boot:spring-boot-starter-web") {
exclude group: "org.springframework.boot", module: "spring-boot-starter-tomcat"
}
implementation("org.springframework.boot:spring-boot-starter-webflux")
implementation("org.springframework.boot:spring-boot-starter-undertow") {
exclude group: "io.undertow", module: "undertow-websockets-jsr"
@ -24,6 +26,7 @@ dependencies {
implementation 'org.ahocorasick:ahocorasick:0.6.3'
implementation "com.slack.api:slack-api-client:1.39.1"
// implementation "io.github.resilience4j:resilience4j-spring-boot3:2.2.0"
implementation 'io.github.resilience4j:resilience4j-all:2.2.0'
implementation "io.github.resilience4j:resilience4j-feign:2.2.0"

40
shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/feign/FmkoreaClientErrorDecoder.java Normal file
View File

@ -0,0 +1,40 @@
package com.myoa.engineering.crawl.shopping.configuration.feign;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.myoa.engineering.crawl.shopping.dto.ExceptionMessage;
import feign.Response;
import feign.codec.ErrorDecoder;
import io.undertow.util.BadRequestException;
import javassist.NotFoundException;
import lombok.extern.slf4j.Slf4j;
import java.io.IOException;
import java.io.InputStream;
@Slf4j
public class FmkoreaClientErrorDecoder implements ErrorDecoder {
private final ErrorDecoder errorDecoder = new Default();
@Override
public Exception decode(String methodsKey, Response response) {
ExceptionMessage message = null;
try (InputStream bodyIs = response.body()
.asInputStream()) {
ObjectMapper mapper = new ObjectMapper();
message = mapper.readValue(bodyIs, ExceptionMessage.class);
} catch (IOException e) {
return new Exception(e.getMessage());
}
switch (response.status()) {
case 400:
return new BadRequestException(message.getMessage() != null ? message.getMessage() : "Bad Request");
case 404:
return new NotFoundException(message.getMessage() != null ? message.getMessage() : "Not found");
default:
return errorDecoder.decode(methodsKey, response);
}
}
}

7
shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/feign/FmkoreaClientFeignConfiguration.java
View File

@ -1,19 +1,24 @@
package com.myoa.engineering.crawl.shopping.configuration.feign;
import feign.RequestInterceptor;
import io.github.resilience4j.circuitbreaker.annotation.CircuitBreaker;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@Configuration
public class FmkoreaClientFeignConfiguration {
@Bean
public RequestInterceptor requestInterceptor() {
// TODO ignore 4xx
return requestTemplate -> new FakeUserAgentInterceptor().apply(requestTemplate);
}
@Bean
public FmkoreaClientErrorDecoder errorDecoder() {
return new FmkoreaClientErrorDecoder();
}
/*
@Bean
public FmkoreaBoardClient fmkoreaBoardClient(RateLimiterRegistry rateLimiterRegistry,

17
shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/controller/TestAPIController.java
View File

@ -1,7 +1,7 @@
package com.myoa.engineering.crawl.shopping.controller;
import com.myoa.engineering.crawl.shopping.crawlhandler.CrawlHandler;
import com.myoa.engineering.crawl.shopping.infra.client.fmkorea.FmkoreaBoardClient;
import com.myoa.engineering.crawl.shopping.infra.client.fmkorea.FmkoreaBoardClientV2;
import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget;
import com.slack.api.methods.MethodsClient;
import com.slack.api.methods.SlackApiException;
@ -23,14 +23,14 @@ public class TestAPIController {
private final MethodsClient methodsClient;
private final List<CrawlHandler> crawlHandlers;
private final FmkoreaBoardClient fmkoreaBoardClient;
private final FmkoreaBoardClientV2 fmkoreaBoardClientV2;
public TestAPIController(MethodsClient methodsClient,
List<CrawlHandler> crawlHandlers,
FmkoreaBoardClient fmkoreaBoardClient) {
FmkoreaBoardClientV2 fmkoreaBoardClientV2) {
this.methodsClient = methodsClient;
this.crawlHandlers = crawlHandlers;
this.fmkoreaBoardClient = fmkoreaBoardClient;
this.fmkoreaBoardClientV2 = fmkoreaBoardClientV2;
}
@GetMapping("/triggers")
@ -43,16 +43,9 @@ public class TestAPIController {
@GetMapping("/ratelimiter")
public void triggerExploit() {
log.info("will be called page 1");
fmkoreaBoardClient.getBoardHtml("/index.php", generateRequestParams(1));
fmkoreaBoardClientV2.getBoardHtml(1, null);
log.info("called page 1");
// log.info("will be called page 2");
// fmkoreaBoardClient.getBoardHtml("/index.php", generateRequestParams(2));
// log.info("called page 2");
//
// log.info("will be called page 3");
// fmkoreaBoardClient.getBoardHtml("/index.php", generateRequestParams(3));
// log.info("called page 3");
}
private Map<String, String> generateRequestParams(int pageId) {

22
shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/crawlhandler/FmkoreaCookieService.java Normal file
View File

@ -0,0 +1,22 @@
package com.myoa.engineering.crawl.shopping.crawlhandler;
import com.myoa.engineering.crawl.shopping.infra.client.fmkorea.FmkoreaBoardClientV2;
import org.springframework.cache.annotation.Cacheable;
import org.springframework.stereotype.Service;
@Service
public class FmkoreaCookieService {
private final FmkoreaBoardClientV2 fmkoreaBoardClientV2;
public FmkoreaCookieService(FmkoreaBoardClientV2 fmkoreaBoardClientV2) {
this.fmkoreaBoardClientV2 = fmkoreaBoardClientV2;
}
@Cacheable(cacheNames = "crawltarget.fmkorea", key = "#root.methodName")
public String getCookie() {
String fakeHtml = fmkoreaBoardClientV2.getBoardHtml(1, null);
return FmkoreaFake430Resolver.resolveFake430(fakeHtml);
}
}

21
shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/crawlhandler/FmkoreaCrawlHandler.java
View File

@ -2,7 +2,7 @@ package com.myoa.engineering.crawl.shopping.crawlhandler;
import com.myoa.engineering.crawl.shopping.crawlhandler.parser.FmkoreaArticleParser;
import com.myoa.engineering.crawl.shopping.domain.entity.v2.Article;
import com.myoa.engineering.crawl.shopping.infra.client.fmkorea.FmkoreaBoardClient;
import com.myoa.engineering.crawl.shopping.infra.client.fmkorea.FmkoreaBoardClientV2;
import com.myoa.engineering.crawl.shopping.service.ArticleCommandService;
import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget;
import lombok.extern.slf4j.Slf4j;
@ -17,15 +17,17 @@ import java.util.stream.Stream;
@Component
public class FmkoreaCrawlHandler implements CrawlHandler {
private final FmkoreaBoardClient fmkoreaBoardClient;
private final FmkoreaBoardClientV2 fmkoreaBoardClientV2;
private final FmkoreaArticleParser fmkoreaArticleParser;
private final ArticleCommandService articleCommandService;
private final FmkoreaCookieService fmkoreaCookieService;
public FmkoreaCrawlHandler(FmkoreaBoardClient fmkoreaBoardClient,
FmkoreaArticleParser fmkoreaArticleParser, ArticleCommandService articleCommandService) {
this.fmkoreaBoardClient = fmkoreaBoardClient;
public FmkoreaCrawlHandler(FmkoreaBoardClientV2 fmkoreaBoardClientV2,
FmkoreaArticleParser fmkoreaArticleParser, ArticleCommandService articleCommandService, FmkoreaCookieService fmkoreaCookieService) {
this.fmkoreaBoardClientV2 = fmkoreaBoardClientV2;
this.fmkoreaArticleParser = fmkoreaArticleParser;
this.articleCommandService = articleCommandService;
this.fmkoreaCookieService = fmkoreaCookieService;
}
@Override
@ -35,15 +37,12 @@ public class FmkoreaCrawlHandler implements CrawlHandler {
@Override
public void handle() {
String cookie = fmkoreaCookieService.getCookie();
String fakeHtml = fmkoreaBoardClient.getBoardHtml("/index.php", generateRequestParams(1, null));
String cookie = FmkoreaFake430Resolver.resolveFake430(fakeHtml);
String boardHtmlPage1 = fmkoreaBoardClient.getBoardHtml("/index.php", generateRequestParams(1, cookie));
String boardHtmlPage1 = fmkoreaBoardClientV2.getBoardHtml(1, cookie);
List<Article> parsedPage1 = fmkoreaArticleParser.parse(boardHtmlPage1);
String boardHtmlPage2 = fmkoreaBoardClient.getBoardHtml("/index.php", generateRequestParams(2, cookie));
String boardHtmlPage2 = fmkoreaBoardClientV2.getBoardHtml(2, cookie);
List<Article> parsedPage2 = fmkoreaArticleParser.parse(boardHtmlPage2);
List<Article> merged = Stream.of(parsedPage1, parsedPage2)

2
shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/model/v2/ArticleModel.java
View File

@ -23,6 +23,6 @@ public class ArticleModel {
private ZonedDateTime registeredAt;
public String convertArticletoMessage() {
return "- <" + this.getArticleUrl() + "|" + this.getTitle() + ">";
return "• <" + this.getArticleUrl() + "|" + this.getTitle() + ">";
}
}

19
shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/ExceptionMessage.java Normal file
View File

@ -0,0 +1,19 @@
package com.myoa.engineering.crawl.shopping.dto;
import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
@Getter
@Setter
@NoArgsConstructor
@AllArgsConstructor
public class ExceptionMessage {
private String timestamp;
private int status;
private String error;
private String message;
private String path;
}

39
shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/event/handler/ArticleUpsertEventListener.java
View File

@ -9,8 +9,10 @@ import com.myoa.engineering.crawl.shopping.service.AppUserQueryService;
import com.myoa.engineering.crawl.shopping.service.SubscribedKeywordCacheService;
import com.myoa.engineering.crawl.shopping.service.slack.UserNotifyService;
import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget;
import com.myoa.engineering.crawl.shopping.util.SlackMessageUtils;
import com.slack.api.methods.request.chat.ChatPostMessageRequest;
import com.slack.api.methods.response.chat.ChatPostMessageResponse;
import com.slack.api.model.block.Blocks;
import org.springframework.context.event.EventListener;
import org.springframework.stereotype.Component;
@ -55,7 +57,7 @@ public class ArticleUpsertEventListener {
return subscribedKeywords.entrySet()
.stream()
.map(entry -> {
List<ArticleModel> filtered = doAhoCorasick(articleMap.get(entry.getKey())).apply(entry.getValue());
List<ArticleModel> filtered = doAhocorasick(articleMap.get(entry.getKey()), entry.getValue());
return UserNotifyModel.builder()
.slackId(user.getSlackId())
.articles(filtered)
@ -68,11 +70,14 @@ public class ArticleUpsertEventListener {
private Function<SubscribedKeywordAggregatedModel, List<ArticleModel>> doAhoCorasick(
List<ArticleModel> articles) {
return userTrieModel -> filterAhocorasick(articles, userTrieModel);
return userTrieModel -> doAhocorasick(articles, userTrieModel);
}
private List<ArticleModel> filterAhocorasick(List<ArticleModel> articles,
SubscribedKeywordAggregatedModel trieModel) {
private List<ArticleModel> doAhocorasick(List<ArticleModel> articles,
SubscribedKeywordAggregatedModel trieModel) {
if (articles == null || articles.isEmpty()) {
return List.of();
}
return articles.stream()
.filter(article -> !trieModel.getAhoCorasickTrie()
.parseText(article.getTitle())
@ -81,12 +86,19 @@ public class ArticleUpsertEventListener {
}
private ChatPostMessageResponse notifyMessage(CrawlTarget crawlTarget, List<ArticleModel> articles) {
var sb = new StringBuilder();
sb.append("[").append(crawlTarget.getAlias()).append("]\n");
articles.forEach(article -> sb.append(article.convertArticletoMessage()).append("\n"));
sb.append("-----------------------------------\n");
String composited = articles.stream()
.map(ArticleModel::convertArticletoMessage)
.collect(Collectors.joining("\n"));
ChatPostMessageRequest request =
userNotifyService.generateMessage()
.blocks(Blocks.asBlocks(
SlackMessageUtils.ofHeader(crawlTarget.getAlias()),
SlackMessageUtils.ofSection(composited),
SlackMessageUtils.ofDivider()
))
.build();
ChatPostMessageRequest request = userNotifyService.generateMessage(sb.toString()).build();
return userNotifyService.notify(request);
}
@ -96,9 +108,12 @@ public class ArticleUpsertEventListener {
return;
}
ChatPostMessageRequest request = userNotifyService.generateMessage(userNotifyModel.toCompositedMessage())
.threadTs(userNotifyModel.getChatPostMessageResponse().getTs())
.build();
ChatPostMessageRequest request =
userNotifyService.generateMessage()
.blocks(Blocks.asBlocks(
SlackMessageUtils.ofSection(userNotifyModel.toCompositedMessage())
))
.build();
userNotifyService.notify(request);
}

21
shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/fmkorea/FmkoreaBoardClient.java
View File

@ -1,21 +0,0 @@
package com.myoa.engineering.crawl.shopping.infra.client.fmkorea;
import com.myoa.engineering.crawl.shopping.configuration.feign.FmkoreaClientFeignConfiguration;
import io.github.resilience4j.circuitbreaker.annotation.CircuitBreaker;
import io.github.resilience4j.ratelimiter.annotation.RateLimiter;
import org.springframework.cloud.openfeign.FeignClient;
import org.springframework.cloud.openfeign.SpringQueryMap;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import java.util.Map;
@FeignClient(value = "fmkorea-board-client", url = "https://www.fmkorea.com", configuration = FmkoreaClientFeignConfiguration.class)
public interface FmkoreaBoardClient {
@CircuitBreaker(name = "fmkoreaAvoid429")
@RateLimiter(name = "fmkoreaAvoid429")
@GetMapping("{boardLink}")
String getBoardHtml(@PathVariable("boardLink") String boardLink,
@SpringQueryMap Map<String, String> params);
}

34
shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/fmkorea/FmkoreaBoardClientV2.java Normal file
View File

@ -0,0 +1,34 @@
package com.myoa.engineering.crawl.shopping.infra.client.fmkorea;
import lombok.extern.slf4j.Slf4j;
import org.springframework.http.HttpStatus;
import org.springframework.stereotype.Component;
import org.springframework.web.reactive.function.client.WebClient;
@Slf4j
@Component
public class FmkoreaBoardClientV2 {
private static final String URI_BOARD = "/index.php";
private final WebClient webClient;
public FmkoreaBoardClientV2() {
webClient = WebClient.builder()
.codecs(configurer -> configurer.defaultCodecs()
.maxInMemorySize(2 * 1024 * 1024))
.baseUrl("https://www.fmkorea.com")
.build();
}
public String getBoardHtml(Integer page, String cookie) {
return webClient.get()
.uri(builder -> builder.path(URI_BOARD)
.queryParam("page", page)
.queryParam("mid", "hotdeal")
.build())
.header("Cookie", cookie)
.exchangeToMono(clientResponse -> clientResponse.bodyToMono(String.class))
.block();
}
}

2
shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/scheduler/ParseEventEmitter.java
View File

@ -2,6 +2,7 @@ package com.myoa.engineering.crawl.shopping.scheduler;
import com.myoa.engineering.crawl.shopping.crawlhandler.CrawlHandler;
import lombok.extern.slf4j.Slf4j;
import org.springframework.context.annotation.Profile;
import org.springframework.scheduling.annotation.EnableScheduling;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
@ -11,6 +12,7 @@ import java.util.List;
@Slf4j
@Component
@EnableScheduling
@Profile("!local")
public class ParseEventEmitter {
private final List<CrawlHandler> crawlHandlers;

6
shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/slack/UserNotifyService.java
View File

@ -39,4 +39,10 @@ public class UserNotifyService {
.username(slackSecretProperties.getUsername())
.text(message);
}
public ChatPostMessageRequest.ChatPostMessageRequestBuilder generateMessage() {
return ChatPostMessageRequest.builder()
.channel(slackSecretProperties.getChannel())
.username(slackSecretProperties.getUsername());
}
}

32
shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/util/SlackMessageUtils.java Normal file
View File

@ -0,0 +1,32 @@
package com.myoa.engineering.crawl.shopping.util;
import com.slack.api.model.block.DividerBlock;
import com.slack.api.model.block.HeaderBlock;
import com.slack.api.model.block.SectionBlock;
import com.slack.api.model.block.composition.PlainTextObject;
public final class SlackMessageUtils {
private SlackMessageUtils() {
}
public static HeaderBlock ofHeader(String message) {
return HeaderBlock.builder()
.text(PlainTextObject.builder()
.text(message)
.build())
.build();
}
public static DividerBlock ofDivider() {
return DividerBlock.builder().build();
}
public static SectionBlock ofSection(String message) {
return SectionBlock.builder()
.text(PlainTextObject.builder()
.text(message)
.build())
.build();
}
}

63
shopping-crawler/src/test/java/com/myoa/engineering/crawl/shopping/controller/TestAPIControllerTest.java Normal file
View File

@ -0,0 +1,63 @@
package com.myoa.engineering.crawl.shopping.controller;
import com.slack.api.methods.MethodsClient;
import com.slack.api.methods.SlackApiException;
import com.slack.api.methods.request.chat.ChatPostMessageRequest;
import com.slack.api.model.block.DividerBlock;
import com.slack.api.model.block.HeaderBlock;
import com.slack.api.model.block.LayoutBlock;
import com.slack.api.model.block.SectionBlock;
import com.slack.api.model.block.composition.PlainTextObject;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.ActiveProfiles;
import org.springframework.test.context.junit.jupiter.SpringExtension;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
@ActiveProfiles("local")
@ExtendWith(SpringExtension.class)
@SpringBootTest
class TestAPIControllerTest {
@Autowired
MethodsClient methodsClient;
@Test
void test1() throws SlackApiException, IOException {
List<LayoutBlock> blocks = new ArrayList<>();
HeaderBlock asdf = HeaderBlock.builder()
.text(PlainTextObject.builder()
.text("asdf")
.build())
.build();
DividerBlock dividerBlock = DividerBlock.builder().build();
SectionBlock section = SectionBlock.builder()
.text(PlainTextObject.builder()
.text("• asdf")
.build())
.build();
blocks.add(asdf);
blocks.add(section);
blocks.add(section);
blocks.add(section);
blocks.add(dividerBlock);
ChatPostMessageRequest request = ChatPostMessageRequest.builder()
.channel("notify_shopping")
.blocks(blocks)
.build();
methodsClient.chatPostMessage(request);
}
private void notifyMessage() {
}
}

12
shopping-crawler/src/test/resources/logback-development.xml
View File

@ -1,12 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<included>
<!-- =========== property BETA ========= -->
<property name="DEFAULT_LEVEL" value="${DEFAULT_LEVEL_CONFIG:-INFO}"/>
<!-- =========== include appender =========== -->
<include resource="org/springframework/boot/logging/logback/defaults.xml"/>
<include resource="org/springframework/boot/logging/logback/console-appender.xml"/>
<!-- =========== root logger ============== -->
<root level="${DEFAULT_LEVEL}">
<appender-ref ref="CONSOLE"/>
</root>
</included>

6
shopping-crawler/src/test/resources/logback-spring.xml
View File

@ -1,6 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<configuratiown>
<springProperty name="DEFAULT_LEVEL_CONFIG" source="log.defaultLevel"/>
<include resource="logback-development.xml"/>
</configuratiown>