diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/controller/CrawlAPIController.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/controller/CrawlAPIController.java index c520aad..2e92503 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/controller/CrawlAPIController.java +++ b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/controller/CrawlAPIController.java @@ -1,13 +1,11 @@ package com.myoa.engineering.crawl.ppomppu.processor.controller; -import java.util.List; - +import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; -import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle; import com.myoa.engineering.crawl.ppomppu.processor.dto.FeedParsedResult; import com.myoa.engineering.crawl.ppomppu.processor.service.MessageSenderService; import com.myoa.engineering.crawl.ppomppu.processor.service.PpomppuArticleService; @@ -45,13 +43,14 @@ public class CrawlAPIController { public Mono> crawlBoard(@PathVariable("boardName") PpomppuBoardName boardName) { log.info("got request... {}", boardName); FeedParsedResult result = FeedParsedResult.of(boardName); - Mono> articles = - ppomppuRSSFeedService.getArticles(boardName) - .doOnNext(e -> ppomppuArticleService.filterOnlyNewArticles(boardName, e)) - .doOnNext(e -> messageSenderService.sendMessageToSlack(e)) - .doOnNext(e -> ppomppuArticleService.save(boardName, e)); - return articles.then(Mono.just(APIResponse.success(result.done()))); + Mono publishedMessages = + ppomppuRSSFeedService.getArticles(boardName) + .map(e -> ppomppuArticleService.filterOnlyNewArticles(boardName, e)) + .map(e -> ppomppuArticleService.save(boardName, e)) + .flatMap(messageSenderService::sendMessageToSlack); + + return publishedMessages.then(Mono.just(APIResponse.success(result.done()))); } } diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/PpomppuArticleParser.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/PpomppuArticleParser.java index 20dc7d0..4387642 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/PpomppuArticleParser.java +++ b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/PpomppuArticleParser.java @@ -46,7 +46,7 @@ public final class PpomppuArticleParser { } public static String parseTitle(Element td) { - return td.text(); + return td.getElementsByTag("a").text(); // TODO cdn image extracting } public static String parseArticleUrl(Element td) { diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/PpomppuArticleTransformer.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/PpomppuArticleTransformer.java index 95f4e8a..ede7623 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/PpomppuArticleTransformer.java +++ b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/PpomppuArticleTransformer.java @@ -17,8 +17,11 @@ import com.myoa.engineering.crawl.ppomppu.support.dto.SimpleMessageDTO; * */ public final class PpomppuArticleTransformer { + private PpomppuArticleTransformer() {} + private static final String MESSAGE_FORMAT_V1 = "%s)) `%s` <%s:LINK>"; + private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss") .withZone(ZoneId.of("Asia/Seoul")); @@ -26,7 +29,7 @@ public final class PpomppuArticleTransformer { SimpleMessageDTO.builder() .requestedAt(Instant.now()) .publishedAt(entity.getRegisteredAt()) - .title(String.format("[%s] %s", entity.getBoardName().getMenuName(), entity.getTitle())) + .title(String.format(MESSAGE_FORMAT_V1, entity.getBoardName().getMenuName(), entity.getTitle())) .body(entity.getArticleUrl()) .build(); @@ -44,7 +47,7 @@ public final class PpomppuArticleTransformer { } public static String convertToInlineMessage(PpomppuArticle article) { - return String.format("[%s] %s\n%s", + return String.format(MESSAGE_FORMAT_V1, article.getBoardName().getMenuName(), article.getTitle(), article.getArticleUrl()); } } diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/client/MessageSenderAPIClient.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/client/MessageSenderAPIClient.java index 5110c3e..155214f 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/client/MessageSenderAPIClient.java +++ b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/client/MessageSenderAPIClient.java @@ -1,18 +1,20 @@ package com.myoa.engineering.crawl.ppomppu.processor.infrastructure.client; +import org.springframework.core.ParameterizedTypeReference; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; +import org.springframework.stereotype.Component; +import org.springframework.web.reactive.function.client.WebClient; +import org.springframework.web.reactive.function.client.WebClientRequestException; + import com.myoa.engineering.crawl.ppomppu.processor.dto.constant.WebClientPropertiesUnitName; import com.myoa.engineering.crawl.ppomppu.support.dto.SimpleMessageDTO; import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebClientFilterFactory; import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebFluxExchangeStragiesFactory; import com.myoa.engineering.crawl.ppomppu.support.webclient.properties.WebClientProperties; import com.myoa.engineering.crawl.ppomppu.support.webclient.properties.WebClientProperties.WebClientPropertiesUnit; + import lombok.extern.slf4j.Slf4j; -import org.springframework.core.ParameterizedTypeReference; -import org.springframework.http.HttpHeaders; -import org.springframework.http.MediaType; -import org.springframework.stereotype.Component; -import org.springframework.web.reactive.function.client.WebClient; -import org.springframework.web.reactive.function.client.WebClientRequestException; import reactor.core.publisher.Mono; import reactor.core.scheduler.Schedulers; @@ -35,8 +37,8 @@ public class MessageSenderAPIClient { .baseUrl(webClientPropertiesUnit.getBaseUrl()) .exchangeStrategies(WebFluxExchangeStragiesFactory.ofDefault()) .defaultHeader(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_VALUE) - .filter(WebClientFilterFactory.logRequest()) - .filter(WebClientFilterFactory.logResponse()) +// .filter(WebClientFilterFactory.logRequest()) +// .filter(WebClientFilterFactory.logResponse()) .build(); } @@ -49,7 +51,6 @@ public class MessageSenderAPIClient { .onErrorResume(WebClientRequestException.class, t -> { log.info("Exception occured, ignoring. : {}", t.getClass().getSimpleName()); return Mono.empty(); - }) - .doOnNext(e -> log.info("response: {} ", e)); + }); } } \ No newline at end of file diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/PpomppuArticleService.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/PpomppuArticleService.java index e810c56..ad210bb 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/PpomppuArticleService.java +++ b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/PpomppuArticleService.java @@ -1,16 +1,19 @@ package com.myoa.engineering.crawl.ppomppu.processor.service; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle; import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuBoardFeedStatus; import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository.PpomppuArticleRepository; import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository.PpomppuBoardFeedStatusRepository; import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName; -import java.util.List; -import java.util.Optional; -import java.util.stream.Collectors; + import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; @Slf4j @Service @@ -32,6 +35,7 @@ public class PpomppuArticleService { Long latestArticleId = boardFeedStatus.map(PpomppuBoardFeedStatus::getLatestParsedArticleId) .orElse(0L); + log.info("latestArticleId : {}", latestArticleId); return articles.stream() .filter(e -> e.getArticleId().compareTo(latestArticleId) > 0) .collect(Collectors.toList()); @@ -46,9 +50,12 @@ public class PpomppuArticleService { // save PpomppuBoardFeedStatus Optional boardFeedStatus = ppomppuBoardFeedStatusRepository.findByBoardName(boardName); + log.info("boardName: {}, isPresent?: {}", boardName, boardFeedStatus.isPresent()); boardFeedStatus.ifPresentOrElse(e -> { - e.updateArticleId(latestArticleId); - ppomppuBoardFeedStatusRepository.save(e); + if (latestArticleId.longValue() > 0L) { + e.updateArticleId(latestArticleId); + ppomppuBoardFeedStatusRepository.save(e); + } }, () -> ppomppuBoardFeedStatusRepository.save(PpomppuBoardFeedStatus.of(boardName, latestArticleId))); diff --git a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/scheduler/ParseEventEmitter.java b/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/scheduler/ParseEventEmitter.java index 4a0cc9b..bf94c88 100644 --- a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/scheduler/ParseEventEmitter.java +++ b/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/scheduler/ParseEventEmitter.java @@ -26,15 +26,11 @@ public class ParseEventEmitter { this.processorAPIService = processorAPIService; } - @Scheduled(fixedRate = 60 * 1000L) - public void emitDomesticBoard() { - log.info("[emitDomesticBoard] trigger fired!"); - processorAPIService.emitParseEvent(PpomppuBoardName.PPOMPPU_DOMESTIC_ETC).block(); - } - @Scheduled(fixedRate = 300 * 1000L) - public void emitOverseaBoard() { - log.info("[emitOverseaBoard] trigger fired!"); - processorAPIService.emitParseEvent(PpomppuBoardName.PPOMPPU_OVERSEA_ETC).block(); + public void emitBoards() { + log.info("[emitDomesticBoard] trigger fired!"); + for (PpomppuBoardName boardName : PpomppuBoardName.values()) { + processorAPIService.emitParseEvent(boardName).block(); + } } } diff --git a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/controller/MessageSenderAPIController.java b/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/controller/MessageSenderAPIController.java index 373e5bf..479d101 100644 --- a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/controller/MessageSenderAPIController.java +++ b/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/controller/MessageSenderAPIController.java @@ -5,6 +5,7 @@ import com.myoa.engineering.crawl.ppomppu.support.dto.APIResponse; import com.myoa.engineering.crawl.ppomppu.support.dto.SimpleMessageDTO; import lombok.extern.slf4j.Slf4j; import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; import reactor.core.publisher.Mono; @@ -27,9 +28,7 @@ public class MessageSenderAPIController { } @PostMapping("/messages/sendMessage/messengers/slack") - public Mono> sendMessageToSlack(SimpleMessageDTO dto) { - - log.info("received : {}, \nbody: {}", dto.getTitle(), dto.getBody()); + public Mono> sendMessageToSlack(@RequestBody SimpleMessageDTO dto) { return sender.sendMessage(sender.ofMessage(dto.getBody())) .then(Mono.just(APIResponse.success(dto))); } diff --git a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/controller/TestAPIController.java b/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/controller/TestAPIController.java index 96fcc55..28192e1 100644 --- a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/controller/TestAPIController.java +++ b/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/controller/TestAPIController.java @@ -4,6 +4,8 @@ import com.myoa.engineering.crawl.ppomppu.sender.infrastructure.client.MongeShop import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; + +import lombok.extern.slf4j.Slf4j; import reactor.core.publisher.Mono; /** @@ -12,6 +14,7 @@ import reactor.core.publisher.Mono; * @author Shin Woo-jin (woo-jin.shin@linecorp.com) * @since 2021-11-15 */ +@Slf4j @RestController @RequestMapping("/api/v1") public class TestAPIController { @@ -24,6 +27,7 @@ public class TestAPIController { @GetMapping("/test") public Mono test() { + log.info("received!!!"); return sender.sendMessage(sender.ofMessage("testtesttest!!!")); } } diff --git a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/infrastructure/client/MongeShoppingBotSlackMessageSender.java b/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/infrastructure/client/MongeShoppingBotSlackMessageSender.java index 8ab5b00..6f5b352 100644 --- a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/infrastructure/client/MongeShoppingBotSlackMessageSender.java +++ b/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/infrastructure/client/MongeShoppingBotSlackMessageSender.java @@ -13,7 +13,6 @@ public class MongeShoppingBotSlackMessageSender extends SlackMessageSender { private static final String SLACK_SECRET_UNIT_NAME = "monge_shopping_bot"; private final SlackSecretPropertiesUnit slackProperties; -// private final SlackMessageSender slackMessageSender; public MongeShoppingBotSlackMessageSender(SlackSecretProperties slackSecretProperties) { super(slackSecretProperties.find(SLACK_SECRET_UNIT_NAME).getToken()); diff --git a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/infrastructure/client/SlackMessageSender.java b/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/infrastructure/client/SlackMessageSender.java index 5f349bd..c0f2efb 100644 --- a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/infrastructure/client/SlackMessageSender.java +++ b/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/infrastructure/client/SlackMessageSender.java @@ -33,7 +33,6 @@ public class SlackMessageSender implements MessageSender { .exchangeStrategies(WebFluxExchangeStragiesFactory.ofDefault()) .defaultHeader(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_UTF8_VALUE) .defaultHeader(HttpHeaders.ACCEPT_CHARSET, "UTF-8") - .filter(WebClientFilterFactory.logRequest()) .filter(WebClientFilterFactory.logResponse()) .build(); } @@ -48,8 +47,7 @@ public class SlackMessageSender implements MessageSender { .onErrorResume(WebClientRequestException.class, t -> { log.info("Exception occured, ignoring. : {}", t.getClass().getSimpleName()); return Mono.empty(); - }) - .doOnNext(e -> log.info("response: {}", e)); + }); } } diff --git a/sender/src/main/resources/application.yml b/sender/src/main/resources/application.yml index 670365b..4903239 100644 --- a/sender/src/main/resources/application.yml +++ b/sender/src/main/resources/application.yml @@ -12,11 +12,6 @@ spring: freemarker: enabled: false - jpa: - hibernate: - ddl-auto: create-drop - generate-ddl: true - server: port: 20080 error: diff --git a/support/next_message_tempalte(block_kit_builder).json b/support/next_message_tempalte(block_kit_builder).json new file mode 100644 index 0000000..eae1772 --- /dev/null +++ b/support/next_message_tempalte(block_kit_builder).json @@ -0,0 +1,31 @@ +{ + "blocks": [ + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "기타)) `[쿠팡] 타푸코 이중 진공 스텐 세라믹 코팅 텀블러 (화이트만 11,130원/로켓와우무료) 4 [기타]`" + }, + "accessory": { + "type": "image", + "image_url": "cdn.ppomppu.co.kr/zboard/data3/2021/1121/m_20211121184835_zfyahnow.png", + "alt_text": "alt text for image" + } + }, + { + "type": "divider" + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "기타)) `[롯데온] 1매 149원 올국산 KF94새부리마스크 대형 200매 (29,920원/무료배송) 58 [기타]`" + }, + "accessory": { + "type": "image", + "image_url": "https://s3-media3.fl.yelpcdn.com/bphoto/c7ed05m9lC2EmA3Aruue7A/o.jpg", + "alt_text": "alt text for image" + } + } + ] +} \ No newline at end of file diff --git a/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/dto/code/PpomppuBoardName.java b/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/dto/code/PpomppuBoardName.java index aa15fc2..d66bdc0 100644 --- a/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/dto/code/PpomppuBoardName.java +++ b/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/dto/code/PpomppuBoardName.java @@ -12,7 +12,7 @@ import lombok.Getter; @Getter @AllArgsConstructor public enum PpomppuBoardName { - PPOMPPU_DOMESTIC_ALL("/zboard/zboard.php?id=ppomppu", "전체", true), + PPOMPPU_DOMESTIC_ALL("/zboard/zboard.php?id=ppomppu", "전체", false), PPOMPPU_DOMESTIC_ETC("/zboard/zboard.php?id=ppomppu&category=1", "기타", true), PPOMPPU_DOMESTIC_COMPUTER("/zboard/zboard.php?id=ppomppu&category=4", "컴퓨터", true), PPOMPPU_DOMESTIC_DIGITAL("/zboard/zboard.php?id=ppomppu&category=5", "디지털", true), @@ -24,7 +24,7 @@ public enum PpomppuBoardName { PPOMPPU_DOMESTIC_CLOTHES("/zboard/zboard.php?id=ppomppu&category=12", "의류/잡화", true), PPOMPPU_DOMESTIC_COSMETIC("/zboard/zboard.php?id=ppomppu&category=13", "화장품", true), PPOMPPU_DOMESTIC_OUTDOOR("/zboard/zboard.php?id=ppomppu&category=15", "등산/캠핑", true), - PPOMPPU_OVERSEA_ALL("/zboard/zboard.php?id=ppomppu4", "전체", true), + PPOMPPU_OVERSEA_ALL("/zboard/zboard.php?id=ppomppu4", "전체", false), PPOMPPU_OVERSEA_ETC("/zboard/zboard.php?id=ppomppu4&category=1", "기타", true), PPOMPPU_OVERSEA_APPLIANCES("/zboard/zboard.php?id=ppomppu4&category=7", "가전", true), PPOMPPU_OVERSEA_TVAV("/zboard/zboard.php?id=ppomppu4&category=8", "TV/영상", true), @@ -45,7 +45,7 @@ public enum PpomppuBoardName { public static final String PPOMPPU_URL = "https://www.ppomppu.co.kr"; public static String ofViewPageUrl(String articleUrl) { - return PPOMPPU_URL + "/zboard" + articleUrl; + return PPOMPPU_URL + "/zboard/" + articleUrl; } }