diff --git a/build.gradle b/build.gradle index 265f4ba..c09e9f9 100644 --- a/build.gradle +++ b/build.gradle @@ -32,6 +32,17 @@ allprojects { } } + ext { + set('springCloudVersion', "2020.0.4") + } + + dependencyManagement { + imports { + mavenBom "org.springframework.cloud:spring-cloud-dependencies:${springCloudVersion}" + } + } + + /* task initSourceFolders { sourceSets*.java.srcDirs*.each { if( !it.exists() ) { diff --git a/processor/build.gradle b/processor/build.gradle index ffc9e1d..eb68c4c 100644 --- a/processor/build.gradle +++ b/processor/build.gradle @@ -1,3 +1,4 @@ + dependencies { developmentOnly 'org.springframework.boot:spring-boot-devtools' runtimeOnly 'com.h2database:h2' @@ -9,6 +10,8 @@ dependencies { implementation 'org.springframework.boot:spring-boot-starter-webflux' implementation 'org.springframework.boot:spring-boot-starter-data-jpa' implementation 'org.springframework.boot:spring-boot-starter-jdbc' + implementation 'org.springframework.cloud:spring-cloud-starter-config' + implementation 'org.springframework.boot:spring-boot-starter-actuator' implementation 'com.rometools:rome:1.16.0' implementation 'org.jsoup:jsoup:1.14.2' implementation 'com.h2database:h2:1.4.200' diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/ProcessorApplication.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/ProcessorApplication.java index b310e75..eb9f40c 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/ProcessorApplication.java +++ b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/ProcessorApplication.java @@ -2,6 +2,9 @@ package com.myoa.engineering.crawl.ppomppu.processor; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.context.annotation.Import; + +import com.myoa.engineering.crawl.ppomppu.support.webclient.PpomppuNotifierWebClientConfiguration; /** * ProcessorApplication @@ -9,6 +12,7 @@ import org.springframework.boot.autoconfigure.SpringBootApplication; * @since 2021-08-20 * */ +@Import({ PpomppuNotifierWebClientConfiguration.class }) @SpringBootApplication public class ProcessorApplication { diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/H2ConsoleConfiguration.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/H2ConsoleConfiguration.java index d0d1a9b..3fd4bd3 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/H2ConsoleConfiguration.java +++ b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/H2ConsoleConfiguration.java @@ -11,7 +11,7 @@ import org.springframework.context.event.ContextRefreshedEvent; import org.springframework.context.event.EventListener; @Slf4j -@Profile("development") +@Profile({"datasource-local", "datasource-development"}) @Configuration public class H2ConsoleConfiguration { diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/PpomppuDatasourceConfiguration.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/PpomppuDatasourceConfiguration.java index 64c0a52..b78c475 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/PpomppuDatasourceConfiguration.java +++ b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/PpomppuDatasourceConfiguration.java @@ -32,7 +32,7 @@ import org.springframework.transaction.PlatformTransactionManager; ) public class PpomppuDatasourceConfiguration { - private static final String DATA_SOURCE_UNIT_NAME = "ppomppu"; + private static final String DATA_SOURCE_UNIT_NAME = "ppn_mysql"; private final DatasourceProperties dataSourceProeprties; private final HikariProperties hikariProperties; @@ -52,8 +52,9 @@ public class PpomppuDatasourceConfiguration { final HikariConfig hikariConfig = new HikariConfig(); hikariConfig.setJdbcUrl(dataSourcePropertiesUnit.toCompletedJdbcUrl()); - hikariConfig.setUsername("sa"); - hikariConfig.setPassword("sa"); + hikariConfig.setDriverClassName(dataSourcePropertiesUnit.getDriverClassName()); + hikariConfig.setUsername(dataSourcePropertiesUnit.getUsername()); + hikariConfig.setPassword(dataSourcePropertiesUnit.getPassword()); hikariConfig.setAutoCommit(hikariProperties.getAutoCommit()); hikariConfig.setMaximumPoolSize(hikariProperties.getMaximumPoolSize()); hikariConfig.setMinimumIdle(hikariProperties.getMinimumIdle()); diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/properties/DatasourceProperties.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/properties/DatasourceProperties.java index 34c0ac5..351eb13 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/properties/DatasourceProperties.java +++ b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/properties/DatasourceProperties.java @@ -2,6 +2,7 @@ package com.myoa.engineering.crawl.ppomppu.processor.configuration.properties; import com.myoa.engineering.crawl.ppomppu.support.util.ObjectUtil; import java.util.List; + import lombok.Getter; import lombok.Setter; import org.springframework.boot.context.properties.ConfigurationProperties; @@ -10,7 +11,7 @@ import org.springframework.stereotype.Component; @Component @Setter @Getter -@ConfigurationProperties(prefix = "datasource") +@ConfigurationProperties(prefix = "infra.database") public class DatasourceProperties { private List units; @@ -22,14 +23,17 @@ public class DatasourceProperties { private String unitName; private String schemaName; private String connectionParameters; - private String dbConnectionUrl; - private Boolean simpleConnectionUrl; + private String datasourceUrl; + private Boolean isSimpleConnectionUrl; + private String username; + private String password; + private String driverClassName; public String toCompletedJdbcUrl() { - if (ObjectUtil.isEmpty(simpleConnectionUrl) || simpleConnectionUrl == false) { - return String.format("%s/%s?%s", dbConnectionUrl, schemaName, connectionParameters); + if (ObjectUtil.isEmpty(isSimpleConnectionUrl) || isSimpleConnectionUrl == false) { + return String.format("%s/%s?%s", datasourceUrl, schemaName, connectionParameters); } - return dbConnectionUrl; + return datasourceUrl; } } diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/properties/HibernateProperties.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/properties/HibernateProperties.java index 8de1649..4543d8a 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/properties/HibernateProperties.java +++ b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/properties/HibernateProperties.java @@ -3,6 +3,9 @@ package com.myoa.engineering.crawl.ppomppu.processor.configuration.properties; import java.util.List; import lombok.Getter; import lombok.Setter; + +import org.hibernate.dialect.MySQL8Dialect; +import org.hibernate.dialect.MySQLDialect; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.stereotype.Component; diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/controller/CrawlAPIController.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/controller/CrawlAPIController.java index 3592278..2e92503 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/controller/CrawlAPIController.java +++ b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/controller/CrawlAPIController.java @@ -1,17 +1,19 @@ package com.myoa.engineering.crawl.ppomppu.processor.controller; -import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle; -import com.myoa.engineering.crawl.ppomppu.processor.dto.FeedParsedResult; -import com.myoa.engineering.crawl.ppomppu.processor.service.PpomppuArticleService; -import com.myoa.engineering.crawl.ppomppu.processor.service.PpomppuFeedService; -import com.myoa.engineering.crawl.ppomppu.support.dto.APIResponse; -import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName; -import java.util.List; -import lombok.extern.slf4j.Slf4j; +import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; + +import com.myoa.engineering.crawl.ppomppu.processor.dto.FeedParsedResult; +import com.myoa.engineering.crawl.ppomppu.processor.service.MessageSenderService; +import com.myoa.engineering.crawl.ppomppu.processor.service.PpomppuArticleService; +import com.myoa.engineering.crawl.ppomppu.processor.service.PpomppuFeedService; +import com.myoa.engineering.crawl.ppomppu.support.dto.APIResponse; +import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName; + +import lombok.extern.slf4j.Slf4j; import reactor.core.publisher.Mono; /** @@ -27,22 +29,28 @@ public class CrawlAPIController { private final PpomppuFeedService ppomppuRSSFeedService; private final PpomppuArticleService ppomppuArticleService; + private final MessageSenderService messageSenderService; public CrawlAPIController(PpomppuFeedService ppomppuRSSFeedService, - PpomppuArticleService ppomppuArticleService) { + PpomppuArticleService ppomppuArticleService, + MessageSenderService messageSenderService) { this.ppomppuRSSFeedService = ppomppuRSSFeedService; this.ppomppuArticleService = ppomppuArticleService; + this.messageSenderService = messageSenderService; } @PostMapping("/boards/{boardName}") public Mono> crawlBoard(@PathVariable("boardName") PpomppuBoardName boardName) { log.info("got request... {}", boardName); FeedParsedResult result = FeedParsedResult.of(boardName); - Mono> articles = ppomppuRSSFeedService.getArticles(boardName) - .doOnNext(e -> ppomppuArticleService.filterOnlyNewArticles(boardName, e)) - .doOnNext(e -> ppomppuArticleService.save(boardName, e)); - return articles.then(Mono.just(APIResponse.success(result.done()))); + Mono publishedMessages = + ppomppuRSSFeedService.getArticles(boardName) + .map(e -> ppomppuArticleService.filterOnlyNewArticles(boardName, e)) + .map(e -> ppomppuArticleService.save(boardName, e)) + .flatMap(messageSenderService::sendMessageToSlack); + + return publishedMessages.then(Mono.just(APIResponse.success(result.done()))); } } diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/PpomppuArticleParser.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/PpomppuArticleParser.java new file mode 100644 index 0000000..4387642 --- /dev/null +++ b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/PpomppuArticleParser.java @@ -0,0 +1,79 @@ +package com.myoa.engineering.crawl.ppomppu.processor.dto; + +import java.time.Instant; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; + +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; + +import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle; +import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName; + +/** + * PpomppuArticleTransformer + * + * @author Shin Woo-jin (woozu.shin@kakaoent.com) + * @since 2021-09-08 + */ +public final class PpomppuArticleParser { + + private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yy.MM.dd HH:mm:ss") + .withZone(ZoneId.of("Asia/Seoul")); + + private PpomppuArticleParser() {} + + public static PpomppuArticle toArticle(Elements articleElement) { + final long articleId = PpomppuArticleParser.parseArticleId(articleElement.get(0)); + final String title = PpomppuArticleParser.parseTitle(articleElement.get(2)); + final String articleUrl = PpomppuArticleParser.parseArticleUrl(articleElement.get(2)); + final int recommended = PpomppuArticleParser.parseRecommended(articleElement.get(6)); + final int hit = PpomppuArticleParser.parseHit(articleElement.get(7)); + final Instant registeredAt = PpomppuArticleParser.parseRegisteredAt(articleElement.get(5)); + + return PpomppuArticle.builder() + .articleId(articleId) + .title(title) + .articleUrl(articleUrl) + .recommended(recommended) + .hit(hit) + .registeredAt(registeredAt) + .build(); + } + + public static Long parseArticleId(Element td) { + return Long.parseLong(td.text().trim()); + } + + public static String parseTitle(Element td) { + return td.getElementsByTag("a").text(); // TODO cdn image extracting + } + + public static String parseArticleUrl(Element td) { + return PpomppuBoardName.ofViewPageUrl(td.getElementsByTag("a").attr("href")); + } + + public static Integer parseRecommended(Element td) { + final String voteString = td.text(); + final int recommended; + + if (voteString.isEmpty()) { + recommended = 0; + } else { + final int voteUp = Integer.parseInt(td.text().split(" - ")[0]); + final int voteDown = Integer.parseInt(td.text().split(" - ")[1]); + recommended = voteUp - voteDown; + } + return recommended; + } + + public static Integer parseHit(Element td) { + return Integer.parseInt(td.text()); + } + + public static Instant parseRegisteredAt(Element td) { + final String registeredAtString = td.attr("title"); + return DATE_TIME_FORMATTER.parse(registeredAtString, Instant::from); + } + +} diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/PpomppuArticleTransformer.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/PpomppuArticleTransformer.java index 18bd30f..ede7623 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/PpomppuArticleTransformer.java +++ b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/PpomppuArticleTransformer.java @@ -1,77 +1,53 @@ package com.myoa.engineering.crawl.ppomppu.processor.dto; -import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle; -import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName; import java.time.Instant; import java.time.ZoneId; import java.time.format.DateTimeFormatter; -import org.jsoup.nodes.Element; -import org.jsoup.select.Elements; +import java.util.List; +import java.util.function.Function; +import java.util.stream.Collectors; + +import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle; +import com.myoa.engineering.crawl.ppomppu.support.dto.SimpleMessageDTO; /** * PpomppuArticleTransformer + * @author Shin Woo-jin (woo-jin.shin@linecorp.com) + * @since 2021-11-21 * - * @author Shin Woo-jin (woozu.shin@kakaoent.com) - * @since 2021-09-08 */ public final class PpomppuArticleTransformer { - private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yy.MM.dd HH:mm:ss") - .withZone(ZoneId.of("Asia/Seoul")); - private PpomppuArticleTransformer() {} - public static PpomppuArticle toArticle(Elements articleElement) { - final long articleId = PpomppuArticleTransformer.toArticleId(articleElement.get(0)); - final String title = PpomppuArticleTransformer.toTitle(articleElement.get(2)); - final String articleUrl = PpomppuArticleTransformer.toArticleUrl(articleElement.get(2)); - final int recommended = PpomppuArticleTransformer.toRecommended(articleElement.get(6)); - final int hit = PpomppuArticleTransformer.toHit(articleElement.get(7)); - final Instant registeredAt = PpomppuArticleTransformer.toRegisteredAt(articleElement.get(5)); + private static final String MESSAGE_FORMAT_V1 = "%s)) `%s` <%s:LINK>"; - return PpomppuArticle.builder() - .articleId(articleId) - .title(title) - .articleUrl(articleUrl) - .recommended(recommended) - .hit(hit) - .registeredAt(registeredAt) - .build(); + private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss") + .withZone(ZoneId.of("Asia/Seoul")); + + public static final Function TRANSFORM_TO_MESSAGE_DTO = entity -> + SimpleMessageDTO.builder() + .requestedAt(Instant.now()) + .publishedAt(entity.getRegisteredAt()) + .title(String.format(MESSAGE_FORMAT_V1, entity.getBoardName().getMenuName(), entity.getTitle())) + .body(entity.getArticleUrl()) + .build(); + + // https://stackoverflow.com/questions/24882927/using-streams-to-convert-a-list-of-objects-into-a-string-obtained-from-the-tostr + public static SimpleMessageDTO transform(List articles) { + Instant requestedAt = Instant.now(); + String body = articles.stream() + .map(PpomppuArticleTransformer::convertToInlineMessage) + .collect(Collectors.joining("\n\n")); + return SimpleMessageDTO.builder() + .requestedAt(requestedAt) + .title(DATE_TIME_FORMATTER.format(requestedAt)) + .body(body) + .build(); } - public static Long toArticleId(Element td) { - return Long.parseLong(td.text().trim()); + public static String convertToInlineMessage(PpomppuArticle article) { + return String.format(MESSAGE_FORMAT_V1, + article.getBoardName().getMenuName(), article.getTitle(), article.getArticleUrl()); } - - public static String toTitle(Element td) { - return td.text(); - } - - public static String toArticleUrl(Element td) { - return PpomppuBoardName.ofViewPageUrl(td.getElementsByTag("a").attr("href")); - } - - public static Integer toRecommended(Element td) { - final String voteString = td.text(); - final int recommended; - - if (voteString.isEmpty()) { - recommended = 0; - } else { - final int voteUp = Integer.parseInt(td.text().split(" - ")[0]); - final int voteDown = Integer.parseInt(td.text().split(" - ")[1]); - recommended = voteUp - voteDown; - } - return recommended; - } - - public static Integer toHit(Element td) { - return Integer.parseInt(td.text()); - } - - public static Instant toRegisteredAt(Element td) { - final String registeredAtString = td.attr("title"); - return DATE_TIME_FORMATTER.parse(registeredAtString, Instant::from); - } - } diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/constant/WebClientPropertiesUnitName.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/constant/WebClientPropertiesUnitName.java new file mode 100644 index 0000000..3383138 --- /dev/null +++ b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/constant/WebClientPropertiesUnitName.java @@ -0,0 +1,19 @@ +package com.myoa.engineering.crawl.ppomppu.processor.dto.constant; + +import lombok.AllArgsConstructor; +import lombok.Getter; + +/** + * WebClientPropertiesUnitName + * @author Shin Woo-jin (woo-jin.shin@linecorp.com) + * @since 2021-11-18 + * + */ +@Getter +@AllArgsConstructor +public enum WebClientPropertiesUnitName { + PPOMPPU_NOTIFIER_SENDER_API("ppn-sender-api"), + ; + + private String unitName; +} diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/client/MessageSenderAPIClient.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/client/MessageSenderAPIClient.java new file mode 100644 index 0000000..155214f --- /dev/null +++ b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/client/MessageSenderAPIClient.java @@ -0,0 +1,56 @@ +package com.myoa.engineering.crawl.ppomppu.processor.infrastructure.client; + +import org.springframework.core.ParameterizedTypeReference; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; +import org.springframework.stereotype.Component; +import org.springframework.web.reactive.function.client.WebClient; +import org.springframework.web.reactive.function.client.WebClientRequestException; + +import com.myoa.engineering.crawl.ppomppu.processor.dto.constant.WebClientPropertiesUnitName; +import com.myoa.engineering.crawl.ppomppu.support.dto.SimpleMessageDTO; +import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebClientFilterFactory; +import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebFluxExchangeStragiesFactory; +import com.myoa.engineering.crawl.ppomppu.support.webclient.properties.WebClientProperties; +import com.myoa.engineering.crawl.ppomppu.support.webclient.properties.WebClientProperties.WebClientPropertiesUnit; + +import lombok.extern.slf4j.Slf4j; +import reactor.core.publisher.Mono; +import reactor.core.scheduler.Schedulers; + +/** + * PpomppuNotifierSenderAPIClient + * + * @author Shin Woo-jin (woo-jin.shin@linecorp.com) + * @since 2021-11-17 + */ +@Slf4j +@Component +public class MessageSenderAPIClient { + + private final WebClient webClient; + + public MessageSenderAPIClient(WebClientProperties webClientProperties) { + WebClientPropertiesUnit webClientPropertiesUnit = + webClientProperties.find(WebClientPropertiesUnitName.PPOMPPU_NOTIFIER_SENDER_API.getUnitName()); + this.webClient = WebClient.builder() + .baseUrl(webClientPropertiesUnit.getBaseUrl()) + .exchangeStrategies(WebFluxExchangeStragiesFactory.ofDefault()) + .defaultHeader(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_VALUE) +// .filter(WebClientFilterFactory.logRequest()) +// .filter(WebClientFilterFactory.logResponse()) + .build(); + } + + public Mono sendMessageToSlack(SimpleMessageDTO dto) { + return webClient.post() + .uri("/api/v1/messages/sendMessage/messengers/slack") + .bodyValue(dto) + .exchangeToMono(e -> e.bodyToMono(new ParameterizedTypeReference() {})) + .publishOn(Schedulers.boundedElastic()) + .onErrorResume(WebClientRequestException.class, t -> { + log.info("Exception occured, ignoring. : {}", t.getClass().getSimpleName()); + return Mono.empty(); + }); + } +} \ No newline at end of file diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/client/PpomppuBoardFeedRetriever.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/client/PpomppuBoardFeedRetriever.java index a635288..9db4dab 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/client/PpomppuBoardFeedRetriever.java +++ b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/client/PpomppuBoardFeedRetriever.java @@ -1,9 +1,11 @@ package com.myoa.engineering.crawl.ppomppu.processor.infrastructure.client; -import com.myoa.engineering.crawl.ppomppu.processor.configuration.factory.WebClientFilterFactory; -import com.myoa.engineering.crawl.ppomppu.processor.configuration.factory.WebFluxExchangeStragiesFactory; +import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebClientFilterFactory; +import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebFluxExchangeStragiesFactory; import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName; import lombok.extern.slf4j.Slf4j; + +import org.springframework.core.ParameterizedTypeReference; import org.springframework.stereotype.Component; import org.springframework.web.reactive.function.client.WebClient; import org.springframework.web.reactive.function.client.WebClientRequestException; @@ -38,8 +40,8 @@ public class PpomppuBoardFeedRetriever { .onErrorResume(WebClientRequestException.class, t -> { log.info("Exception occured, ignoring. : {}", t.getClass().getSimpleName()); return Mono.empty(); - }) - .doOnNext(e -> log.info("[getHtml] {}", e)); + }); + // .doOnNext(e -> log.info("[getHtml] {}", e)); } } diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/MessageSenderService.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/MessageSenderService.java new file mode 100644 index 0000000..2a6d0ac --- /dev/null +++ b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/MessageSenderService.java @@ -0,0 +1,38 @@ +package com.myoa.engineering.crawl.ppomppu.processor.service; + +import java.util.List; + +import org.springframework.stereotype.Service; + +import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle; +import com.myoa.engineering.crawl.ppomppu.processor.dto.PpomppuArticleTransformer; +import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.client.MessageSenderAPIClient; + +import lombok.extern.slf4j.Slf4j; +import reactor.core.publisher.Mono; + +/** + * MessageSenderService + * @author Shin Woo-jin (woo-jin.shin@linecorp.com) + * @since 2021-11-21 + * + */ +@Slf4j +@Service +public class MessageSenderService { + + private final MessageSenderAPIClient messageSenderAPIClient; + + public MessageSenderService(MessageSenderAPIClient messageSenderAPIClient) { + this.messageSenderAPIClient = messageSenderAPIClient; + } + + public Mono sendMessageToSlack(PpomppuArticle article) { + return messageSenderAPIClient.sendMessageToSlack(PpomppuArticleTransformer.TRANSFORM_TO_MESSAGE_DTO.apply(article)); + } + + public Mono sendMessageToSlack(List articles) { + return messageSenderAPIClient.sendMessageToSlack(PpomppuArticleTransformer.transform(articles)); + } + +} diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/PpomppuArticleService.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/PpomppuArticleService.java index 632e3c3..ad210bb 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/PpomppuArticleService.java +++ b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/PpomppuArticleService.java @@ -1,16 +1,19 @@ package com.myoa.engineering.crawl.ppomppu.processor.service; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle; import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuBoardFeedStatus; import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository.PpomppuArticleRepository; import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository.PpomppuBoardFeedStatusRepository; import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName; -import java.util.List; -import java.util.Optional; -import java.util.stream.Collectors; + import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; @Slf4j @Service @@ -32,13 +35,14 @@ public class PpomppuArticleService { Long latestArticleId = boardFeedStatus.map(PpomppuBoardFeedStatus::getLatestParsedArticleId) .orElse(0L); + log.info("latestArticleId : {}", latestArticleId); return articles.stream() .filter(e -> e.getArticleId().compareTo(latestArticleId) > 0) .collect(Collectors.toList()); } @Transactional - public void save(PpomppuBoardName boardName, List articles) { + public List save(PpomppuBoardName boardName, List articles) { Long latestArticleId = articles.stream() .map(PpomppuArticle::getArticleId) .max(Long::compareTo) @@ -46,14 +50,17 @@ public class PpomppuArticleService { // save PpomppuBoardFeedStatus Optional boardFeedStatus = ppomppuBoardFeedStatusRepository.findByBoardName(boardName); + log.info("boardName: {}, isPresent?: {}", boardName, boardFeedStatus.isPresent()); boardFeedStatus.ifPresentOrElse(e -> { - e.updateArticleId(latestArticleId); - ppomppuBoardFeedStatusRepository.save(e); + if (latestArticleId.longValue() > 0L) { + e.updateArticleId(latestArticleId); + ppomppuBoardFeedStatusRepository.save(e); + } }, () -> ppomppuBoardFeedStatusRepository.save(PpomppuBoardFeedStatus.of(boardName, latestArticleId))); // save real articles. - ppomppuArticleRepository.saveAll(articles); + return ppomppuArticleRepository.saveAll(articles); } } diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/PpomppuFeedService.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/PpomppuFeedService.java index 461bd2f..b8a3568 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/PpomppuFeedService.java +++ b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/PpomppuFeedService.java @@ -1,7 +1,7 @@ package com.myoa.engineering.crawl.ppomppu.processor.service; import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle; -import com.myoa.engineering.crawl.ppomppu.processor.dto.PpomppuArticleTransformer; +import com.myoa.engineering.crawl.ppomppu.processor.dto.PpomppuArticleParser; import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.client.PpomppuBoardFeedRetriever; import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName; import java.util.List; @@ -30,11 +30,11 @@ public class PpomppuFeedService { public Mono> getArticles(PpomppuBoardName boardName) { final Mono html = ppomppuBoardFeedRetriever.getHtml(boardName.getResourcePath()); - final Mono tbody = extractTbodyFromHtml(html) - .doOnNext(e -> log.info("pre tbody - {}", e.html())); + final Mono tbody = extractTbodyFromHtml(html); + // .doOnNext(e -> log.info("pre tbody - {}", e.html())); return extractArticlesFromTbody(tbody).map(this::convertFromElement) .map(e -> e.updateBoardName(boardName)) - .doOnNext(e -> log.info("parsed Result: {}", e)) + // .doOnNext(e -> log.info("parsed Result: {}", e)) .collectList(); } @@ -42,7 +42,7 @@ public class PpomppuFeedService { return html.map(Jsoup::parse) .mapNotNull(e -> e.getElementById("revolution_main_table")) .map(e -> e.getElementsByTag("tbody")) - .doOnNext(e -> log.info("tbody - {}", e.html())) + // .doOnNext(e -> log.info("tbody - {}", e.html())) .map(e -> e.stream() .findFirst() .orElseThrow(() -> new IndexOutOfBoundsException("no tbody"))); @@ -54,6 +54,6 @@ public class PpomppuFeedService { } private PpomppuArticle convertFromElement(Element element) { - return PpomppuArticleTransformer.toArticle(element.getElementsByTag("td")); + return PpomppuArticleParser.toArticle(element.getElementsByTag("td")); } } diff --git a/processor/src/main/resources/application-development.yml b/processor/src/main/resources/application-development.yml index 6d98894..8ceb30d 100644 --- a/processor/src/main/resources/application-development.yml +++ b/processor/src/main/resources/application-development.yml @@ -3,6 +3,11 @@ spring: activate: on-profile: development import: + - "configserver:http://192.168.0.100:11080" - classpath:/development/webclient.yml - - classpath:/development/temppassword.yml - - classpath:/development/database.yml \ No newline at end of file + + +server: + port: 20081 + + # import: optional:configserver:http://localhost:11080 # can be start up even config server was not found. \ No newline at end of file diff --git a/processor/src/main/resources/application-local.yml b/processor/src/main/resources/application-local.yml new file mode 100644 index 0000000..c2198e0 --- /dev/null +++ b/processor/src/main/resources/application-local.yml @@ -0,0 +1,12 @@ +spring: + config: + activate: + on-profile: local + import: + - "configserver:http://localhost:20085" + - classpath:/local/webclient.yml + +server: + port: 20081 + + # import: optional:configserver:http://localhost:11080 # can be start up even config server was not found. \ No newline at end of file diff --git a/processor/src/main/resources/application-production.yml b/processor/src/main/resources/application-production.yml new file mode 100644 index 0000000..aa661d6 --- /dev/null +++ b/processor/src/main/resources/application-production.yml @@ -0,0 +1,6 @@ +spring: + config: + activate: + on-profile: production + import: + - "configserver:http://192.168.0.100:11080" diff --git a/processor/src/main/resources/application.yml b/processor/src/main/resources/application.yml index aa61bfb..d136cf7 100644 --- a/processor/src/main/resources/application.yml +++ b/processor/src/main/resources/application.yml @@ -1,14 +1,25 @@ spring: + application: + name: ppn-processor main: allow-bean-definition-overriding: true profiles: - active: development + active: ${SPRING_ACTIVE_PROFILE:local} + group: + local: "local,datasource-local" + development: "development,datasource-development" + production: "production, datasource-production" freemarker: enabled: false - server: - port: 20081 + port: 20080 error: whitelabel: enabled: false + +management: + endpoints: + web: + exposure: + include: refresh diff --git a/processor/src/main/resources/development/webclient.yml b/processor/src/main/resources/development/webclient.yml index dc5e59f..06ec4a9 100644 --- a/processor/src/main/resources/development/webclient.yml +++ b/processor/src/main/resources/development/webclient.yml @@ -1,5 +1,5 @@ webclient: - some: test + init: true units: - - unit-name: processor-api - base-url: http://localhost:20081 \ No newline at end of file + - unit-name: ppn-sender-api + base-url: http://localhost:20082 \ No newline at end of file diff --git a/processor/src/main/resources/local/webclient.yml b/processor/src/main/resources/local/webclient.yml new file mode 100644 index 0000000..06ec4a9 --- /dev/null +++ b/processor/src/main/resources/local/webclient.yml @@ -0,0 +1,5 @@ +webclient: + init: true + units: + - unit-name: ppn-sender-api + base-url: http://localhost:20082 \ No newline at end of file diff --git a/processor/src/main/resources/logback-spring.xml b/processor/src/main/resources/logback-spring.xml index f0fcb6c..907a4f5 100644 --- a/processor/src/main/resources/logback-spring.xml +++ b/processor/src/main/resources/logback-spring.xml @@ -1,6 +1,10 @@ + + + + diff --git a/processor/src/main/resources/production/webclient.yml b/processor/src/main/resources/production/webclient.yml index 4bb1d1d..373ba18 100644 --- a/processor/src/main/resources/production/webclient.yml +++ b/processor/src/main/resources/production/webclient.yml @@ -1,4 +1,5 @@ webclient: + init: true units: - - unit-name: processor-api - base-url: http://soundhoundfound-processor:20080 \ No newline at end of file + - unit-name: ppn-sender-api + base-url: http://ppn_sender:20082 \ No newline at end of file diff --git a/receiver/build.gradle b/receiver/build.gradle index f56745d..f7771b1 100644 --- a/receiver/build.gradle +++ b/receiver/build.gradle @@ -8,6 +8,7 @@ dependencies { // https://projectreactor.io/docs/core/release/reference/#debug-activate implementation 'org.springframework.boot:spring-boot-starter-webflux' implementation 'org.springframework.boot:spring-boot-configuration-processor' + implementation 'org.springframework.cloud:spring-cloud-starter-config' implementation 'org.telegram:telegrambots:5.3.0' annotationProcessor 'org.springframework.boot:spring-boot-configuration-processor' diff --git a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/ReceiverApplication.java b/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/ReceiverApplication.java index a50c70b..339a01d 100644 --- a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/ReceiverApplication.java +++ b/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/ReceiverApplication.java @@ -3,8 +3,10 @@ package com.myoa.engineering.crawl.ppomppu.receiver; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Import; import com.myoa.engineering.crawl.ppomppu.receiver.configuration.properties.TelegramBotProperties; +import com.myoa.engineering.crawl.ppomppu.support.webclient.PpomppuNotifierWebClientConfiguration; /** * ReceiverApplication @@ -12,8 +14,9 @@ import com.myoa.engineering.crawl.ppomppu.receiver.configuration.properties.Tele * @since 2021-08-20 * */ +@Import({ PpomppuNotifierWebClientConfiguration.class}) @SpringBootApplication -@EnableConfigurationProperties({ TelegramBotProperties.class }) +// @EnableConfigurationProperties({ TelegramBotProperties.class }) public class ReceiverApplication { public static void main(String[] args) { diff --git a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/configuration/TelegramBotConfiguration.java b/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/configuration/TelegramBotConfiguration.java index 31ab08b..dd120dc 100644 --- a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/configuration/TelegramBotConfiguration.java +++ b/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/configuration/TelegramBotConfiguration.java @@ -9,6 +9,7 @@ import org.telegram.telegrambots.meta.exceptions.TelegramApiException; import org.telegram.telegrambots.updatesreceivers.DefaultBotSession; import com.myoa.engineering.crawl.ppomppu.receiver.configuration.properties.TelegramBotProperties; +import com.myoa.engineering.crawl.ppomppu.receiver.configuration.properties.TelegramBotProperties.TelegramBotPropertiesUnit; import com.myoa.engineering.crawl.ppomppu.receiver.dispatch.MessageDispatcher; import com.myoa.engineering.crawl.ppomppu.receiver.handler.message.MessageHandler; @@ -21,6 +22,7 @@ import com.myoa.engineering.crawl.ppomppu.receiver.handler.message.MessageHandle @Configuration public class TelegramBotConfiguration { + private static final String BOT_PROPERTIES_UNIT_NAME = "ppomppu_notify_bot"; @Bean public TelegramBotsApi telegramBotsApi(MessageDispatcher messageDispatcher) throws TelegramApiException { TelegramBotsApi api = new TelegramBotsApi(DefaultBotSession.class); @@ -30,7 +32,8 @@ public class TelegramBotConfiguration { @Bean public MessageDispatcher messageDispatcher(List messageHandlers, - TelegramBotProperties botProperties) { - return new MessageDispatcher(messageHandlers, botProperties.getName(), botProperties.getToken()); + TelegramBotProperties telegramBotProperties) { + TelegramBotPropertiesUnit propertiesUnit = telegramBotProperties.find(BOT_PROPERTIES_UNIT_NAME); + return new MessageDispatcher(messageHandlers, propertiesUnit.getName(), propertiesUnit.getToken()); } } diff --git a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/configuration/properties/TelegramBotProperties.java b/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/configuration/properties/TelegramBotProperties.java index f524798..e438889 100644 --- a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/configuration/properties/TelegramBotProperties.java +++ b/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/configuration/properties/TelegramBotProperties.java @@ -1,9 +1,17 @@ package com.myoa.engineering.crawl.ppomppu.receiver.configuration.properties; +import java.util.ArrayList; +import java.util.List; + import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.boot.context.properties.ConstructorBinding; +import org.springframework.stereotype.Component; +import lombok.AllArgsConstructor; +import lombok.Data; import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; /** * TelegramBotProperties @@ -11,15 +19,26 @@ import lombok.Getter; * @since 2021-09-05 * */ +@Component +@Setter @Getter -@ConstructorBinding -@ConfigurationProperties(prefix = "telegram.bot") +@ConfigurationProperties(prefix = "infra.telegram.bot") public class TelegramBotProperties { - private final String name; - private final String token; - public TelegramBotProperties(final String name, final String token) { - this.name = name; - this.token = token; + private List units = new ArrayList<>(); + + @Data + public static class TelegramBotPropertiesUnit { + private String unitName; + private String name; + private String token; + } + + public TelegramBotPropertiesUnit find(String unitName) { + return units.stream() + .filter(e -> e.getUnitName().equals(unitName)) + .findFirst() + .orElseThrow( + () -> new IllegalArgumentException(this.getClass().getName() + ": unitName Not found. " + unitName)); } } diff --git a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/dto/constant/WebClientPropertiesUnitName.java b/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/dto/constant/WebClientPropertiesUnitName.java new file mode 100644 index 0000000..bdab14d --- /dev/null +++ b/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/dto/constant/WebClientPropertiesUnitName.java @@ -0,0 +1,19 @@ +package com.myoa.engineering.crawl.ppomppu.receiver.dto.constant; + +import lombok.AllArgsConstructor; +import lombok.Getter; + +/** + * WebClientPropertiesUnitName + * @author Shin Woo-jin (woo-jin.shin@linecorp.com) + * @since 2021-11-18 + * + */ +@Getter +@AllArgsConstructor +public enum WebClientPropertiesUnitName { + PPOMPPU_NOTIFIER_PROCESSOR_API("ppn-processor-api"), + ; + + private String unitName; +} diff --git a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/infrastructure/client/ProcessorAPIWebClient.java b/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/infrastructure/client/ProcessorAPIWebClient.java index 87b763e..2dbca2a 100644 --- a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/infrastructure/client/ProcessorAPIWebClient.java +++ b/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/infrastructure/client/ProcessorAPIWebClient.java @@ -1,12 +1,15 @@ package com.myoa.engineering.crawl.ppomppu.receiver.infrastructure.client; -import org.springframework.beans.factory.annotation.Value; import org.springframework.core.ParameterizedTypeReference; import org.springframework.stereotype.Component; import org.springframework.web.reactive.function.client.WebClient; import org.springframework.web.reactive.function.client.WebClientRequestException; +import com.myoa.engineering.crawl.ppomppu.receiver.dto.constant.WebClientPropertiesUnitName; import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName; +import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebClientFilterFactory; +import com.myoa.engineering.crawl.ppomppu.support.webclient.properties.WebClientProperties; +import com.myoa.engineering.crawl.ppomppu.support.webclient.properties.WebClientProperties.WebClientPropertiesUnit; import lombok.extern.slf4j.Slf4j; import reactor.core.publisher.Mono; @@ -18,10 +21,14 @@ public class ProcessorAPIWebClient { private final WebClient webClient; - public ProcessorAPIWebClient(WebClient.Builder webClientBuilder, - @Value("${webclient.base-url}") String baseUrl) { - this.webClient = webClientBuilder.baseUrl(baseUrl) - .build(); + public ProcessorAPIWebClient(WebClientProperties webClientProperties) { + WebClientPropertiesUnit webClientPropertiesUnit = + webClientProperties.find(WebClientPropertiesUnitName.PPOMPPU_NOTIFIER_PROCESSOR_API.getUnitName()); + this.webClient = WebClient.builder() + .baseUrl(webClientPropertiesUnit.getBaseUrl()) + .filter(WebClientFilterFactory.logRequest()) + .filter(WebClientFilterFactory.logResponse()) + .build(); } public Mono emitParseEvent(PpomppuBoardName boardName) { diff --git a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/scheduler/ParseEventEmitter.java b/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/scheduler/ParseEventEmitter.java index 4a0cc9b..bf94c88 100644 --- a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/scheduler/ParseEventEmitter.java +++ b/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/scheduler/ParseEventEmitter.java @@ -26,15 +26,11 @@ public class ParseEventEmitter { this.processorAPIService = processorAPIService; } - @Scheduled(fixedRate = 60 * 1000L) - public void emitDomesticBoard() { - log.info("[emitDomesticBoard] trigger fired!"); - processorAPIService.emitParseEvent(PpomppuBoardName.PPOMPPU_DOMESTIC_ETC).block(); - } - @Scheduled(fixedRate = 300 * 1000L) - public void emitOverseaBoard() { - log.info("[emitOverseaBoard] trigger fired!"); - processorAPIService.emitParseEvent(PpomppuBoardName.PPOMPPU_OVERSEA_ETC).block(); + public void emitBoards() { + log.info("[emitDomesticBoard] trigger fired!"); + for (PpomppuBoardName boardName : PpomppuBoardName.values()) { + processorAPIService.emitParseEvent(boardName).block(); + } } } diff --git a/receiver/src/main/resources/application-development.yml b/receiver/src/main/resources/application-development.yml index 34f30c9..ad67d50 100644 --- a/receiver/src/main/resources/application-development.yml +++ b/receiver/src/main/resources/application-development.yml @@ -4,4 +4,4 @@ spring: on-profile: development import: - classpath:/development/webclient.yml - - classpath:/development/temppassword.yml + - "configserver:http://192.168.0.100:11080" \ No newline at end of file diff --git a/receiver/src/main/resources/application-local.yml b/receiver/src/main/resources/application-local.yml new file mode 100644 index 0000000..297ac57 --- /dev/null +++ b/receiver/src/main/resources/application-local.yml @@ -0,0 +1,7 @@ +spring: + config: + activate: + on-profile: local + import: + - classpath:/local/webclient.yml + - "configserver:http://localhost:20085" \ No newline at end of file diff --git a/receiver/src/main/resources/application-production.yml b/receiver/src/main/resources/application-production.yml index 7c1069c..831bcb9 100644 --- a/receiver/src/main/resources/application-production.yml +++ b/receiver/src/main/resources/application-production.yml @@ -3,4 +3,5 @@ spring: activate: on-profile: production import: - - classpath:/production/webclient.yml \ No newline at end of file + - classpath:/production/webclient.yml + - "configserver:http://192.168.0.100:11080" \ No newline at end of file diff --git a/receiver/src/main/resources/application.yml b/receiver/src/main/resources/application.yml index 782cd5a..a89a769 100644 --- a/receiver/src/main/resources/application.yml +++ b/receiver/src/main/resources/application.yml @@ -1,10 +1,15 @@ spring: + application: + name: ppn-receiver main: allow-bean-definition-overriding: true profiles: - active: development + active: ${SPRING_ACTIVE_PROFILE:local} freemarker: enabled: false server: - port: 20080 \ No newline at end of file + port: 20080 + error: + whitelabel: + enabled: false \ No newline at end of file diff --git a/receiver/src/main/resources/development/webclient.yml b/receiver/src/main/resources/development/webclient.yml index abe8b3b..9fbfe3a 100644 --- a/receiver/src/main/resources/development/webclient.yml +++ b/receiver/src/main/resources/development/webclient.yml @@ -1,5 +1,5 @@ webclient: - base-url: http://localhost:20081 + init: true units: - - unit-name: processor-api + - unit-name: ppn-processor-api base-url: http://localhost:20081 \ No newline at end of file diff --git a/receiver/src/main/resources/local/webclient.yml b/receiver/src/main/resources/local/webclient.yml new file mode 100644 index 0000000..9fbfe3a --- /dev/null +++ b/receiver/src/main/resources/local/webclient.yml @@ -0,0 +1,5 @@ +webclient: + init: true + units: + - unit-name: ppn-processor-api + base-url: http://localhost:20081 \ No newline at end of file diff --git a/receiver/src/main/resources/logback-spring.xml b/receiver/src/main/resources/logback-spring.xml new file mode 100644 index 0000000..907a4f5 --- /dev/null +++ b/receiver/src/main/resources/logback-spring.xml @@ -0,0 +1,15 @@ + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/receiver/src/main/resources/logback/component/logback-file.xml b/receiver/src/main/resources/logback/component/logback-file.xml new file mode 100644 index 0000000..65e54db --- /dev/null +++ b/receiver/src/main/resources/logback/component/logback-file.xml @@ -0,0 +1,23 @@ + + + + + ${DIRECTORY}/${LOG_FILE_BASE}_log + + ${DIRECTORY}/${LOG_FILE_BASE}_log.%d{yyyyMMdd}.%i + + 1000MB + + 60 + + + ${FILE_LOG_PATTERN} + ${IMMEDIATE_FLUSH} + + + + 1024 + + + diff --git a/receiver/src/main/resources/logback/logback-development.xml b/receiver/src/main/resources/logback/logback-development.xml new file mode 100644 index 0000000..458e3d8 --- /dev/null +++ b/receiver/src/main/resources/logback/logback-development.xml @@ -0,0 +1,19 @@ + + + + + + + + + + + + + + + + + + + diff --git a/receiver/src/main/resources/logback/logback-production.xml b/receiver/src/main/resources/logback/logback-production.xml new file mode 100644 index 0000000..f824e41 --- /dev/null +++ b/receiver/src/main/resources/logback/logback-production.xml @@ -0,0 +1,19 @@ + + + + + + + + + + + + + + + + + + + diff --git a/receiver/src/main/resources/production/production.yml b/receiver/src/main/resources/production/production.yml deleted file mode 100644 index 8bc6223..0000000 --- a/receiver/src/main/resources/production/production.yml +++ /dev/null @@ -1,5 +0,0 @@ -webclient: - base-url: http://ppomppu_notifier_processor:20080 - units: - - unit-name: processor-api - base-url: http://ppomppu_notifier_processor:20080 diff --git a/receiver/src/main/resources/production/webclient.yml b/receiver/src/main/resources/production/webclient.yml new file mode 100644 index 0000000..aa3fb1b --- /dev/null +++ b/receiver/src/main/resources/production/webclient.yml @@ -0,0 +1,5 @@ +webclient: + init: true + units: + - unit-name: ppn-processor-api + base-url: http://ppn_processor:20080 diff --git a/sender/build.gradle b/sender/build.gradle new file mode 100644 index 0000000..3181ae3 --- /dev/null +++ b/sender/build.gradle @@ -0,0 +1,23 @@ +dependencies { + developmentOnly 'org.springframework.boot:spring-boot-devtools' + runtimeOnly 'mysql:mysql-connector-java' + compileOnly 'org.projectlombok:lombok' + + implementation project(':support') + // https://projectreactor.io/docs/core/release/reference/#debug-activate + implementation 'org.springframework.boot:spring-boot-starter-webflux' + implementation 'org.springframework.boot:spring-boot-starter-actuator' + implementation 'org.springframework.cloud:spring-cloud-starter-config' + + annotationProcessor 'org.springframework.boot:spring-boot-configuration-processor' + annotationProcessor 'org.projectlombok:lombok' + + testImplementation 'org.springframework.boot:spring-boot-starter-test' +} + +test { + useJUnitPlatform() + testLogging { + events "passed", "skipped", "failed" + } +} \ No newline at end of file diff --git a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/SenderApplication.java b/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/SenderApplication.java index 5f41116..20f8126 100644 --- a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/SenderApplication.java +++ b/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/SenderApplication.java @@ -1,10 +1,22 @@ package com.myoa.engineering.crawl.ppomppu.sender; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.context.annotation.Import; + +import com.myoa.engineering.crawl.ppomppu.support.webclient.PpomppuNotifierWebClientConfiguration; + /** * SenderApplication * @author Shin Woo-jin (woo-jin.shin@linecorp.com) * @since 2021-08-20 * */ +@Import({ PpomppuNotifierWebClientConfiguration.class }) +@SpringBootApplication public class SenderApplication { + + public static void main(String[] args) { + SpringApplication.run(SenderApplication.class, args); + } } diff --git a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/configuration/properties/SlackSecretProperties.java b/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/configuration/properties/SlackSecretProperties.java new file mode 100644 index 0000000..d47983c --- /dev/null +++ b/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/configuration/properties/SlackSecretProperties.java @@ -0,0 +1,34 @@ +package com.myoa.engineering.crawl.ppomppu.sender.configuration.properties; + +import java.util.List; +import lombok.Data; +import lombok.Getter; +import lombok.Setter; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +@Getter +@Setter +@Component +@ConfigurationProperties("infra.slack.bot") +public class SlackSecretProperties { + + private List units; + + @Data + public static class SlackSecretPropertiesUnit { + + private String botName; + private String username; + private String iconEmoji; + private String channel; + private String token; + } + + public SlackSecretPropertiesUnit find(String botUnitName) { + return units.stream() + .filter(e -> e.getBotName().equals(botUnitName)) + .findFirst() + .orElseThrow(() -> new IllegalArgumentException("not found bot unit name : " + botUnitName)); + } +} diff --git a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/controller/MessageSenderAPIController.java b/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/controller/MessageSenderAPIController.java new file mode 100644 index 0000000..479d101 --- /dev/null +++ b/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/controller/MessageSenderAPIController.java @@ -0,0 +1,35 @@ +package com.myoa.engineering.crawl.ppomppu.sender.controller; + +import com.myoa.engineering.crawl.ppomppu.sender.infrastructure.client.MongeShoppingBotSlackMessageSender; +import com.myoa.engineering.crawl.ppomppu.support.dto.APIResponse; +import com.myoa.engineering.crawl.ppomppu.support.dto.SimpleMessageDTO; +import lombok.extern.slf4j.Slf4j; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; +import reactor.core.publisher.Mono; + +/** + * MessageSenderAPIController + * @author Shin Woo-jin (woo-jin.shin@linecorp.com) + * @since 2021-11-21 + * + */ +@Slf4j +@RestController +@RequestMapping("/api/v1") +public class MessageSenderAPIController { + + private final MongeShoppingBotSlackMessageSender sender; + + public MessageSenderAPIController(MongeShoppingBotSlackMessageSender sender) { + this.sender = sender; + } + + @PostMapping("/messages/sendMessage/messengers/slack") + public Mono> sendMessageToSlack(@RequestBody SimpleMessageDTO dto) { + return sender.sendMessage(sender.ofMessage(dto.getBody())) + .then(Mono.just(APIResponse.success(dto))); + } +} diff --git a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/controller/TestAPIController.java b/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/controller/TestAPIController.java new file mode 100644 index 0000000..28192e1 --- /dev/null +++ b/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/controller/TestAPIController.java @@ -0,0 +1,33 @@ +package com.myoa.engineering.crawl.ppomppu.sender.controller; + +import com.myoa.engineering.crawl.ppomppu.sender.infrastructure.client.MongeShoppingBotSlackMessageSender; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import lombok.extern.slf4j.Slf4j; +import reactor.core.publisher.Mono; + +/** + * TestAPIController + * + * @author Shin Woo-jin (woo-jin.shin@linecorp.com) + * @since 2021-11-15 + */ +@Slf4j +@RestController +@RequestMapping("/api/v1") +public class TestAPIController { + + private final MongeShoppingBotSlackMessageSender sender; + + public TestAPIController(MongeShoppingBotSlackMessageSender sender) { + this.sender = sender; + } + + @GetMapping("/test") + public Mono test() { + log.info("received!!!"); + return sender.sendMessage(sender.ofMessage("testtesttest!!!")); + } +} diff --git a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/dto/MessageDTO.java b/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/dto/MessageDTO.java new file mode 100644 index 0000000..db45e43 --- /dev/null +++ b/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/dto/MessageDTO.java @@ -0,0 +1,12 @@ +package com.myoa.engineering.crawl.ppomppu.sender.dto; + +import java.io.Serializable; + +/** + * MessageDTO + * @author Shin Woo-jin (woo-jin.shin@linecorp.com) + * @since 2021-11-14 + * + */ +public interface MessageDTO extends Serializable { +} diff --git a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/dto/SlackMessageDTO.java b/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/dto/SlackMessageDTO.java new file mode 100644 index 0000000..faf7ef5 --- /dev/null +++ b/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/dto/SlackMessageDTO.java @@ -0,0 +1,39 @@ +package com.myoa.engineering.crawl.ppomppu.sender.dto; + +import com.fasterxml.jackson.annotation.JsonProperty; + +import lombok.Builder; +import lombok.Getter; +import lombok.NoArgsConstructor; + +/** + * SlackMessageDTO + * @author Shin Woo-jin (woo-jin.shin@linecorp.com) + * @since 2021-11-14 + * + */ +@Getter +@NoArgsConstructor +public class SlackMessageDTO implements MessageDTO { + + private final static long serialVersionUID = 4737608709660494713L; + + private String text; + private String channel; + private String username; + + @JsonProperty("icon_emoji") + private String iconEmoji; + + @Builder + public SlackMessageDTO(String text, String channel, String username, String iconEmoji) { + this.text = text; + this.channel = channel; + this.username = username; + this.iconEmoji = iconEmoji; + } + + public void applyText(String text) { + this.text = text; + } +} diff --git a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/infrastructure/client/MessageSender.java b/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/infrastructure/client/MessageSender.java new file mode 100644 index 0000000..4153bb4 --- /dev/null +++ b/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/infrastructure/client/MessageSender.java @@ -0,0 +1,17 @@ +package com.myoa.engineering.crawl.ppomppu.sender.infrastructure.client; + +import com.myoa.engineering.crawl.ppomppu.sender.dto.MessageDTO; + +import reactor.core.publisher.Mono; + +/** + * MessageSender + * @author Shin Woo-jin (woo-jin.shin@linecorp.com) + * @since 2021-11-14 + * + */ +public interface MessageSender { + + Mono sendMessage(T message); + +} diff --git a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/infrastructure/client/MongeShoppingBotSlackMessageSender.java b/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/infrastructure/client/MongeShoppingBotSlackMessageSender.java new file mode 100644 index 0000000..6f5b352 --- /dev/null +++ b/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/infrastructure/client/MongeShoppingBotSlackMessageSender.java @@ -0,0 +1,38 @@ +package com.myoa.engineering.crawl.ppomppu.sender.infrastructure.client; + +import com.myoa.engineering.crawl.ppomppu.sender.configuration.properties.SlackSecretProperties; +import com.myoa.engineering.crawl.ppomppu.sender.configuration.properties.SlackSecretProperties.SlackSecretPropertiesUnit; +import com.myoa.engineering.crawl.ppomppu.sender.dto.SlackMessageDTO; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; + +@Slf4j +@Component +public class MongeShoppingBotSlackMessageSender extends SlackMessageSender { + + private static final String SLACK_SECRET_UNIT_NAME = "monge_shopping_bot"; + + private final SlackSecretPropertiesUnit slackProperties; + + public MongeShoppingBotSlackMessageSender(SlackSecretProperties slackSecretProperties) { + super(slackSecretProperties.find(SLACK_SECRET_UNIT_NAME).getToken()); + this.slackProperties = slackSecretProperties.find(SLACK_SECRET_UNIT_NAME); + } + + public SlackMessageDTO ofMessageTemplate() { + return SlackMessageDTO.builder() + .channel(slackProperties.getChannel()) + .iconEmoji(slackProperties.getIconEmoji()) + .username(slackProperties.getUsername()) + .build(); + } + + public SlackMessageDTO ofMessage(String text) { + return SlackMessageDTO.builder() + .channel(slackProperties.getChannel()) + .iconEmoji(slackProperties.getIconEmoji()) + .username(slackProperties.getUsername()) + .text(text) + .build(); + } +} diff --git a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/infrastructure/client/SlackMessageSender.java b/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/infrastructure/client/SlackMessageSender.java new file mode 100644 index 0000000..c0f2efb --- /dev/null +++ b/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/infrastructure/client/SlackMessageSender.java @@ -0,0 +1,53 @@ +package com.myoa.engineering.crawl.ppomppu.sender.infrastructure.client; + +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; +import org.springframework.web.reactive.function.client.WebClient; +import org.springframework.web.reactive.function.client.WebClientRequestException; + +import com.myoa.engineering.crawl.ppomppu.sender.dto.SlackMessageDTO; +import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebClientFilterFactory; +import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebFluxExchangeStragiesFactory; + +import lombok.extern.slf4j.Slf4j; +import reactor.core.publisher.Mono; +import reactor.core.scheduler.Schedulers; + +/** + * SlackMessageSender + * + * @author Shin Woo-jin (woozu.shin@kakaoent.com) + * @since 2021-09-08 + */ +@Slf4j +public class SlackMessageSender implements MessageSender { + + private static final String SLACK_API_URL = "https://slack.com/api"; + + private final WebClient webClient; + + public SlackMessageSender(String apiSecret) { + this.webClient = WebClient.builder() + .baseUrl(SLACK_API_URL) + .defaultHeader(HttpHeaders.AUTHORIZATION, "Bearer " + apiSecret) + .exchangeStrategies(WebFluxExchangeStragiesFactory.ofDefault()) + .defaultHeader(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_UTF8_VALUE) + .defaultHeader(HttpHeaders.ACCEPT_CHARSET, "UTF-8") + .filter(WebClientFilterFactory.logResponse()) + .build(); + } + + @Override + public Mono sendMessage(SlackMessageDTO message) { + return webClient.post() + .uri("/chat.postMessage") + .bodyValue(message) + .exchangeToMono(e -> e.bodyToMono(String.class)) + .publishOn(Schedulers.boundedElastic()) + .onErrorResume(WebClientRequestException.class, t -> { + log.info("Exception occured, ignoring. : {}", t.getClass().getSimpleName()); + return Mono.empty(); + }); + } + +} diff --git a/sender/src/main/resources/application-development.yml b/sender/src/main/resources/application-development.yml new file mode 100644 index 0000000..da50d27 --- /dev/null +++ b/sender/src/main/resources/application-development.yml @@ -0,0 +1,10 @@ +spring: + config: + activate: + on-profile: development + import: + - "configserver:http://192.168.0.100:11080" + # import: optional:configserver:http://localhost:11080 # can be start up even config server was not found. + +server: + port: 20082 diff --git a/sender/src/main/resources/application-local.yml b/sender/src/main/resources/application-local.yml new file mode 100644 index 0000000..5c6bfbb --- /dev/null +++ b/sender/src/main/resources/application-local.yml @@ -0,0 +1,10 @@ +spring: + config: + activate: + on-profile: local + import: + - "configserver:http://localhost:20085" + # import: optional:configserver:http://localhost:11080 # can be start up even config server was not found. + +server: + port: 20082 diff --git a/sender/src/main/resources/application-production.yml b/sender/src/main/resources/application-production.yml new file mode 100644 index 0000000..aa661d6 --- /dev/null +++ b/sender/src/main/resources/application-production.yml @@ -0,0 +1,6 @@ +spring: + config: + activate: + on-profile: production + import: + - "configserver:http://192.168.0.100:11080" diff --git a/sender/src/main/resources/application.yml b/sender/src/main/resources/application.yml index e69de29..4903239 100644 --- a/sender/src/main/resources/application.yml +++ b/sender/src/main/resources/application.yml @@ -0,0 +1,25 @@ +spring: + application: + name: ppn-sender + main: + allow-bean-definition-overriding: true + profiles: + active: ${SPRING_ACTIVE_PROFILE:local} + group: + local: local, slackapi-local + development: development, slackapi-development + production: production, slackapi-production + freemarker: + enabled: false + +server: + port: 20080 + error: + whitelabel: + enabled: false + +management: + endpoints: + web: + exposure: + include: refresh, env diff --git a/sender/src/main/resources/logback-spring.xml b/sender/src/main/resources/logback-spring.xml new file mode 100644 index 0000000..32276f3 --- /dev/null +++ b/sender/src/main/resources/logback-spring.xml @@ -0,0 +1,15 @@ + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/sender/src/main/resources/logback/component/logback-file.xml b/sender/src/main/resources/logback/component/logback-file.xml new file mode 100644 index 0000000..65e54db --- /dev/null +++ b/sender/src/main/resources/logback/component/logback-file.xml @@ -0,0 +1,23 @@ + + + + + ${DIRECTORY}/${LOG_FILE_BASE}_log + + ${DIRECTORY}/${LOG_FILE_BASE}_log.%d{yyyyMMdd}.%i + + 1000MB + + 60 + + + ${FILE_LOG_PATTERN} + ${IMMEDIATE_FLUSH} + + + + 1024 + + + diff --git a/sender/src/main/resources/logback/logback-development.xml b/sender/src/main/resources/logback/logback-development.xml new file mode 100644 index 0000000..458e3d8 --- /dev/null +++ b/sender/src/main/resources/logback/logback-development.xml @@ -0,0 +1,19 @@ + + + + + + + + + + + + + + + + + + + diff --git a/sender/src/main/resources/logback/logback-production.xml b/sender/src/main/resources/logback/logback-production.xml new file mode 100644 index 0000000..f824e41 --- /dev/null +++ b/sender/src/main/resources/logback/logback-production.xml @@ -0,0 +1,19 @@ + + + + + + + + + + + + + + + + + + + diff --git a/support/build.gradle b/support/build.gradle index 4d76cf7..42415c5 100644 --- a/support/build.gradle +++ b/support/build.gradle @@ -1,11 +1,9 @@ dependencies { - runtimeOnly 'mysql:mysql-connector-java' compileOnly 'org.projectlombok:lombok' // https://projectreactor.io/docs/core/release/reference/#debug-activate - - annotationProcessor 'org.springframework.boot:spring-boot-configuration-processor' annotationProcessor 'org.projectlombok:lombok' + implementation 'org.springframework.boot:spring-boot-starter-webflux' } diff --git a/support/next_message_tempalte(block_kit_builder).json b/support/next_message_tempalte(block_kit_builder).json new file mode 100644 index 0000000..eae1772 --- /dev/null +++ b/support/next_message_tempalte(block_kit_builder).json @@ -0,0 +1,31 @@ +{ + "blocks": [ + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "기타)) `[쿠팡] 타푸코 이중 진공 스텐 세라믹 코팅 텀블러 (화이트만 11,130원/로켓와우무료) 4 [기타]`" + }, + "accessory": { + "type": "image", + "image_url": "cdn.ppomppu.co.kr/zboard/data3/2021/1121/m_20211121184835_zfyahnow.png", + "alt_text": "alt text for image" + } + }, + { + "type": "divider" + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "기타)) `[롯데온] 1매 149원 올국산 KF94새부리마스크 대형 200매 (29,920원/무료배송) 58 [기타]`" + }, + "accessory": { + "type": "image", + "image_url": "https://s3-media3.fl.yelpcdn.com/bphoto/c7ed05m9lC2EmA3Aruue7A/o.jpg", + "alt_text": "alt text for image" + } + } + ] +} \ No newline at end of file diff --git a/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/dto/SimpleMessageDTO.java b/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/dto/SimpleMessageDTO.java new file mode 100644 index 0000000..1a2c026 --- /dev/null +++ b/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/dto/SimpleMessageDTO.java @@ -0,0 +1,37 @@ + package com.myoa.engineering.crawl.ppomppu.support.dto; + +import java.io.Serializable; +import java.time.Instant; + +import lombok.Builder; +import lombok.Getter; +import lombok.NoArgsConstructor; + +/** + * SimpleMessageDTO + * @author Shin Woo-jin (woo-jin.shin@linecorp.com) + * @since 2021-11-21 + * + */ +@Getter +@NoArgsConstructor +public class SimpleMessageDTO implements Serializable { + + private static final long serialVersionUID = 2203955567672404428L; + + private String title; + private String body; + private String url; + private Instant publishedAt; + private Instant requestedAt; + + @Builder + public SimpleMessageDTO(String title, String body, String url, Instant publishedAt, Instant requestedAt) { + this.title = title; + this.body = body; + this.url = url; + this.publishedAt = publishedAt; + this.requestedAt = requestedAt; + } + +} diff --git a/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/dto/code/PpomppuBoardName.java b/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/dto/code/PpomppuBoardName.java index 1fad0b9..d66bdc0 100644 --- a/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/dto/code/PpomppuBoardName.java +++ b/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/dto/code/PpomppuBoardName.java @@ -12,40 +12,40 @@ import lombok.Getter; @Getter @AllArgsConstructor public enum PpomppuBoardName { - PPOMPPU_DOMESTIC_ALL("zboard/zboard.php?id=ppomppu", "전체", true), - PPOMPPU_DOMESTIC_ETC("zboard/zboard.php?id=ppomppu&category=1", "기타", true), - PPOMPPU_DOMESTIC_COMPUTER("zboard/zboard.php?id=ppomppu&category=4", "컴퓨터", true), - PPOMPPU_DOMESTIC_DIGITAL("zboard/zboard.php?id=ppomppu&category=5", "디지털", true), - PPOMPPU_DOMESTIC_FOOD("zboard/zboard.php?id=ppomppu&category=6", "식품/건강", true), - PPOMPPU_DOMESTIC_BOOK("zboard/zboard.php?id=ppomppu&category=8", "서적", true), - PPOMPPU_DOMESTIC_APPLIANCES("zboard/zboard.php?id=ppomppu&category=9", "가전/가구", true), - PPOMPPU_DOMESTIC_PARENTING("zboard/zboard.php?id=ppomppu&category=10", "육아", true), - PPOMPPU_DOMESTIC_GIFTCARD("zboard/zboard.php?id=ppomppu&category=11", "상품권", true), - PPOMPPU_DOMESTIC_CLOTHES("zboard/zboard.php?id=ppomppu&category=12", "의류/잡화", true), - PPOMPPU_DOMESTIC_COSMETIC("zboard/zboard.php?id=ppomppu&category=13", "화장품", true), - PPOMPPU_DOMESTIC_OUTDOOR("zboard/zboard.php?id=ppomppu&category=15", "등산/캠핑", true), - PPOMPPU_OVERSEA_ALL("zboard/zboard.php?id=ppomppu4", "전체", true), - PPOMPPU_OVERSEA_ETC("zboard/zboard.php?id=ppomppu4&category=1", "기타", true), - PPOMPPU_OVERSEA_APPLIANCES("zboard/zboard.php?id=ppomppu4&category=7", "가전", true), - PPOMPPU_OVERSEA_TVAV("zboard/zboard.php?id=ppomppu4&category=8", "TV/영상", true), - PPOMPPU_OVERSEA_COMPUTER("zboard/zboard.php?id=ppomppu4&category=3", "컴퓨터", true), - PPOMPPU_OVERSEA_DIGITAL("zboard/zboard.php?id=ppomppu4&category=4", "디지털", true), - PPOMPPU_OVERSEA_MOBILEACCESSORY("zboard/zboard.php?id=ppomppu4&category=9", "액세서리", true), - PPOMPPU_OVERSEA_CLOTHES("zboard/zboard.php?id=ppomppu4&category=5", "의류/잡화", true), - PPOMPPU_OVERSEA_WATCH("zboard/zboard.php?id=ppomppu4&category=2", "시계", true), - PPOMPPU_OVERSEA_SHOES("zboard/zboard.php?id=ppomppu4&category=11", "신발", true), - PPOMPPU_OVERSEA_FOOD("zboard/zboard.php?id=ppomppu4&category=10", "식품/건강", true), - PPOMPPU_OVERSEA_PARENTING("zboard/zboard.php?id=ppomppu4&category=6", "육아", true), + PPOMPPU_DOMESTIC_ALL("/zboard/zboard.php?id=ppomppu", "전체", false), + PPOMPPU_DOMESTIC_ETC("/zboard/zboard.php?id=ppomppu&category=1", "기타", true), + PPOMPPU_DOMESTIC_COMPUTER("/zboard/zboard.php?id=ppomppu&category=4", "컴퓨터", true), + PPOMPPU_DOMESTIC_DIGITAL("/zboard/zboard.php?id=ppomppu&category=5", "디지털", true), + PPOMPPU_DOMESTIC_FOOD("/zboard/zboard.php?id=ppomppu&category=6", "식품/건강", true), + PPOMPPU_DOMESTIC_BOOK("/zboard/zboard.php?id=ppomppu&category=8", "서적", true), + PPOMPPU_DOMESTIC_APPLIANCES("/zboard/zboard.php?id=ppomppu&category=9", "가전/가구", true), + PPOMPPU_DOMESTIC_PARENTING("/zboard/zboard.php?id=ppomppu&category=10", "육아", true), + PPOMPPU_DOMESTIC_GIFTCARD("/zboard/zboard.php?id=ppomppu&category=11", "상품권", true), + PPOMPPU_DOMESTIC_CLOTHES("/zboard/zboard.php?id=ppomppu&category=12", "의류/잡화", true), + PPOMPPU_DOMESTIC_COSMETIC("/zboard/zboard.php?id=ppomppu&category=13", "화장품", true), + PPOMPPU_DOMESTIC_OUTDOOR("/zboard/zboard.php?id=ppomppu&category=15", "등산/캠핑", true), + PPOMPPU_OVERSEA_ALL("/zboard/zboard.php?id=ppomppu4", "전체", false), + PPOMPPU_OVERSEA_ETC("/zboard/zboard.php?id=ppomppu4&category=1", "기타", true), + PPOMPPU_OVERSEA_APPLIANCES("/zboard/zboard.php?id=ppomppu4&category=7", "가전", true), + PPOMPPU_OVERSEA_TVAV("/zboard/zboard.php?id=ppomppu4&category=8", "TV/영상", true), + PPOMPPU_OVERSEA_COMPUTER("/zboard/zboard.php?id=ppomppu4&category=3", "컴퓨터", true), + PPOMPPU_OVERSEA_DIGITAL("/zboard/zboard.php?id=ppomppu4&category=4", "디지털", true), + PPOMPPU_OVERSEA_MOBILEACCESSORY("/zboard/zboard.php?id=ppomppu4&category=9", "액세서리", true), + PPOMPPU_OVERSEA_CLOTHES("/zboard/zboard.php?id=ppomppu4&category=5", "의류/잡화", true), + PPOMPPU_OVERSEA_WATCH("/zboard/zboard.php?id=ppomppu4&category=2", "시계", true), + PPOMPPU_OVERSEA_SHOES("/zboard/zboard.php?id=ppomppu4&category=11", "신발", true), + PPOMPPU_OVERSEA_FOOD("/zboard/zboard.php?id=ppomppu4&category=10", "식품/건강", true), + PPOMPPU_OVERSEA_PARENTING("/zboard/zboard.php?id=ppomppu4&category=6", "육아", true), ; private String resourcePath; private String menuName; private boolean crawlWithDefaultTimer; - public static final String PPOMPPU_URL = "https://www.ppomppu.co.kr/"; + public static final String PPOMPPU_URL = "https://www.ppomppu.co.kr"; public static String ofViewPageUrl(String articleUrl) { - return PPOMPPU_URL + "zboard/" + articleUrl; + return PPOMPPU_URL + "/zboard/" + articleUrl; } } diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/util/ObjectMapperFactory.java b/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/util/ObjectMapperFactory.java similarity index 98% rename from processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/util/ObjectMapperFactory.java rename to support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/util/ObjectMapperFactory.java index 811c76f..fcad823 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/util/ObjectMapperFactory.java +++ b/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/util/ObjectMapperFactory.java @@ -1,4 +1,4 @@ -package com.myoa.engineering.crawl.ppomppu.processor.util; +package com.myoa.engineering.crawl.ppomppu.support.util; import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.core.JsonParser.Feature; diff --git a/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/webclient/PpomppuNotifierWebClientConfiguration.java b/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/webclient/PpomppuNotifierWebClientConfiguration.java new file mode 100644 index 0000000..b907331 --- /dev/null +++ b/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/webclient/PpomppuNotifierWebClientConfiguration.java @@ -0,0 +1,19 @@ +package com.myoa.engineering.crawl.ppomppu.support.webclient; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Configuration; + +/** + * PpomppuNotifierWebClientConfiguration + * @author Shin Woo-jin (woo-jin.shin@linecorp.com) + * @since 2021-11-18 + * + */ +@Configuration +@ConditionalOnProperty(value = "webclient.init", havingValue = "true") +@ComponentScan(basePackageClasses = WebClientBaseScan.class) +public class PpomppuNotifierWebClientConfiguration { + + +} diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/factory/WebClientFilterFactory.java b/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/webclient/factory/WebClientFilterFactory.java similarity index 87% rename from processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/factory/WebClientFilterFactory.java rename to support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/webclient/factory/WebClientFilterFactory.java index dd74293..89d1ec9 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/factory/WebClientFilterFactory.java +++ b/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/webclient/factory/WebClientFilterFactory.java @@ -1,4 +1,4 @@ -package com.myoa.engineering.crawl.ppomppu.processor.configuration.factory; +package com.myoa.engineering.crawl.ppomppu.support.webclient.factory; import lombok.extern.slf4j.Slf4j; import org.springframework.web.reactive.function.client.ClientRequest; @@ -27,8 +27,8 @@ public final class WebClientFilterFactory { private static Mono writeRequest(ClientRequest clientRequest) { try { - log.info("[WEBCLIENT REQUEST] uri : {} method : {} headers : {}", - clientRequest.url(), clientRequest.method(), clientRequest.headers()); + log.info("[WEBCLIENT REQUEST] uri : {} method : {} headers : {}, body: {}", + clientRequest.url(), clientRequest.method(), clientRequest.headers(), clientRequest.body()); } catch (Exception e) { log.error("[WEBCLIENT REQUEST] write request failed", e); } @@ -37,7 +37,7 @@ public final class WebClientFilterFactory { private static Mono writeResponse(ClientResponse clientResponse) { try { - log.info("[WEBCLIENT REQUEST] statusCode : {} headers : {}", + log.info("[WEBCLIENT RESPONSE] statusCode : {} headers : {}", clientResponse.rawStatusCode(), clientResponse.headers().asHttpHeaders()); } catch (Exception e) { log.error("[WEBCLIENT RESPONSE] write response failed", e); diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/factory/WebFluxExchangeStragiesFactory.java b/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/webclient/factory/WebFluxExchangeStragiesFactory.java similarity index 94% rename from processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/factory/WebFluxExchangeStragiesFactory.java rename to support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/webclient/factory/WebFluxExchangeStragiesFactory.java index ebd9c68..6558638 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/factory/WebFluxExchangeStragiesFactory.java +++ b/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/webclient/factory/WebFluxExchangeStragiesFactory.java @@ -1,7 +1,8 @@ -package com.myoa.engineering.crawl.ppomppu.processor.configuration.factory; +package com.myoa.engineering.crawl.ppomppu.support.webclient.factory; import com.fasterxml.jackson.databind.ObjectMapper; -import com.myoa.engineering.crawl.ppomppu.processor.util.ObjectMapperFactory; +import com.myoa.engineering.crawl.ppomppu.support.util.ObjectMapperFactory; + import org.springframework.http.codec.json.Jackson2JsonDecoder; import org.springframework.http.codec.json.Jackson2JsonEncoder; import org.springframework.util.MimeTypeUtils; diff --git a/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/webclient/properties/WebClientProperties.java b/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/webclient/properties/WebClientProperties.java new file mode 100644 index 0000000..64b89f8 --- /dev/null +++ b/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/webclient/properties/WebClientProperties.java @@ -0,0 +1,44 @@ +package com.myoa.engineering.crawl.ppomppu.support.webclient.properties; + +import java.util.ArrayList; +import java.util.List; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.lang.NonNull; +import org.springframework.stereotype.Component; + +import lombok.Data; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +/** + * WebClientPropertiesUnit + * @author Shin Woo-jin (woo-jin.shin@linecorp.com) + * @since 2021-11-18 + * + */ +@NoArgsConstructor +@Setter +@Getter +@Component +@ConfigurationProperties(prefix = "webclient") +public class WebClientProperties { + + private List units = new ArrayList<>(); + + @Data + public static class WebClientPropertiesUnit { + private String unitName; + private String baseUrl; + // TODO headers + } + + public WebClientPropertiesUnit find(@NonNull String unitName) { + return units.stream() + .filter(x -> x.getUnitName().equals(unitName)) + .findFirst() + .orElseThrow( + () -> new IllegalArgumentException("Not found properties unit. unitName : " + unitName)); + } +}