diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/H2ConsoleConfiguration.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/H2ConsoleConfiguration.java index d0d1a9b..993d7f8 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/H2ConsoleConfiguration.java +++ b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/H2ConsoleConfiguration.java @@ -11,7 +11,7 @@ import org.springframework.context.event.ContextRefreshedEvent; import org.springframework.context.event.EventListener; @Slf4j -@Profile("development") +@Profile("datasource-development") @Configuration public class H2ConsoleConfiguration { diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/PpomppuDatasourceConfiguration.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/PpomppuDatasourceConfiguration.java index 64c0a52..b78c475 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/PpomppuDatasourceConfiguration.java +++ b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/PpomppuDatasourceConfiguration.java @@ -32,7 +32,7 @@ import org.springframework.transaction.PlatformTransactionManager; ) public class PpomppuDatasourceConfiguration { - private static final String DATA_SOURCE_UNIT_NAME = "ppomppu"; + private static final String DATA_SOURCE_UNIT_NAME = "ppn_mysql"; private final DatasourceProperties dataSourceProeprties; private final HikariProperties hikariProperties; @@ -52,8 +52,9 @@ public class PpomppuDatasourceConfiguration { final HikariConfig hikariConfig = new HikariConfig(); hikariConfig.setJdbcUrl(dataSourcePropertiesUnit.toCompletedJdbcUrl()); - hikariConfig.setUsername("sa"); - hikariConfig.setPassword("sa"); + hikariConfig.setDriverClassName(dataSourcePropertiesUnit.getDriverClassName()); + hikariConfig.setUsername(dataSourcePropertiesUnit.getUsername()); + hikariConfig.setPassword(dataSourcePropertiesUnit.getPassword()); hikariConfig.setAutoCommit(hikariProperties.getAutoCommit()); hikariConfig.setMaximumPoolSize(hikariProperties.getMaximumPoolSize()); hikariConfig.setMinimumIdle(hikariProperties.getMinimumIdle()); diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/properties/DatasourceProperties.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/properties/DatasourceProperties.java index 34c0ac5..351eb13 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/properties/DatasourceProperties.java +++ b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/properties/DatasourceProperties.java @@ -2,6 +2,7 @@ package com.myoa.engineering.crawl.ppomppu.processor.configuration.properties; import com.myoa.engineering.crawl.ppomppu.support.util.ObjectUtil; import java.util.List; + import lombok.Getter; import lombok.Setter; import org.springframework.boot.context.properties.ConfigurationProperties; @@ -10,7 +11,7 @@ import org.springframework.stereotype.Component; @Component @Setter @Getter -@ConfigurationProperties(prefix = "datasource") +@ConfigurationProperties(prefix = "infra.database") public class DatasourceProperties { private List units; @@ -22,14 +23,17 @@ public class DatasourceProperties { private String unitName; private String schemaName; private String connectionParameters; - private String dbConnectionUrl; - private Boolean simpleConnectionUrl; + private String datasourceUrl; + private Boolean isSimpleConnectionUrl; + private String username; + private String password; + private String driverClassName; public String toCompletedJdbcUrl() { - if (ObjectUtil.isEmpty(simpleConnectionUrl) || simpleConnectionUrl == false) { - return String.format("%s/%s?%s", dbConnectionUrl, schemaName, connectionParameters); + if (ObjectUtil.isEmpty(isSimpleConnectionUrl) || isSimpleConnectionUrl == false) { + return String.format("%s/%s?%s", datasourceUrl, schemaName, connectionParameters); } - return dbConnectionUrl; + return datasourceUrl; } } diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/properties/HibernateProperties.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/properties/HibernateProperties.java index 8de1649..4543d8a 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/properties/HibernateProperties.java +++ b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/properties/HibernateProperties.java @@ -3,6 +3,9 @@ package com.myoa.engineering.crawl.ppomppu.processor.configuration.properties; import java.util.List; import lombok.Getter; import lombok.Setter; + +import org.hibernate.dialect.MySQL8Dialect; +import org.hibernate.dialect.MySQLDialect; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.stereotype.Component; diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/controller/CrawlAPIController.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/controller/CrawlAPIController.java index 3592278..1e72f78 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/controller/CrawlAPIController.java +++ b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/controller/CrawlAPIController.java @@ -1,17 +1,21 @@ package com.myoa.engineering.crawl.ppomppu.processor.controller; -import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle; -import com.myoa.engineering.crawl.ppomppu.processor.dto.FeedParsedResult; -import com.myoa.engineering.crawl.ppomppu.processor.service.PpomppuArticleService; -import com.myoa.engineering.crawl.ppomppu.processor.service.PpomppuFeedService; -import com.myoa.engineering.crawl.ppomppu.support.dto.APIResponse; -import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName; import java.util.List; -import lombok.extern.slf4j.Slf4j; + import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; + +import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle; +import com.myoa.engineering.crawl.ppomppu.processor.dto.FeedParsedResult; +import com.myoa.engineering.crawl.ppomppu.processor.service.MessageSenderService; +import com.myoa.engineering.crawl.ppomppu.processor.service.PpomppuArticleService; +import com.myoa.engineering.crawl.ppomppu.processor.service.PpomppuFeedService; +import com.myoa.engineering.crawl.ppomppu.support.dto.APIResponse; +import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName; + +import lombok.extern.slf4j.Slf4j; import reactor.core.publisher.Mono; /** @@ -27,20 +31,25 @@ public class CrawlAPIController { private final PpomppuFeedService ppomppuRSSFeedService; private final PpomppuArticleService ppomppuArticleService; + private final MessageSenderService messageSenderService; public CrawlAPIController(PpomppuFeedService ppomppuRSSFeedService, - PpomppuArticleService ppomppuArticleService) { + PpomppuArticleService ppomppuArticleService, + MessageSenderService messageSenderService) { this.ppomppuRSSFeedService = ppomppuRSSFeedService; this.ppomppuArticleService = ppomppuArticleService; + this.messageSenderService = messageSenderService; } @PostMapping("/boards/{boardName}") public Mono> crawlBoard(@PathVariable("boardName") PpomppuBoardName boardName) { log.info("got request... {}", boardName); FeedParsedResult result = FeedParsedResult.of(boardName); - Mono> articles = ppomppuRSSFeedService.getArticles(boardName) - .doOnNext(e -> ppomppuArticleService.filterOnlyNewArticles(boardName, e)) - .doOnNext(e -> ppomppuArticleService.save(boardName, e)); + Mono> articles = + ppomppuRSSFeedService.getArticles(boardName) + .doOnNext(e -> ppomppuArticleService.filterOnlyNewArticles(boardName, e)) + .doOnNext(e -> ppomppuArticleService.save(boardName, e)) + .doOnNext(messageSenderService::sendMessageToSlack); return articles.then(Mono.just(APIResponse.success(result.done()))); } diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/PpomppuArticleParser.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/PpomppuArticleParser.java new file mode 100644 index 0000000..20dc7d0 --- /dev/null +++ b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/PpomppuArticleParser.java @@ -0,0 +1,79 @@ +package com.myoa.engineering.crawl.ppomppu.processor.dto; + +import java.time.Instant; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; + +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; + +import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle; +import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName; + +/** + * PpomppuArticleTransformer + * + * @author Shin Woo-jin (woozu.shin@kakaoent.com) + * @since 2021-09-08 + */ +public final class PpomppuArticleParser { + + private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yy.MM.dd HH:mm:ss") + .withZone(ZoneId.of("Asia/Seoul")); + + private PpomppuArticleParser() {} + + public static PpomppuArticle toArticle(Elements articleElement) { + final long articleId = PpomppuArticleParser.parseArticleId(articleElement.get(0)); + final String title = PpomppuArticleParser.parseTitle(articleElement.get(2)); + final String articleUrl = PpomppuArticleParser.parseArticleUrl(articleElement.get(2)); + final int recommended = PpomppuArticleParser.parseRecommended(articleElement.get(6)); + final int hit = PpomppuArticleParser.parseHit(articleElement.get(7)); + final Instant registeredAt = PpomppuArticleParser.parseRegisteredAt(articleElement.get(5)); + + return PpomppuArticle.builder() + .articleId(articleId) + .title(title) + .articleUrl(articleUrl) + .recommended(recommended) + .hit(hit) + .registeredAt(registeredAt) + .build(); + } + + public static Long parseArticleId(Element td) { + return Long.parseLong(td.text().trim()); + } + + public static String parseTitle(Element td) { + return td.text(); + } + + public static String parseArticleUrl(Element td) { + return PpomppuBoardName.ofViewPageUrl(td.getElementsByTag("a").attr("href")); + } + + public static Integer parseRecommended(Element td) { + final String voteString = td.text(); + final int recommended; + + if (voteString.isEmpty()) { + recommended = 0; + } else { + final int voteUp = Integer.parseInt(td.text().split(" - ")[0]); + final int voteDown = Integer.parseInt(td.text().split(" - ")[1]); + recommended = voteUp - voteDown; + } + return recommended; + } + + public static Integer parseHit(Element td) { + return Integer.parseInt(td.text()); + } + + public static Instant parseRegisteredAt(Element td) { + final String registeredAtString = td.attr("title"); + return DATE_TIME_FORMATTER.parse(registeredAtString, Instant::from); + } + +} diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/PpomppuArticleTransformer.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/PpomppuArticleTransformer.java index 18bd30f..95f4e8a 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/PpomppuArticleTransformer.java +++ b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/PpomppuArticleTransformer.java @@ -1,77 +1,50 @@ package com.myoa.engineering.crawl.ppomppu.processor.dto; -import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle; -import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName; import java.time.Instant; import java.time.ZoneId; import java.time.format.DateTimeFormatter; -import org.jsoup.nodes.Element; -import org.jsoup.select.Elements; +import java.util.List; +import java.util.function.Function; +import java.util.stream.Collectors; + +import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle; +import com.myoa.engineering.crawl.ppomppu.support.dto.SimpleMessageDTO; /** * PpomppuArticleTransformer + * @author Shin Woo-jin (woo-jin.shin@linecorp.com) + * @since 2021-11-21 * - * @author Shin Woo-jin (woozu.shin@kakaoent.com) - * @since 2021-09-08 */ public final class PpomppuArticleTransformer { - - private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yy.MM.dd HH:mm:ss") - .withZone(ZoneId.of("Asia/Seoul")); - private PpomppuArticleTransformer() {} - public static PpomppuArticle toArticle(Elements articleElement) { - final long articleId = PpomppuArticleTransformer.toArticleId(articleElement.get(0)); - final String title = PpomppuArticleTransformer.toTitle(articleElement.get(2)); - final String articleUrl = PpomppuArticleTransformer.toArticleUrl(articleElement.get(2)); - final int recommended = PpomppuArticleTransformer.toRecommended(articleElement.get(6)); - final int hit = PpomppuArticleTransformer.toHit(articleElement.get(7)); - final Instant registeredAt = PpomppuArticleTransformer.toRegisteredAt(articleElement.get(5)); + private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss") + .withZone(ZoneId.of("Asia/Seoul")); - return PpomppuArticle.builder() - .articleId(articleId) - .title(title) - .articleUrl(articleUrl) - .recommended(recommended) - .hit(hit) - .registeredAt(registeredAt) - .build(); + public static final Function TRANSFORM_TO_MESSAGE_DTO = entity -> + SimpleMessageDTO.builder() + .requestedAt(Instant.now()) + .publishedAt(entity.getRegisteredAt()) + .title(String.format("[%s] %s", entity.getBoardName().getMenuName(), entity.getTitle())) + .body(entity.getArticleUrl()) + .build(); + + // https://stackoverflow.com/questions/24882927/using-streams-to-convert-a-list-of-objects-into-a-string-obtained-from-the-tostr + public static SimpleMessageDTO transform(List articles) { + Instant requestedAt = Instant.now(); + String body = articles.stream() + .map(PpomppuArticleTransformer::convertToInlineMessage) + .collect(Collectors.joining("\n\n")); + return SimpleMessageDTO.builder() + .requestedAt(requestedAt) + .title(DATE_TIME_FORMATTER.format(requestedAt)) + .body(body) + .build(); } - public static Long toArticleId(Element td) { - return Long.parseLong(td.text().trim()); + public static String convertToInlineMessage(PpomppuArticle article) { + return String.format("[%s] %s\n%s", + article.getBoardName().getMenuName(), article.getTitle(), article.getArticleUrl()); } - - public static String toTitle(Element td) { - return td.text(); - } - - public static String toArticleUrl(Element td) { - return PpomppuBoardName.ofViewPageUrl(td.getElementsByTag("a").attr("href")); - } - - public static Integer toRecommended(Element td) { - final String voteString = td.text(); - final int recommended; - - if (voteString.isEmpty()) { - recommended = 0; - } else { - final int voteUp = Integer.parseInt(td.text().split(" - ")[0]); - final int voteDown = Integer.parseInt(td.text().split(" - ")[1]); - recommended = voteUp - voteDown; - } - return recommended; - } - - public static Integer toHit(Element td) { - return Integer.parseInt(td.text()); - } - - public static Instant toRegisteredAt(Element td) { - final String registeredAtString = td.attr("title"); - return DATE_TIME_FORMATTER.parse(registeredAtString, Instant::from); - } - } diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/client/SenderAPIClient.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/client/MessageSenderAPIClient.java similarity index 57% rename from processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/client/SenderAPIClient.java rename to processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/client/MessageSenderAPIClient.java index e9de2a8..1fd80ae 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/client/SenderAPIClient.java +++ b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/client/MessageSenderAPIClient.java @@ -2,13 +2,17 @@ package com.myoa.engineering.crawl.ppomppu.processor.infrastructure.client; import org.springframework.stereotype.Component; import org.springframework.web.reactive.function.client.WebClient; +import org.springframework.web.reactive.function.client.WebClientRequestException; import com.myoa.engineering.crawl.ppomppu.processor.dto.constant.WebClientPropertiesUnitName; +import com.myoa.engineering.crawl.ppomppu.support.dto.SimpleMessageDTO; import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebClientFilterFactory; import com.myoa.engineering.crawl.ppomppu.support.webclient.properties.WebClientProperties; import com.myoa.engineering.crawl.ppomppu.support.webclient.properties.WebClientProperties.WebClientPropertiesUnit; import lombok.extern.slf4j.Slf4j; +import reactor.core.publisher.Mono; +import reactor.core.scheduler.Schedulers; /** * PpomppuNotifierSenderAPIClient @@ -18,11 +22,11 @@ import lombok.extern.slf4j.Slf4j; */ @Slf4j @Component -public class SenderAPIClient { +public class MessageSenderAPIClient { private final WebClient webClient; - public SenderAPIClient(WebClientProperties webClientProperties) { + public MessageSenderAPIClient(WebClientProperties webClientProperties) { WebClientPropertiesUnit webClientPropertiesUnit = webClientProperties.find(WebClientPropertiesUnitName.PPOMPPU_NOTIFIER_SENDER_API.getUnitName()); this.webClient = WebClient.builder() @@ -31,6 +35,16 @@ public class SenderAPIClient { .filter(WebClientFilterFactory.logResponse()) .build(); } -} -// TODO webclient properties \ No newline at end of file + public Mono sendMessageToSlack(SimpleMessageDTO dto) { + return webClient.post() + .uri("/api/v1/messages/sendMessage/messengers/slack") + .bodyValue(dto) + .exchangeToMono(e -> e.bodyToMono(String.class)) + .publishOn(Schedulers.boundedElastic()) + .onErrorResume(WebClientRequestException.class, t -> { + log.info("Exception occured, ignoring. : {}", t.getClass().getSimpleName()); + return Mono.empty(); + }); + } +} \ No newline at end of file diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/client/PpomppuBoardFeedRetriever.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/client/PpomppuBoardFeedRetriever.java index 9347c63..9db4dab 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/client/PpomppuBoardFeedRetriever.java +++ b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/client/PpomppuBoardFeedRetriever.java @@ -40,8 +40,8 @@ public class PpomppuBoardFeedRetriever { .onErrorResume(WebClientRequestException.class, t -> { log.info("Exception occured, ignoring. : {}", t.getClass().getSimpleName()); return Mono.empty(); - }) - .doOnNext(e -> log.info("[getHtml] {}", e)); + }); + // .doOnNext(e -> log.info("[getHtml] {}", e)); } } diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/MessageSenderService.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/MessageSenderService.java new file mode 100644 index 0000000..60f1d0d --- /dev/null +++ b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/MessageSenderService.java @@ -0,0 +1,37 @@ +package com.myoa.engineering.crawl.ppomppu.processor.service; + +import java.util.List; + +import org.springframework.stereotype.Service; + +import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle; +import com.myoa.engineering.crawl.ppomppu.processor.dto.PpomppuArticleTransformer; +import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.client.MessageSenderAPIClient; + +import lombok.extern.slf4j.Slf4j; + +/** + * MessageSenderService + * @author Shin Woo-jin (woo-jin.shin@linecorp.com) + * @since 2021-11-21 + * + */ +@Slf4j +@Service +public class MessageSenderService { + + private final MessageSenderAPIClient messageSenderAPIClient; + + public MessageSenderService(MessageSenderAPIClient messageSenderAPIClient) { + this.messageSenderAPIClient = messageSenderAPIClient; + } + + public void sendMessageToSlack(PpomppuArticle article) { + messageSenderAPIClient.sendMessageToSlack(PpomppuArticleTransformer.TRANSFORM_TO_MESSAGE_DTO.apply(article)); + } + + public void sendMessageToSlack(List articles) { + messageSenderAPIClient.sendMessageToSlack(PpomppuArticleTransformer.transform(articles)); + } + +} diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/PpomppuFeedService.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/PpomppuFeedService.java index 461bd2f..b8a3568 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/PpomppuFeedService.java +++ b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/PpomppuFeedService.java @@ -1,7 +1,7 @@ package com.myoa.engineering.crawl.ppomppu.processor.service; import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle; -import com.myoa.engineering.crawl.ppomppu.processor.dto.PpomppuArticleTransformer; +import com.myoa.engineering.crawl.ppomppu.processor.dto.PpomppuArticleParser; import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.client.PpomppuBoardFeedRetriever; import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName; import java.util.List; @@ -30,11 +30,11 @@ public class PpomppuFeedService { public Mono> getArticles(PpomppuBoardName boardName) { final Mono html = ppomppuBoardFeedRetriever.getHtml(boardName.getResourcePath()); - final Mono tbody = extractTbodyFromHtml(html) - .doOnNext(e -> log.info("pre tbody - {}", e.html())); + final Mono tbody = extractTbodyFromHtml(html); + // .doOnNext(e -> log.info("pre tbody - {}", e.html())); return extractArticlesFromTbody(tbody).map(this::convertFromElement) .map(e -> e.updateBoardName(boardName)) - .doOnNext(e -> log.info("parsed Result: {}", e)) + // .doOnNext(e -> log.info("parsed Result: {}", e)) .collectList(); } @@ -42,7 +42,7 @@ public class PpomppuFeedService { return html.map(Jsoup::parse) .mapNotNull(e -> e.getElementById("revolution_main_table")) .map(e -> e.getElementsByTag("tbody")) - .doOnNext(e -> log.info("tbody - {}", e.html())) + // .doOnNext(e -> log.info("tbody - {}", e.html())) .map(e -> e.stream() .findFirst() .orElseThrow(() -> new IndexOutOfBoundsException("no tbody"))); @@ -54,6 +54,6 @@ public class PpomppuFeedService { } private PpomppuArticle convertFromElement(Element element) { - return PpomppuArticleTransformer.toArticle(element.getElementsByTag("td")); + return PpomppuArticleParser.toArticle(element.getElementsByTag("td")); } } diff --git a/processor/src/main/resources/application-development.yml b/processor/src/main/resources/application-development.yml index 6341010..8ceb30d 100644 --- a/processor/src/main/resources/application-development.yml +++ b/processor/src/main/resources/application-development.yml @@ -5,9 +5,7 @@ spring: import: - "configserver:http://192.168.0.100:11080" - classpath:/development/webclient.yml - profiles: - group: - development: development, datasource-development + server: port: 20081 diff --git a/processor/src/main/resources/application-production.yml b/processor/src/main/resources/application-production.yml index 892c27d..aa661d6 100644 --- a/processor/src/main/resources/application-production.yml +++ b/processor/src/main/resources/application-production.yml @@ -1,10 +1,6 @@ spring: config: activate: - on-profile: development + on-profile: production import: - "configserver:http://192.168.0.100:11080" - - classpath:/development/webclient.yml - profiles: - group: - development: development, datasource-development diff --git a/processor/src/main/resources/application.yml b/processor/src/main/resources/application.yml index b8b84f6..0cc3fd9 100644 --- a/processor/src/main/resources/application.yml +++ b/processor/src/main/resources/application.yml @@ -5,10 +5,12 @@ spring: allow-bean-definition-overriding: true profiles: active: ${SPRING_ACTIVE_PROFILE:development} + group: + development: "development,datasource-development" + production: "production, datasource-production" freemarker: enabled: false - server: port: 20080 error: diff --git a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/controller/MessageSenderAPIController.java b/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/controller/MessageSenderAPIController.java new file mode 100644 index 0000000..a049fe1 --- /dev/null +++ b/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/controller/MessageSenderAPIController.java @@ -0,0 +1,31 @@ +package com.myoa.engineering.crawl.ppomppu.sender.controller; + +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import com.myoa.engineering.crawl.ppomppu.support.dto.APIResponse; +import com.myoa.engineering.crawl.ppomppu.support.dto.SimpleMessageDTO; + +import lombok.extern.slf4j.Slf4j; +import reactor.core.publisher.Mono; + +/** + * MessageSenderAPIController + * @author Shin Woo-jin (woo-jin.shin@linecorp.com) + * @since 2021-11-21 + * + */ +@Slf4j +@RestController +@RequestMapping("/api/v1") +public class MessageSenderAPIController { + + @PostMapping("/messages/sendMessage/messengers/slack") + public Mono> sendMessageToSlack(SimpleMessageDTO dto) { + + log.info("received : {}, \nbody: {}", dto.getTitle(), dto.getBody()); + // TODO transform + return Mono.just(APIResponse.success(dto)); + } +} diff --git a/sender/src/main/resources/application-development.yml b/sender/src/main/resources/application-development.yml index b821007..da50d27 100644 --- a/sender/src/main/resources/application-development.yml +++ b/sender/src/main/resources/application-development.yml @@ -3,12 +3,8 @@ spring: activate: on-profile: development import: - - classpath:/development/webclient.yml - "configserver:http://192.168.0.100:11080" # import: optional:configserver:http://localhost:11080 # can be start up even config server was not found. - profiles: - group: - development: development, slackapi-development server: - port: 20081 + port: 20082 diff --git a/sender/src/main/resources/application-production.yml b/sender/src/main/resources/application-production.yml index 5028c79..aa661d6 100644 --- a/sender/src/main/resources/application-production.yml +++ b/sender/src/main/resources/application-production.yml @@ -2,5 +2,5 @@ spring: config: activate: on-profile: production -# import: -# - classpath:/production/webclient.yml \ No newline at end of file + import: + - "configserver:http://192.168.0.100:11080" diff --git a/sender/src/main/resources/application.yml b/sender/src/main/resources/application.yml index cb42b9d..ee15561 100644 --- a/sender/src/main/resources/application.yml +++ b/sender/src/main/resources/application.yml @@ -5,9 +5,17 @@ spring: allow-bean-definition-overriding: true profiles: active: ${SPRING_ACTIVE_PROFILE:development} + group: + development: development, slackapi-development + production: production, slackapi-production freemarker: enabled: false + jpa: + hibernate: + ddl-auto: create-drop + generate-ddl: true + server: port: 20080 error: diff --git a/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/dto/SimpleMessageDTO.java b/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/dto/SimpleMessageDTO.java new file mode 100644 index 0000000..1a2c026 --- /dev/null +++ b/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/dto/SimpleMessageDTO.java @@ -0,0 +1,37 @@ + package com.myoa.engineering.crawl.ppomppu.support.dto; + +import java.io.Serializable; +import java.time.Instant; + +import lombok.Builder; +import lombok.Getter; +import lombok.NoArgsConstructor; + +/** + * SimpleMessageDTO + * @author Shin Woo-jin (woo-jin.shin@linecorp.com) + * @since 2021-11-21 + * + */ +@Getter +@NoArgsConstructor +public class SimpleMessageDTO implements Serializable { + + private static final long serialVersionUID = 2203955567672404428L; + + private String title; + private String body; + private String url; + private Instant publishedAt; + private Instant requestedAt; + + @Builder + public SimpleMessageDTO(String title, String body, String url, Instant publishedAt, Instant requestedAt) { + this.title = title; + this.body = body; + this.url = url; + this.publishedAt = publishedAt; + this.requestedAt = requestedAt; + } + +}