[PPN-211113] Implement common sender component #12

Merged
nthfuncx merged 6 commits from feature/PPN-211113 into develop 2021-11-21 13:39:22 +00:00
19 changed files with 296 additions and 108 deletions
Showing only changes of commit 1505227037 - Show all commits

View File

@ -11,7 +11,7 @@ import org.springframework.context.event.ContextRefreshedEvent;
import org.springframework.context.event.EventListener;
@Slf4j
@Profile("development")
@Profile("datasource-development")
@Configuration
public class H2ConsoleConfiguration {

View File

@ -32,7 +32,7 @@ import org.springframework.transaction.PlatformTransactionManager;
)
public class PpomppuDatasourceConfiguration {
private static final String DATA_SOURCE_UNIT_NAME = "ppomppu";
private static final String DATA_SOURCE_UNIT_NAME = "ppn_mysql";
private final DatasourceProperties dataSourceProeprties;
private final HikariProperties hikariProperties;
@ -52,8 +52,9 @@ public class PpomppuDatasourceConfiguration {
final HikariConfig hikariConfig = new HikariConfig();
hikariConfig.setJdbcUrl(dataSourcePropertiesUnit.toCompletedJdbcUrl());
hikariConfig.setUsername("sa");
hikariConfig.setPassword("sa");
hikariConfig.setDriverClassName(dataSourcePropertiesUnit.getDriverClassName());
hikariConfig.setUsername(dataSourcePropertiesUnit.getUsername());
hikariConfig.setPassword(dataSourcePropertiesUnit.getPassword());
hikariConfig.setAutoCommit(hikariProperties.getAutoCommit());
hikariConfig.setMaximumPoolSize(hikariProperties.getMaximumPoolSize());
hikariConfig.setMinimumIdle(hikariProperties.getMinimumIdle());

View File

@ -2,6 +2,7 @@ package com.myoa.engineering.crawl.ppomppu.processor.configuration.properties;
import com.myoa.engineering.crawl.ppomppu.support.util.ObjectUtil;
import java.util.List;
import lombok.Getter;
import lombok.Setter;
import org.springframework.boot.context.properties.ConfigurationProperties;
@ -10,7 +11,7 @@ import org.springframework.stereotype.Component;
@Component
@Setter
@Getter
@ConfigurationProperties(prefix = "datasource")
@ConfigurationProperties(prefix = "infra.database")
public class DatasourceProperties {
private List<DataSourcePropertiesUnit> units;
@ -22,14 +23,17 @@ public class DatasourceProperties {
private String unitName;
private String schemaName;
private String connectionParameters;
private String dbConnectionUrl;
private Boolean simpleConnectionUrl;
private String datasourceUrl;
private Boolean isSimpleConnectionUrl;
private String username;
private String password;
private String driverClassName;
public String toCompletedJdbcUrl() {
if (ObjectUtil.isEmpty(simpleConnectionUrl) || simpleConnectionUrl == false) {
return String.format("%s/%s?%s", dbConnectionUrl, schemaName, connectionParameters);
if (ObjectUtil.isEmpty(isSimpleConnectionUrl) || isSimpleConnectionUrl == false) {
return String.format("%s/%s?%s", datasourceUrl, schemaName, connectionParameters);
}
return dbConnectionUrl;
return datasourceUrl;
}
}

View File

@ -3,6 +3,9 @@ package com.myoa.engineering.crawl.ppomppu.processor.configuration.properties;
import java.util.List;
import lombok.Getter;
import lombok.Setter;
import org.hibernate.dialect.MySQL8Dialect;
import org.hibernate.dialect.MySQLDialect;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;

View File

@ -1,17 +1,21 @@
package com.myoa.engineering.crawl.ppomppu.processor.controller;
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
import com.myoa.engineering.crawl.ppomppu.processor.dto.FeedParsedResult;
import com.myoa.engineering.crawl.ppomppu.processor.service.PpomppuArticleService;
import com.myoa.engineering.crawl.ppomppu.processor.service.PpomppuFeedService;
import com.myoa.engineering.crawl.ppomppu.support.dto.APIResponse;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import java.util.List;
import lombok.extern.slf4j.Slf4j;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
import com.myoa.engineering.crawl.ppomppu.processor.dto.FeedParsedResult;
import com.myoa.engineering.crawl.ppomppu.processor.service.MessageSenderService;
import com.myoa.engineering.crawl.ppomppu.processor.service.PpomppuArticleService;
import com.myoa.engineering.crawl.ppomppu.processor.service.PpomppuFeedService;
import com.myoa.engineering.crawl.ppomppu.support.dto.APIResponse;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import lombok.extern.slf4j.Slf4j;
import reactor.core.publisher.Mono;
/**
@ -27,20 +31,25 @@ public class CrawlAPIController {
private final PpomppuFeedService ppomppuRSSFeedService;
private final PpomppuArticleService ppomppuArticleService;
private final MessageSenderService messageSenderService;
public CrawlAPIController(PpomppuFeedService ppomppuRSSFeedService,
PpomppuArticleService ppomppuArticleService) {
PpomppuArticleService ppomppuArticleService,
MessageSenderService messageSenderService) {
this.ppomppuRSSFeedService = ppomppuRSSFeedService;
this.ppomppuArticleService = ppomppuArticleService;
this.messageSenderService = messageSenderService;
}
@PostMapping("/boards/{boardName}")
public Mono<APIResponse<FeedParsedResult>> crawlBoard(@PathVariable("boardName") PpomppuBoardName boardName) {
log.info("got request... {}", boardName);
FeedParsedResult result = FeedParsedResult.of(boardName);
Mono<List<PpomppuArticle>> articles = ppomppuRSSFeedService.getArticles(boardName)
.doOnNext(e -> ppomppuArticleService.filterOnlyNewArticles(boardName, e))
.doOnNext(e -> ppomppuArticleService.save(boardName, e));
Mono<List<PpomppuArticle>> articles =
ppomppuRSSFeedService.getArticles(boardName)
.doOnNext(e -> ppomppuArticleService.filterOnlyNewArticles(boardName, e))
.doOnNext(e -> ppomppuArticleService.save(boardName, e))
.doOnNext(messageSenderService::sendMessageToSlack);
return articles.then(Mono.just(APIResponse.success(result.done())));
}

View File

@ -0,0 +1,79 @@
package com.myoa.engineering.crawl.ppomppu.processor.dto;
import java.time.Instant;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
/**
* PpomppuArticleTransformer
*
* @author Shin Woo-jin (woozu.shin@kakaoent.com)
* @since 2021-09-08
*/
public final class PpomppuArticleParser {
private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yy.MM.dd HH:mm:ss")
.withZone(ZoneId.of("Asia/Seoul"));
private PpomppuArticleParser() {}
public static PpomppuArticle toArticle(Elements articleElement) {
final long articleId = PpomppuArticleParser.parseArticleId(articleElement.get(0));
final String title = PpomppuArticleParser.parseTitle(articleElement.get(2));
final String articleUrl = PpomppuArticleParser.parseArticleUrl(articleElement.get(2));
final int recommended = PpomppuArticleParser.parseRecommended(articleElement.get(6));
final int hit = PpomppuArticleParser.parseHit(articleElement.get(7));
final Instant registeredAt = PpomppuArticleParser.parseRegisteredAt(articleElement.get(5));
return PpomppuArticle.builder()
.articleId(articleId)
.title(title)
.articleUrl(articleUrl)
.recommended(recommended)
.hit(hit)
.registeredAt(registeredAt)
.build();
}
public static Long parseArticleId(Element td) {
return Long.parseLong(td.text().trim());
}
public static String parseTitle(Element td) {
return td.text();
}
public static String parseArticleUrl(Element td) {
return PpomppuBoardName.ofViewPageUrl(td.getElementsByTag("a").attr("href"));
}
public static Integer parseRecommended(Element td) {
final String voteString = td.text();
final int recommended;
if (voteString.isEmpty()) {
recommended = 0;
} else {
final int voteUp = Integer.parseInt(td.text().split(" - ")[0]);
final int voteDown = Integer.parseInt(td.text().split(" - ")[1]);
recommended = voteUp - voteDown;
}
return recommended;
}
public static Integer parseHit(Element td) {
return Integer.parseInt(td.text());
}
public static Instant parseRegisteredAt(Element td) {
final String registeredAtString = td.attr("title");
return DATE_TIME_FORMATTER.parse(registeredAtString, Instant::from);
}
}

View File

@ -1,77 +1,50 @@
package com.myoa.engineering.crawl.ppomppu.processor.dto;
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import java.time.Instant;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import java.util.List;
import java.util.function.Function;
import java.util.stream.Collectors;
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
import com.myoa.engineering.crawl.ppomppu.support.dto.SimpleMessageDTO;
/**
* PpomppuArticleTransformer
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-11-21
*
* @author Shin Woo-jin (woozu.shin@kakaoent.com)
* @since 2021-09-08
*/
public final class PpomppuArticleTransformer {
private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yy.MM.dd HH:mm:ss")
.withZone(ZoneId.of("Asia/Seoul"));
private PpomppuArticleTransformer() {}
public static PpomppuArticle toArticle(Elements articleElement) {
final long articleId = PpomppuArticleTransformer.toArticleId(articleElement.get(0));
final String title = PpomppuArticleTransformer.toTitle(articleElement.get(2));
final String articleUrl = PpomppuArticleTransformer.toArticleUrl(articleElement.get(2));
final int recommended = PpomppuArticleTransformer.toRecommended(articleElement.get(6));
final int hit = PpomppuArticleTransformer.toHit(articleElement.get(7));
final Instant registeredAt = PpomppuArticleTransformer.toRegisteredAt(articleElement.get(5));
private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")
.withZone(ZoneId.of("Asia/Seoul"));
return PpomppuArticle.builder()
.articleId(articleId)
.title(title)
.articleUrl(articleUrl)
.recommended(recommended)
.hit(hit)
.registeredAt(registeredAt)
.build();
public static final Function<PpomppuArticle, SimpleMessageDTO> TRANSFORM_TO_MESSAGE_DTO = entity ->
SimpleMessageDTO.builder()
.requestedAt(Instant.now())
.publishedAt(entity.getRegisteredAt())
.title(String.format("[%s] %s", entity.getBoardName().getMenuName(), entity.getTitle()))
.body(entity.getArticleUrl())
.build();
// https://stackoverflow.com/questions/24882927/using-streams-to-convert-a-list-of-objects-into-a-string-obtained-from-the-tostr
public static SimpleMessageDTO transform(List<PpomppuArticle> articles) {
Instant requestedAt = Instant.now();
String body = articles.stream()
.map(PpomppuArticleTransformer::convertToInlineMessage)
.collect(Collectors.joining("\n\n"));
return SimpleMessageDTO.builder()
.requestedAt(requestedAt)
.title(DATE_TIME_FORMATTER.format(requestedAt))
.body(body)
.build();
}
public static Long toArticleId(Element td) {
return Long.parseLong(td.text().trim());
public static String convertToInlineMessage(PpomppuArticle article) {
return String.format("[%s] %s\n%s",
article.getBoardName().getMenuName(), article.getTitle(), article.getArticleUrl());
}
public static String toTitle(Element td) {
return td.text();
}
public static String toArticleUrl(Element td) {
return PpomppuBoardName.ofViewPageUrl(td.getElementsByTag("a").attr("href"));
}
public static Integer toRecommended(Element td) {
final String voteString = td.text();
final int recommended;
if (voteString.isEmpty()) {
recommended = 0;
} else {
final int voteUp = Integer.parseInt(td.text().split(" - ")[0]);
final int voteDown = Integer.parseInt(td.text().split(" - ")[1]);
recommended = voteUp - voteDown;
}
return recommended;
}
public static Integer toHit(Element td) {
return Integer.parseInt(td.text());
}
public static Instant toRegisteredAt(Element td) {
final String registeredAtString = td.attr("title");
return DATE_TIME_FORMATTER.parse(registeredAtString, Instant::from);
}
}

View File

@ -2,13 +2,17 @@ package com.myoa.engineering.crawl.ppomppu.processor.infrastructure.client;
import org.springframework.stereotype.Component;
import org.springframework.web.reactive.function.client.WebClient;
import org.springframework.web.reactive.function.client.WebClientRequestException;
import com.myoa.engineering.crawl.ppomppu.processor.dto.constant.WebClientPropertiesUnitName;
import com.myoa.engineering.crawl.ppomppu.support.dto.SimpleMessageDTO;
import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebClientFilterFactory;
import com.myoa.engineering.crawl.ppomppu.support.webclient.properties.WebClientProperties;
import com.myoa.engineering.crawl.ppomppu.support.webclient.properties.WebClientProperties.WebClientPropertiesUnit;
import lombok.extern.slf4j.Slf4j;
import reactor.core.publisher.Mono;
import reactor.core.scheduler.Schedulers;
/**
* PpomppuNotifierSenderAPIClient
@ -18,11 +22,11 @@ import lombok.extern.slf4j.Slf4j;
*/
@Slf4j
@Component
public class SenderAPIClient {
public class MessageSenderAPIClient {
private final WebClient webClient;
public SenderAPIClient(WebClientProperties webClientProperties) {
public MessageSenderAPIClient(WebClientProperties webClientProperties) {
WebClientPropertiesUnit webClientPropertiesUnit =
webClientProperties.find(WebClientPropertiesUnitName.PPOMPPU_NOTIFIER_SENDER_API.getUnitName());
this.webClient = WebClient.builder()
@ -31,6 +35,16 @@ public class SenderAPIClient {
.filter(WebClientFilterFactory.logResponse())
.build();
}
}
// TODO webclient properties
public Mono<String> sendMessageToSlack(SimpleMessageDTO dto) {
return webClient.post()
.uri("/api/v1/messages/sendMessage/messengers/slack")
.bodyValue(dto)
.exchangeToMono(e -> e.bodyToMono(String.class))
.publishOn(Schedulers.boundedElastic())
.onErrorResume(WebClientRequestException.class, t -> {
log.info("Exception occured, ignoring. : {}", t.getClass().getSimpleName());
return Mono.empty();
});
}
}

View File

@ -40,8 +40,8 @@ public class PpomppuBoardFeedRetriever {
.onErrorResume(WebClientRequestException.class, t -> {
log.info("Exception occured, ignoring. : {}", t.getClass().getSimpleName());
return Mono.empty();
})
.doOnNext(e -> log.info("[getHtml] {}", e));
});
// .doOnNext(e -> log.info("[getHtml] {}", e));
}
}

View File

@ -0,0 +1,37 @@
package com.myoa.engineering.crawl.ppomppu.processor.service;
import java.util.List;
import org.springframework.stereotype.Service;
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
import com.myoa.engineering.crawl.ppomppu.processor.dto.PpomppuArticleTransformer;
import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.client.MessageSenderAPIClient;
import lombok.extern.slf4j.Slf4j;
/**
* MessageSenderService
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-11-21
*
*/
@Slf4j
@Service
public class MessageSenderService {
private final MessageSenderAPIClient messageSenderAPIClient;
public MessageSenderService(MessageSenderAPIClient messageSenderAPIClient) {
this.messageSenderAPIClient = messageSenderAPIClient;
}
public void sendMessageToSlack(PpomppuArticle article) {
messageSenderAPIClient.sendMessageToSlack(PpomppuArticleTransformer.TRANSFORM_TO_MESSAGE_DTO.apply(article));
}
public void sendMessageToSlack(List<PpomppuArticle> articles) {
messageSenderAPIClient.sendMessageToSlack(PpomppuArticleTransformer.transform(articles));
}
}

View File

@ -1,7 +1,7 @@
package com.myoa.engineering.crawl.ppomppu.processor.service;
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
import com.myoa.engineering.crawl.ppomppu.processor.dto.PpomppuArticleTransformer;
import com.myoa.engineering.crawl.ppomppu.processor.dto.PpomppuArticleParser;
import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.client.PpomppuBoardFeedRetriever;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import java.util.List;
@ -30,11 +30,11 @@ public class PpomppuFeedService {
public Mono<List<PpomppuArticle>> getArticles(PpomppuBoardName boardName) {
final Mono<String> html = ppomppuBoardFeedRetriever.getHtml(boardName.getResourcePath());
final Mono<Element> tbody = extractTbodyFromHtml(html)
.doOnNext(e -> log.info("pre tbody - {}", e.html()));
final Mono<Element> tbody = extractTbodyFromHtml(html);
// .doOnNext(e -> log.info("pre tbody - {}", e.html()));
return extractArticlesFromTbody(tbody).map(this::convertFromElement)
.map(e -> e.updateBoardName(boardName))
.doOnNext(e -> log.info("parsed Result: {}", e))
// .doOnNext(e -> log.info("parsed Result: {}", e))
.collectList();
}
@ -42,7 +42,7 @@ public class PpomppuFeedService {
return html.map(Jsoup::parse)
.mapNotNull(e -> e.getElementById("revolution_main_table"))
.map(e -> e.getElementsByTag("tbody"))
.doOnNext(e -> log.info("tbody - {}", e.html()))
// .doOnNext(e -> log.info("tbody - {}", e.html()))
.map(e -> e.stream()
.findFirst()
.orElseThrow(() -> new IndexOutOfBoundsException("no tbody")));
@ -54,6 +54,6 @@ public class PpomppuFeedService {
}
private PpomppuArticle convertFromElement(Element element) {
return PpomppuArticleTransformer.toArticle(element.getElementsByTag("td"));
return PpomppuArticleParser.toArticle(element.getElementsByTag("td"));
}
}

View File

@ -5,9 +5,7 @@ spring:
import:
- "configserver:http://192.168.0.100:11080"
- classpath:/development/webclient.yml
profiles:
group:
development: development, datasource-development
server:
port: 20081

View File

@ -1,10 +1,6 @@
spring:
config:
activate:
on-profile: development
on-profile: production
import:
- "configserver:http://192.168.0.100:11080"
- classpath:/development/webclient.yml
profiles:
group:
development: development, datasource-development

View File

@ -5,10 +5,12 @@ spring:
allow-bean-definition-overriding: true
profiles:
active: ${SPRING_ACTIVE_PROFILE:development}
group:
development: "development,datasource-development"
production: "production, datasource-production"
freemarker:
enabled: false
server:
port: 20080
error:

View File

@ -0,0 +1,31 @@
package com.myoa.engineering.crawl.ppomppu.sender.controller;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import com.myoa.engineering.crawl.ppomppu.support.dto.APIResponse;
import com.myoa.engineering.crawl.ppomppu.support.dto.SimpleMessageDTO;
import lombok.extern.slf4j.Slf4j;
import reactor.core.publisher.Mono;
/**
* MessageSenderAPIController
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-11-21
*
*/
@Slf4j
@RestController
@RequestMapping("/api/v1")
public class MessageSenderAPIController {
@PostMapping("/messages/sendMessage/messengers/slack")
public Mono<APIResponse<SimpleMessageDTO>> sendMessageToSlack(SimpleMessageDTO dto) {
log.info("received : {}, \nbody: {}", dto.getTitle(), dto.getBody());
// TODO transform
return Mono.just(APIResponse.success(dto));
}
}

View File

@ -3,12 +3,8 @@ spring:
activate:
on-profile: development
import:
- classpath:/development/webclient.yml
- "configserver:http://192.168.0.100:11080"
# import: optional:configserver:http://localhost:11080 # can be start up even config server was not found.
profiles:
group:
development: development, slackapi-development
server:
port: 20081
port: 20082

View File

@ -2,5 +2,5 @@ spring:
config:
activate:
on-profile: production
# import:
# - classpath:/production/webclient.yml
import:
- "configserver:http://192.168.0.100:11080"

View File

@ -5,9 +5,17 @@ spring:
allow-bean-definition-overriding: true
profiles:
active: ${SPRING_ACTIVE_PROFILE:development}
group:
development: development, slackapi-development
production: production, slackapi-production
freemarker:
enabled: false
jpa:
hibernate:
ddl-auto: create-drop
generate-ddl: true
server:
port: 20080
error:

View File

@ -0,0 +1,37 @@
package com.myoa.engineering.crawl.ppomppu.support.dto;
import java.io.Serializable;
import java.time.Instant;
import lombok.Builder;
import lombok.Getter;
import lombok.NoArgsConstructor;
/**
* SimpleMessageDTO
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-11-21
*
*/
@Getter
@NoArgsConstructor
public class SimpleMessageDTO implements Serializable {
private static final long serialVersionUID = 2203955567672404428L;
private String title;
private String body;
private String url;
private Instant publishedAt;
private Instant requestedAt;
@Builder
public SimpleMessageDTO(String title, String body, String url, Instant publishedAt, Instant requestedAt) {
this.title = title;
this.body = body;
this.url = url;
this.publishedAt = publishedAt;
this.requestedAt = requestedAt;
}
}