[PPN-211113] Modify application properties
This commit is contained in:
parent
bbf4affc16
commit
1505227037
|
@ -11,7 +11,7 @@ import org.springframework.context.event.ContextRefreshedEvent;
|
||||||
import org.springframework.context.event.EventListener;
|
import org.springframework.context.event.EventListener;
|
||||||
|
|
||||||
@Slf4j
|
@Slf4j
|
||||||
@Profile("development")
|
@Profile("datasource-development")
|
||||||
@Configuration
|
@Configuration
|
||||||
public class H2ConsoleConfiguration {
|
public class H2ConsoleConfiguration {
|
||||||
|
|
||||||
|
|
|
@ -32,7 +32,7 @@ import org.springframework.transaction.PlatformTransactionManager;
|
||||||
)
|
)
|
||||||
public class PpomppuDatasourceConfiguration {
|
public class PpomppuDatasourceConfiguration {
|
||||||
|
|
||||||
private static final String DATA_SOURCE_UNIT_NAME = "ppomppu";
|
private static final String DATA_SOURCE_UNIT_NAME = "ppn_mysql";
|
||||||
|
|
||||||
private final DatasourceProperties dataSourceProeprties;
|
private final DatasourceProperties dataSourceProeprties;
|
||||||
private final HikariProperties hikariProperties;
|
private final HikariProperties hikariProperties;
|
||||||
|
@ -52,8 +52,9 @@ public class PpomppuDatasourceConfiguration {
|
||||||
|
|
||||||
final HikariConfig hikariConfig = new HikariConfig();
|
final HikariConfig hikariConfig = new HikariConfig();
|
||||||
hikariConfig.setJdbcUrl(dataSourcePropertiesUnit.toCompletedJdbcUrl());
|
hikariConfig.setJdbcUrl(dataSourcePropertiesUnit.toCompletedJdbcUrl());
|
||||||
hikariConfig.setUsername("sa");
|
hikariConfig.setDriverClassName(dataSourcePropertiesUnit.getDriverClassName());
|
||||||
hikariConfig.setPassword("sa");
|
hikariConfig.setUsername(dataSourcePropertiesUnit.getUsername());
|
||||||
|
hikariConfig.setPassword(dataSourcePropertiesUnit.getPassword());
|
||||||
hikariConfig.setAutoCommit(hikariProperties.getAutoCommit());
|
hikariConfig.setAutoCommit(hikariProperties.getAutoCommit());
|
||||||
hikariConfig.setMaximumPoolSize(hikariProperties.getMaximumPoolSize());
|
hikariConfig.setMaximumPoolSize(hikariProperties.getMaximumPoolSize());
|
||||||
hikariConfig.setMinimumIdle(hikariProperties.getMinimumIdle());
|
hikariConfig.setMinimumIdle(hikariProperties.getMinimumIdle());
|
||||||
|
|
|
@ -2,6 +2,7 @@ package com.myoa.engineering.crawl.ppomppu.processor.configuration.properties;
|
||||||
|
|
||||||
import com.myoa.engineering.crawl.ppomppu.support.util.ObjectUtil;
|
import com.myoa.engineering.crawl.ppomppu.support.util.ObjectUtil;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
import lombok.Getter;
|
import lombok.Getter;
|
||||||
import lombok.Setter;
|
import lombok.Setter;
|
||||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||||
|
@ -10,7 +11,7 @@ import org.springframework.stereotype.Component;
|
||||||
@Component
|
@Component
|
||||||
@Setter
|
@Setter
|
||||||
@Getter
|
@Getter
|
||||||
@ConfigurationProperties(prefix = "datasource")
|
@ConfigurationProperties(prefix = "infra.database")
|
||||||
public class DatasourceProperties {
|
public class DatasourceProperties {
|
||||||
|
|
||||||
private List<DataSourcePropertiesUnit> units;
|
private List<DataSourcePropertiesUnit> units;
|
||||||
|
@ -22,14 +23,17 @@ public class DatasourceProperties {
|
||||||
private String unitName;
|
private String unitName;
|
||||||
private String schemaName;
|
private String schemaName;
|
||||||
private String connectionParameters;
|
private String connectionParameters;
|
||||||
private String dbConnectionUrl;
|
private String datasourceUrl;
|
||||||
private Boolean simpleConnectionUrl;
|
private Boolean isSimpleConnectionUrl;
|
||||||
|
private String username;
|
||||||
|
private String password;
|
||||||
|
private String driverClassName;
|
||||||
|
|
||||||
public String toCompletedJdbcUrl() {
|
public String toCompletedJdbcUrl() {
|
||||||
if (ObjectUtil.isEmpty(simpleConnectionUrl) || simpleConnectionUrl == false) {
|
if (ObjectUtil.isEmpty(isSimpleConnectionUrl) || isSimpleConnectionUrl == false) {
|
||||||
return String.format("%s/%s?%s", dbConnectionUrl, schemaName, connectionParameters);
|
return String.format("%s/%s?%s", datasourceUrl, schemaName, connectionParameters);
|
||||||
}
|
}
|
||||||
return dbConnectionUrl;
|
return datasourceUrl;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -3,6 +3,9 @@ package com.myoa.engineering.crawl.ppomppu.processor.configuration.properties;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import lombok.Getter;
|
import lombok.Getter;
|
||||||
import lombok.Setter;
|
import lombok.Setter;
|
||||||
|
|
||||||
|
import org.hibernate.dialect.MySQL8Dialect;
|
||||||
|
import org.hibernate.dialect.MySQLDialect;
|
||||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||||
import org.springframework.stereotype.Component;
|
import org.springframework.stereotype.Component;
|
||||||
|
|
||||||
|
|
|
@ -1,17 +1,21 @@
|
||||||
package com.myoa.engineering.crawl.ppomppu.processor.controller;
|
package com.myoa.engineering.crawl.ppomppu.processor.controller;
|
||||||
|
|
||||||
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
|
|
||||||
import com.myoa.engineering.crawl.ppomppu.processor.dto.FeedParsedResult;
|
|
||||||
import com.myoa.engineering.crawl.ppomppu.processor.service.PpomppuArticleService;
|
|
||||||
import com.myoa.engineering.crawl.ppomppu.processor.service.PpomppuFeedService;
|
|
||||||
import com.myoa.engineering.crawl.ppomppu.support.dto.APIResponse;
|
|
||||||
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
|
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
|
||||||
import org.springframework.web.bind.annotation.PathVariable;
|
import org.springframework.web.bind.annotation.PathVariable;
|
||||||
import org.springframework.web.bind.annotation.PostMapping;
|
import org.springframework.web.bind.annotation.PostMapping;
|
||||||
import org.springframework.web.bind.annotation.RequestMapping;
|
import org.springframework.web.bind.annotation.RequestMapping;
|
||||||
import org.springframework.web.bind.annotation.RestController;
|
import org.springframework.web.bind.annotation.RestController;
|
||||||
|
|
||||||
|
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
|
||||||
|
import com.myoa.engineering.crawl.ppomppu.processor.dto.FeedParsedResult;
|
||||||
|
import com.myoa.engineering.crawl.ppomppu.processor.service.MessageSenderService;
|
||||||
|
import com.myoa.engineering.crawl.ppomppu.processor.service.PpomppuArticleService;
|
||||||
|
import com.myoa.engineering.crawl.ppomppu.processor.service.PpomppuFeedService;
|
||||||
|
import com.myoa.engineering.crawl.ppomppu.support.dto.APIResponse;
|
||||||
|
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
|
||||||
|
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import reactor.core.publisher.Mono;
|
import reactor.core.publisher.Mono;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -27,20 +31,25 @@ public class CrawlAPIController {
|
||||||
|
|
||||||
private final PpomppuFeedService ppomppuRSSFeedService;
|
private final PpomppuFeedService ppomppuRSSFeedService;
|
||||||
private final PpomppuArticleService ppomppuArticleService;
|
private final PpomppuArticleService ppomppuArticleService;
|
||||||
|
private final MessageSenderService messageSenderService;
|
||||||
|
|
||||||
public CrawlAPIController(PpomppuFeedService ppomppuRSSFeedService,
|
public CrawlAPIController(PpomppuFeedService ppomppuRSSFeedService,
|
||||||
PpomppuArticleService ppomppuArticleService) {
|
PpomppuArticleService ppomppuArticleService,
|
||||||
|
MessageSenderService messageSenderService) {
|
||||||
this.ppomppuRSSFeedService = ppomppuRSSFeedService;
|
this.ppomppuRSSFeedService = ppomppuRSSFeedService;
|
||||||
this.ppomppuArticleService = ppomppuArticleService;
|
this.ppomppuArticleService = ppomppuArticleService;
|
||||||
|
this.messageSenderService = messageSenderService;
|
||||||
}
|
}
|
||||||
|
|
||||||
@PostMapping("/boards/{boardName}")
|
@PostMapping("/boards/{boardName}")
|
||||||
public Mono<APIResponse<FeedParsedResult>> crawlBoard(@PathVariable("boardName") PpomppuBoardName boardName) {
|
public Mono<APIResponse<FeedParsedResult>> crawlBoard(@PathVariable("boardName") PpomppuBoardName boardName) {
|
||||||
log.info("got request... {}", boardName);
|
log.info("got request... {}", boardName);
|
||||||
FeedParsedResult result = FeedParsedResult.of(boardName);
|
FeedParsedResult result = FeedParsedResult.of(boardName);
|
||||||
Mono<List<PpomppuArticle>> articles = ppomppuRSSFeedService.getArticles(boardName)
|
Mono<List<PpomppuArticle>> articles =
|
||||||
|
ppomppuRSSFeedService.getArticles(boardName)
|
||||||
.doOnNext(e -> ppomppuArticleService.filterOnlyNewArticles(boardName, e))
|
.doOnNext(e -> ppomppuArticleService.filterOnlyNewArticles(boardName, e))
|
||||||
.doOnNext(e -> ppomppuArticleService.save(boardName, e));
|
.doOnNext(e -> ppomppuArticleService.save(boardName, e))
|
||||||
|
.doOnNext(messageSenderService::sendMessageToSlack);
|
||||||
|
|
||||||
return articles.then(Mono.just(APIResponse.success(result.done())));
|
return articles.then(Mono.just(APIResponse.success(result.done())));
|
||||||
}
|
}
|
||||||
|
|
|
@ -0,0 +1,79 @@
|
||||||
|
package com.myoa.engineering.crawl.ppomppu.processor.dto;
|
||||||
|
|
||||||
|
import java.time.Instant;
|
||||||
|
import java.time.ZoneId;
|
||||||
|
import java.time.format.DateTimeFormatter;
|
||||||
|
|
||||||
|
import org.jsoup.nodes.Element;
|
||||||
|
import org.jsoup.select.Elements;
|
||||||
|
|
||||||
|
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
|
||||||
|
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* PpomppuArticleTransformer
|
||||||
|
*
|
||||||
|
* @author Shin Woo-jin (woozu.shin@kakaoent.com)
|
||||||
|
* @since 2021-09-08
|
||||||
|
*/
|
||||||
|
public final class PpomppuArticleParser {
|
||||||
|
|
||||||
|
private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yy.MM.dd HH:mm:ss")
|
||||||
|
.withZone(ZoneId.of("Asia/Seoul"));
|
||||||
|
|
||||||
|
private PpomppuArticleParser() {}
|
||||||
|
|
||||||
|
public static PpomppuArticle toArticle(Elements articleElement) {
|
||||||
|
final long articleId = PpomppuArticleParser.parseArticleId(articleElement.get(0));
|
||||||
|
final String title = PpomppuArticleParser.parseTitle(articleElement.get(2));
|
||||||
|
final String articleUrl = PpomppuArticleParser.parseArticleUrl(articleElement.get(2));
|
||||||
|
final int recommended = PpomppuArticleParser.parseRecommended(articleElement.get(6));
|
||||||
|
final int hit = PpomppuArticleParser.parseHit(articleElement.get(7));
|
||||||
|
final Instant registeredAt = PpomppuArticleParser.parseRegisteredAt(articleElement.get(5));
|
||||||
|
|
||||||
|
return PpomppuArticle.builder()
|
||||||
|
.articleId(articleId)
|
||||||
|
.title(title)
|
||||||
|
.articleUrl(articleUrl)
|
||||||
|
.recommended(recommended)
|
||||||
|
.hit(hit)
|
||||||
|
.registeredAt(registeredAt)
|
||||||
|
.build();
|
||||||
|
}
|
||||||
|
|
||||||
|
public static Long parseArticleId(Element td) {
|
||||||
|
return Long.parseLong(td.text().trim());
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String parseTitle(Element td) {
|
||||||
|
return td.text();
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String parseArticleUrl(Element td) {
|
||||||
|
return PpomppuBoardName.ofViewPageUrl(td.getElementsByTag("a").attr("href"));
|
||||||
|
}
|
||||||
|
|
||||||
|
public static Integer parseRecommended(Element td) {
|
||||||
|
final String voteString = td.text();
|
||||||
|
final int recommended;
|
||||||
|
|
||||||
|
if (voteString.isEmpty()) {
|
||||||
|
recommended = 0;
|
||||||
|
} else {
|
||||||
|
final int voteUp = Integer.parseInt(td.text().split(" - ")[0]);
|
||||||
|
final int voteDown = Integer.parseInt(td.text().split(" - ")[1]);
|
||||||
|
recommended = voteUp - voteDown;
|
||||||
|
}
|
||||||
|
return recommended;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static Integer parseHit(Element td) {
|
||||||
|
return Integer.parseInt(td.text());
|
||||||
|
}
|
||||||
|
|
||||||
|
public static Instant parseRegisteredAt(Element td) {
|
||||||
|
final String registeredAtString = td.attr("title");
|
||||||
|
return DATE_TIME_FORMATTER.parse(registeredAtString, Instant::from);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -1,77 +1,50 @@
|
||||||
package com.myoa.engineering.crawl.ppomppu.processor.dto;
|
package com.myoa.engineering.crawl.ppomppu.processor.dto;
|
||||||
|
|
||||||
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
|
|
||||||
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
|
|
||||||
import java.time.Instant;
|
import java.time.Instant;
|
||||||
import java.time.ZoneId;
|
import java.time.ZoneId;
|
||||||
import java.time.format.DateTimeFormatter;
|
import java.time.format.DateTimeFormatter;
|
||||||
import org.jsoup.nodes.Element;
|
import java.util.List;
|
||||||
import org.jsoup.select.Elements;
|
import java.util.function.Function;
|
||||||
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
|
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
|
||||||
|
import com.myoa.engineering.crawl.ppomppu.support.dto.SimpleMessageDTO;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* PpomppuArticleTransformer
|
* PpomppuArticleTransformer
|
||||||
|
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
|
||||||
|
* @since 2021-11-21
|
||||||
*
|
*
|
||||||
* @author Shin Woo-jin (woozu.shin@kakaoent.com)
|
|
||||||
* @since 2021-09-08
|
|
||||||
*/
|
*/
|
||||||
public final class PpomppuArticleTransformer {
|
public final class PpomppuArticleTransformer {
|
||||||
|
|
||||||
private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yy.MM.dd HH:mm:ss")
|
|
||||||
.withZone(ZoneId.of("Asia/Seoul"));
|
|
||||||
|
|
||||||
private PpomppuArticleTransformer() {}
|
private PpomppuArticleTransformer() {}
|
||||||
|
|
||||||
public static PpomppuArticle toArticle(Elements articleElement) {
|
private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")
|
||||||
final long articleId = PpomppuArticleTransformer.toArticleId(articleElement.get(0));
|
.withZone(ZoneId.of("Asia/Seoul"));
|
||||||
final String title = PpomppuArticleTransformer.toTitle(articleElement.get(2));
|
|
||||||
final String articleUrl = PpomppuArticleTransformer.toArticleUrl(articleElement.get(2));
|
|
||||||
final int recommended = PpomppuArticleTransformer.toRecommended(articleElement.get(6));
|
|
||||||
final int hit = PpomppuArticleTransformer.toHit(articleElement.get(7));
|
|
||||||
final Instant registeredAt = PpomppuArticleTransformer.toRegisteredAt(articleElement.get(5));
|
|
||||||
|
|
||||||
return PpomppuArticle.builder()
|
public static final Function<PpomppuArticle, SimpleMessageDTO> TRANSFORM_TO_MESSAGE_DTO = entity ->
|
||||||
.articleId(articleId)
|
SimpleMessageDTO.builder()
|
||||||
.title(title)
|
.requestedAt(Instant.now())
|
||||||
.articleUrl(articleUrl)
|
.publishedAt(entity.getRegisteredAt())
|
||||||
.recommended(recommended)
|
.title(String.format("[%s] %s", entity.getBoardName().getMenuName(), entity.getTitle()))
|
||||||
.hit(hit)
|
.body(entity.getArticleUrl())
|
||||||
.registeredAt(registeredAt)
|
.build();
|
||||||
|
|
||||||
|
// https://stackoverflow.com/questions/24882927/using-streams-to-convert-a-list-of-objects-into-a-string-obtained-from-the-tostr
|
||||||
|
public static SimpleMessageDTO transform(List<PpomppuArticle> articles) {
|
||||||
|
Instant requestedAt = Instant.now();
|
||||||
|
String body = articles.stream()
|
||||||
|
.map(PpomppuArticleTransformer::convertToInlineMessage)
|
||||||
|
.collect(Collectors.joining("\n\n"));
|
||||||
|
return SimpleMessageDTO.builder()
|
||||||
|
.requestedAt(requestedAt)
|
||||||
|
.title(DATE_TIME_FORMATTER.format(requestedAt))
|
||||||
|
.body(body)
|
||||||
.build();
|
.build();
|
||||||
}
|
}
|
||||||
|
|
||||||
public static Long toArticleId(Element td) {
|
public static String convertToInlineMessage(PpomppuArticle article) {
|
||||||
return Long.parseLong(td.text().trim());
|
return String.format("[%s] %s\n%s",
|
||||||
|
article.getBoardName().getMenuName(), article.getTitle(), article.getArticleUrl());
|
||||||
}
|
}
|
||||||
|
|
||||||
public static String toTitle(Element td) {
|
|
||||||
return td.text();
|
|
||||||
}
|
|
||||||
|
|
||||||
public static String toArticleUrl(Element td) {
|
|
||||||
return PpomppuBoardName.ofViewPageUrl(td.getElementsByTag("a").attr("href"));
|
|
||||||
}
|
|
||||||
|
|
||||||
public static Integer toRecommended(Element td) {
|
|
||||||
final String voteString = td.text();
|
|
||||||
final int recommended;
|
|
||||||
|
|
||||||
if (voteString.isEmpty()) {
|
|
||||||
recommended = 0;
|
|
||||||
} else {
|
|
||||||
final int voteUp = Integer.parseInt(td.text().split(" - ")[0]);
|
|
||||||
final int voteDown = Integer.parseInt(td.text().split(" - ")[1]);
|
|
||||||
recommended = voteUp - voteDown;
|
|
||||||
}
|
|
||||||
return recommended;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static Integer toHit(Element td) {
|
|
||||||
return Integer.parseInt(td.text());
|
|
||||||
}
|
|
||||||
|
|
||||||
public static Instant toRegisteredAt(Element td) {
|
|
||||||
final String registeredAtString = td.attr("title");
|
|
||||||
return DATE_TIME_FORMATTER.parse(registeredAtString, Instant::from);
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -2,13 +2,17 @@ package com.myoa.engineering.crawl.ppomppu.processor.infrastructure.client;
|
||||||
|
|
||||||
import org.springframework.stereotype.Component;
|
import org.springframework.stereotype.Component;
|
||||||
import org.springframework.web.reactive.function.client.WebClient;
|
import org.springframework.web.reactive.function.client.WebClient;
|
||||||
|
import org.springframework.web.reactive.function.client.WebClientRequestException;
|
||||||
|
|
||||||
import com.myoa.engineering.crawl.ppomppu.processor.dto.constant.WebClientPropertiesUnitName;
|
import com.myoa.engineering.crawl.ppomppu.processor.dto.constant.WebClientPropertiesUnitName;
|
||||||
|
import com.myoa.engineering.crawl.ppomppu.support.dto.SimpleMessageDTO;
|
||||||
import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebClientFilterFactory;
|
import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebClientFilterFactory;
|
||||||
import com.myoa.engineering.crawl.ppomppu.support.webclient.properties.WebClientProperties;
|
import com.myoa.engineering.crawl.ppomppu.support.webclient.properties.WebClientProperties;
|
||||||
import com.myoa.engineering.crawl.ppomppu.support.webclient.properties.WebClientProperties.WebClientPropertiesUnit;
|
import com.myoa.engineering.crawl.ppomppu.support.webclient.properties.WebClientProperties.WebClientPropertiesUnit;
|
||||||
|
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import reactor.core.publisher.Mono;
|
||||||
|
import reactor.core.scheduler.Schedulers;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* PpomppuNotifierSenderAPIClient
|
* PpomppuNotifierSenderAPIClient
|
||||||
|
@ -18,11 +22,11 @@ import lombok.extern.slf4j.Slf4j;
|
||||||
*/
|
*/
|
||||||
@Slf4j
|
@Slf4j
|
||||||
@Component
|
@Component
|
||||||
public class SenderAPIClient {
|
public class MessageSenderAPIClient {
|
||||||
|
|
||||||
private final WebClient webClient;
|
private final WebClient webClient;
|
||||||
|
|
||||||
public SenderAPIClient(WebClientProperties webClientProperties) {
|
public MessageSenderAPIClient(WebClientProperties webClientProperties) {
|
||||||
WebClientPropertiesUnit webClientPropertiesUnit =
|
WebClientPropertiesUnit webClientPropertiesUnit =
|
||||||
webClientProperties.find(WebClientPropertiesUnitName.PPOMPPU_NOTIFIER_SENDER_API.getUnitName());
|
webClientProperties.find(WebClientPropertiesUnitName.PPOMPPU_NOTIFIER_SENDER_API.getUnitName());
|
||||||
this.webClient = WebClient.builder()
|
this.webClient = WebClient.builder()
|
||||||
|
@ -31,6 +35,16 @@ public class SenderAPIClient {
|
||||||
.filter(WebClientFilterFactory.logResponse())
|
.filter(WebClientFilterFactory.logResponse())
|
||||||
.build();
|
.build();
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
// TODO webclient properties
|
public Mono<String> sendMessageToSlack(SimpleMessageDTO dto) {
|
||||||
|
return webClient.post()
|
||||||
|
.uri("/api/v1/messages/sendMessage/messengers/slack")
|
||||||
|
.bodyValue(dto)
|
||||||
|
.exchangeToMono(e -> e.bodyToMono(String.class))
|
||||||
|
.publishOn(Schedulers.boundedElastic())
|
||||||
|
.onErrorResume(WebClientRequestException.class, t -> {
|
||||||
|
log.info("Exception occured, ignoring. : {}", t.getClass().getSimpleName());
|
||||||
|
return Mono.empty();
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
|
@ -40,8 +40,8 @@ public class PpomppuBoardFeedRetriever {
|
||||||
.onErrorResume(WebClientRequestException.class, t -> {
|
.onErrorResume(WebClientRequestException.class, t -> {
|
||||||
log.info("Exception occured, ignoring. : {}", t.getClass().getSimpleName());
|
log.info("Exception occured, ignoring. : {}", t.getClass().getSimpleName());
|
||||||
return Mono.empty();
|
return Mono.empty();
|
||||||
})
|
});
|
||||||
.doOnNext(e -> log.info("[getHtml] {}", e));
|
// .doOnNext(e -> log.info("[getHtml] {}", e));
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -0,0 +1,37 @@
|
||||||
|
package com.myoa.engineering.crawl.ppomppu.processor.service;
|
||||||
|
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
|
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
|
||||||
|
import com.myoa.engineering.crawl.ppomppu.processor.dto.PpomppuArticleTransformer;
|
||||||
|
import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.client.MessageSenderAPIClient;
|
||||||
|
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* MessageSenderService
|
||||||
|
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
|
||||||
|
* @since 2021-11-21
|
||||||
|
*
|
||||||
|
*/
|
||||||
|
@Slf4j
|
||||||
|
@Service
|
||||||
|
public class MessageSenderService {
|
||||||
|
|
||||||
|
private final MessageSenderAPIClient messageSenderAPIClient;
|
||||||
|
|
||||||
|
public MessageSenderService(MessageSenderAPIClient messageSenderAPIClient) {
|
||||||
|
this.messageSenderAPIClient = messageSenderAPIClient;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void sendMessageToSlack(PpomppuArticle article) {
|
||||||
|
messageSenderAPIClient.sendMessageToSlack(PpomppuArticleTransformer.TRANSFORM_TO_MESSAGE_DTO.apply(article));
|
||||||
|
}
|
||||||
|
|
||||||
|
public void sendMessageToSlack(List<PpomppuArticle> articles) {
|
||||||
|
messageSenderAPIClient.sendMessageToSlack(PpomppuArticleTransformer.transform(articles));
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -1,7 +1,7 @@
|
||||||
package com.myoa.engineering.crawl.ppomppu.processor.service;
|
package com.myoa.engineering.crawl.ppomppu.processor.service;
|
||||||
|
|
||||||
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
|
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
|
||||||
import com.myoa.engineering.crawl.ppomppu.processor.dto.PpomppuArticleTransformer;
|
import com.myoa.engineering.crawl.ppomppu.processor.dto.PpomppuArticleParser;
|
||||||
import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.client.PpomppuBoardFeedRetriever;
|
import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.client.PpomppuBoardFeedRetriever;
|
||||||
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
|
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
@ -30,11 +30,11 @@ public class PpomppuFeedService {
|
||||||
|
|
||||||
public Mono<List<PpomppuArticle>> getArticles(PpomppuBoardName boardName) {
|
public Mono<List<PpomppuArticle>> getArticles(PpomppuBoardName boardName) {
|
||||||
final Mono<String> html = ppomppuBoardFeedRetriever.getHtml(boardName.getResourcePath());
|
final Mono<String> html = ppomppuBoardFeedRetriever.getHtml(boardName.getResourcePath());
|
||||||
final Mono<Element> tbody = extractTbodyFromHtml(html)
|
final Mono<Element> tbody = extractTbodyFromHtml(html);
|
||||||
.doOnNext(e -> log.info("pre tbody - {}", e.html()));
|
// .doOnNext(e -> log.info("pre tbody - {}", e.html()));
|
||||||
return extractArticlesFromTbody(tbody).map(this::convertFromElement)
|
return extractArticlesFromTbody(tbody).map(this::convertFromElement)
|
||||||
.map(e -> e.updateBoardName(boardName))
|
.map(e -> e.updateBoardName(boardName))
|
||||||
.doOnNext(e -> log.info("parsed Result: {}", e))
|
// .doOnNext(e -> log.info("parsed Result: {}", e))
|
||||||
.collectList();
|
.collectList();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -42,7 +42,7 @@ public class PpomppuFeedService {
|
||||||
return html.map(Jsoup::parse)
|
return html.map(Jsoup::parse)
|
||||||
.mapNotNull(e -> e.getElementById("revolution_main_table"))
|
.mapNotNull(e -> e.getElementById("revolution_main_table"))
|
||||||
.map(e -> e.getElementsByTag("tbody"))
|
.map(e -> e.getElementsByTag("tbody"))
|
||||||
.doOnNext(e -> log.info("tbody - {}", e.html()))
|
// .doOnNext(e -> log.info("tbody - {}", e.html()))
|
||||||
.map(e -> e.stream()
|
.map(e -> e.stream()
|
||||||
.findFirst()
|
.findFirst()
|
||||||
.orElseThrow(() -> new IndexOutOfBoundsException("no tbody")));
|
.orElseThrow(() -> new IndexOutOfBoundsException("no tbody")));
|
||||||
|
@ -54,6 +54,6 @@ public class PpomppuFeedService {
|
||||||
}
|
}
|
||||||
|
|
||||||
private PpomppuArticle convertFromElement(Element element) {
|
private PpomppuArticle convertFromElement(Element element) {
|
||||||
return PpomppuArticleTransformer.toArticle(element.getElementsByTag("td"));
|
return PpomppuArticleParser.toArticle(element.getElementsByTag("td"));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -5,9 +5,7 @@ spring:
|
||||||
import:
|
import:
|
||||||
- "configserver:http://192.168.0.100:11080"
|
- "configserver:http://192.168.0.100:11080"
|
||||||
- classpath:/development/webclient.yml
|
- classpath:/development/webclient.yml
|
||||||
profiles:
|
|
||||||
group:
|
|
||||||
development: development, datasource-development
|
|
||||||
|
|
||||||
server:
|
server:
|
||||||
port: 20081
|
port: 20081
|
||||||
|
|
|
@ -1,10 +1,6 @@
|
||||||
spring:
|
spring:
|
||||||
config:
|
config:
|
||||||
activate:
|
activate:
|
||||||
on-profile: development
|
on-profile: production
|
||||||
import:
|
import:
|
||||||
- "configserver:http://192.168.0.100:11080"
|
- "configserver:http://192.168.0.100:11080"
|
||||||
- classpath:/development/webclient.yml
|
|
||||||
profiles:
|
|
||||||
group:
|
|
||||||
development: development, datasource-development
|
|
||||||
|
|
|
@ -5,10 +5,12 @@ spring:
|
||||||
allow-bean-definition-overriding: true
|
allow-bean-definition-overriding: true
|
||||||
profiles:
|
profiles:
|
||||||
active: ${SPRING_ACTIVE_PROFILE:development}
|
active: ${SPRING_ACTIVE_PROFILE:development}
|
||||||
|
group:
|
||||||
|
development: "development,datasource-development"
|
||||||
|
production: "production, datasource-production"
|
||||||
freemarker:
|
freemarker:
|
||||||
enabled: false
|
enabled: false
|
||||||
|
|
||||||
|
|
||||||
server:
|
server:
|
||||||
port: 20080
|
port: 20080
|
||||||
error:
|
error:
|
||||||
|
|
|
@ -0,0 +1,31 @@
|
||||||
|
package com.myoa.engineering.crawl.ppomppu.sender.controller;
|
||||||
|
|
||||||
|
import org.springframework.web.bind.annotation.PostMapping;
|
||||||
|
import org.springframework.web.bind.annotation.RequestMapping;
|
||||||
|
import org.springframework.web.bind.annotation.RestController;
|
||||||
|
|
||||||
|
import com.myoa.engineering.crawl.ppomppu.support.dto.APIResponse;
|
||||||
|
import com.myoa.engineering.crawl.ppomppu.support.dto.SimpleMessageDTO;
|
||||||
|
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import reactor.core.publisher.Mono;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* MessageSenderAPIController
|
||||||
|
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
|
||||||
|
* @since 2021-11-21
|
||||||
|
*
|
||||||
|
*/
|
||||||
|
@Slf4j
|
||||||
|
@RestController
|
||||||
|
@RequestMapping("/api/v1")
|
||||||
|
public class MessageSenderAPIController {
|
||||||
|
|
||||||
|
@PostMapping("/messages/sendMessage/messengers/slack")
|
||||||
|
public Mono<APIResponse<SimpleMessageDTO>> sendMessageToSlack(SimpleMessageDTO dto) {
|
||||||
|
|
||||||
|
log.info("received : {}, \nbody: {}", dto.getTitle(), dto.getBody());
|
||||||
|
// TODO transform
|
||||||
|
return Mono.just(APIResponse.success(dto));
|
||||||
|
}
|
||||||
|
}
|
|
@ -3,12 +3,8 @@ spring:
|
||||||
activate:
|
activate:
|
||||||
on-profile: development
|
on-profile: development
|
||||||
import:
|
import:
|
||||||
- classpath:/development/webclient.yml
|
|
||||||
- "configserver:http://192.168.0.100:11080"
|
- "configserver:http://192.168.0.100:11080"
|
||||||
# import: optional:configserver:http://localhost:11080 # can be start up even config server was not found.
|
# import: optional:configserver:http://localhost:11080 # can be start up even config server was not found.
|
||||||
profiles:
|
|
||||||
group:
|
|
||||||
development: development, slackapi-development
|
|
||||||
|
|
||||||
server:
|
server:
|
||||||
port: 20081
|
port: 20082
|
||||||
|
|
|
@ -2,5 +2,5 @@ spring:
|
||||||
config:
|
config:
|
||||||
activate:
|
activate:
|
||||||
on-profile: production
|
on-profile: production
|
||||||
# import:
|
import:
|
||||||
# - classpath:/production/webclient.yml
|
- "configserver:http://192.168.0.100:11080"
|
||||||
|
|
|
@ -5,9 +5,17 @@ spring:
|
||||||
allow-bean-definition-overriding: true
|
allow-bean-definition-overriding: true
|
||||||
profiles:
|
profiles:
|
||||||
active: ${SPRING_ACTIVE_PROFILE:development}
|
active: ${SPRING_ACTIVE_PROFILE:development}
|
||||||
|
group:
|
||||||
|
development: development, slackapi-development
|
||||||
|
production: production, slackapi-production
|
||||||
freemarker:
|
freemarker:
|
||||||
enabled: false
|
enabled: false
|
||||||
|
|
||||||
|
jpa:
|
||||||
|
hibernate:
|
||||||
|
ddl-auto: create-drop
|
||||||
|
generate-ddl: true
|
||||||
|
|
||||||
server:
|
server:
|
||||||
port: 20080
|
port: 20080
|
||||||
error:
|
error:
|
||||||
|
|
|
@ -0,0 +1,37 @@
|
||||||
|
package com.myoa.engineering.crawl.ppomppu.support.dto;
|
||||||
|
|
||||||
|
import java.io.Serializable;
|
||||||
|
import java.time.Instant;
|
||||||
|
|
||||||
|
import lombok.Builder;
|
||||||
|
import lombok.Getter;
|
||||||
|
import lombok.NoArgsConstructor;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* SimpleMessageDTO
|
||||||
|
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
|
||||||
|
* @since 2021-11-21
|
||||||
|
*
|
||||||
|
*/
|
||||||
|
@Getter
|
||||||
|
@NoArgsConstructor
|
||||||
|
public class SimpleMessageDTO implements Serializable {
|
||||||
|
|
||||||
|
private static final long serialVersionUID = 2203955567672404428L;
|
||||||
|
|
||||||
|
private String title;
|
||||||
|
private String body;
|
||||||
|
private String url;
|
||||||
|
private Instant publishedAt;
|
||||||
|
private Instant requestedAt;
|
||||||
|
|
||||||
|
@Builder
|
||||||
|
public SimpleMessageDTO(String title, String body, String url, Instant publishedAt, Instant requestedAt) {
|
||||||
|
this.title = title;
|
||||||
|
this.body = body;
|
||||||
|
this.url = url;
|
||||||
|
this.publishedAt = publishedAt;
|
||||||
|
this.requestedAt = requestedAt;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
Loading…
Reference in New Issue