Merge pull request '[PPN-211113] Implement common sender component' (#12) from feature/PPN-211113 into develop
Reviewed-on: OutworldDestroyer/PpomppuNotifier#12
This commit is contained in:
commit
26520fba79
11
build.gradle
11
build.gradle
|
@ -32,6 +32,17 @@ allprojects {
|
|||
}
|
||||
}
|
||||
|
||||
ext {
|
||||
set('springCloudVersion', "2020.0.4")
|
||||
}
|
||||
|
||||
dependencyManagement {
|
||||
imports {
|
||||
mavenBom "org.springframework.cloud:spring-cloud-dependencies:${springCloudVersion}"
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/* task initSourceFolders {
|
||||
sourceSets*.java.srcDirs*.each {
|
||||
if( !it.exists() ) {
|
||||
|
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
dependencies {
|
||||
developmentOnly 'org.springframework.boot:spring-boot-devtools'
|
||||
runtimeOnly 'com.h2database:h2'
|
||||
|
@ -9,6 +10,8 @@ dependencies {
|
|||
implementation 'org.springframework.boot:spring-boot-starter-webflux'
|
||||
implementation 'org.springframework.boot:spring-boot-starter-data-jpa'
|
||||
implementation 'org.springframework.boot:spring-boot-starter-jdbc'
|
||||
implementation 'org.springframework.cloud:spring-cloud-starter-config'
|
||||
implementation 'org.springframework.boot:spring-boot-starter-actuator'
|
||||
implementation 'com.rometools:rome:1.16.0'
|
||||
implementation 'org.jsoup:jsoup:1.14.2'
|
||||
implementation 'com.h2database:h2:1.4.200'
|
||||
|
|
|
@ -2,6 +2,9 @@ package com.myoa.engineering.crawl.ppomppu.processor;
|
|||
|
||||
import org.springframework.boot.SpringApplication;
|
||||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||
import org.springframework.context.annotation.Import;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.support.webclient.PpomppuNotifierWebClientConfiguration;
|
||||
|
||||
/**
|
||||
* ProcessorApplication
|
||||
|
@ -9,6 +12,7 @@ import org.springframework.boot.autoconfigure.SpringBootApplication;
|
|||
* @since 2021-08-20
|
||||
*
|
||||
*/
|
||||
@Import({ PpomppuNotifierWebClientConfiguration.class })
|
||||
@SpringBootApplication
|
||||
public class ProcessorApplication {
|
||||
|
||||
|
|
|
@ -11,7 +11,7 @@ import org.springframework.context.event.ContextRefreshedEvent;
|
|||
import org.springframework.context.event.EventListener;
|
||||
|
||||
@Slf4j
|
||||
@Profile("development")
|
||||
@Profile({"datasource-local", "datasource-development"})
|
||||
@Configuration
|
||||
public class H2ConsoleConfiguration {
|
||||
|
||||
|
|
|
@ -32,7 +32,7 @@ import org.springframework.transaction.PlatformTransactionManager;
|
|||
)
|
||||
public class PpomppuDatasourceConfiguration {
|
||||
|
||||
private static final String DATA_SOURCE_UNIT_NAME = "ppomppu";
|
||||
private static final String DATA_SOURCE_UNIT_NAME = "ppn_mysql";
|
||||
|
||||
private final DatasourceProperties dataSourceProeprties;
|
||||
private final HikariProperties hikariProperties;
|
||||
|
@ -52,8 +52,9 @@ public class PpomppuDatasourceConfiguration {
|
|||
|
||||
final HikariConfig hikariConfig = new HikariConfig();
|
||||
hikariConfig.setJdbcUrl(dataSourcePropertiesUnit.toCompletedJdbcUrl());
|
||||
hikariConfig.setUsername("sa");
|
||||
hikariConfig.setPassword("sa");
|
||||
hikariConfig.setDriverClassName(dataSourcePropertiesUnit.getDriverClassName());
|
||||
hikariConfig.setUsername(dataSourcePropertiesUnit.getUsername());
|
||||
hikariConfig.setPassword(dataSourcePropertiesUnit.getPassword());
|
||||
hikariConfig.setAutoCommit(hikariProperties.getAutoCommit());
|
||||
hikariConfig.setMaximumPoolSize(hikariProperties.getMaximumPoolSize());
|
||||
hikariConfig.setMinimumIdle(hikariProperties.getMinimumIdle());
|
||||
|
|
|
@ -2,6 +2,7 @@ package com.myoa.engineering.crawl.ppomppu.processor.configuration.properties;
|
|||
|
||||
import com.myoa.engineering.crawl.ppomppu.support.util.ObjectUtil;
|
||||
import java.util.List;
|
||||
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
|
@ -10,7 +11,7 @@ import org.springframework.stereotype.Component;
|
|||
@Component
|
||||
@Setter
|
||||
@Getter
|
||||
@ConfigurationProperties(prefix = "datasource")
|
||||
@ConfigurationProperties(prefix = "infra.database")
|
||||
public class DatasourceProperties {
|
||||
|
||||
private List<DataSourcePropertiesUnit> units;
|
||||
|
@ -22,14 +23,17 @@ public class DatasourceProperties {
|
|||
private String unitName;
|
||||
private String schemaName;
|
||||
private String connectionParameters;
|
||||
private String dbConnectionUrl;
|
||||
private Boolean simpleConnectionUrl;
|
||||
private String datasourceUrl;
|
||||
private Boolean isSimpleConnectionUrl;
|
||||
private String username;
|
||||
private String password;
|
||||
private String driverClassName;
|
||||
|
||||
public String toCompletedJdbcUrl() {
|
||||
if (ObjectUtil.isEmpty(simpleConnectionUrl) || simpleConnectionUrl == false) {
|
||||
return String.format("%s/%s?%s", dbConnectionUrl, schemaName, connectionParameters);
|
||||
if (ObjectUtil.isEmpty(isSimpleConnectionUrl) || isSimpleConnectionUrl == false) {
|
||||
return String.format("%s/%s?%s", datasourceUrl, schemaName, connectionParameters);
|
||||
}
|
||||
return dbConnectionUrl;
|
||||
return datasourceUrl;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -3,6 +3,9 @@ package com.myoa.engineering.crawl.ppomppu.processor.configuration.properties;
|
|||
import java.util.List;
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
|
||||
import org.hibernate.dialect.MySQL8Dialect;
|
||||
import org.hibernate.dialect.MySQLDialect;
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
|
|
|
@ -1,17 +1,19 @@
|
|||
package com.myoa.engineering.crawl.ppomppu.processor.controller;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
|
||||
import com.myoa.engineering.crawl.ppomppu.processor.dto.FeedParsedResult;
|
||||
import com.myoa.engineering.crawl.ppomppu.processor.service.PpomppuArticleService;
|
||||
import com.myoa.engineering.crawl.ppomppu.processor.service.PpomppuFeedService;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.dto.APIResponse;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
|
||||
import java.util.List;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.processor.dto.FeedParsedResult;
|
||||
import com.myoa.engineering.crawl.ppomppu.processor.service.MessageSenderService;
|
||||
import com.myoa.engineering.crawl.ppomppu.processor.service.PpomppuArticleService;
|
||||
import com.myoa.engineering.crawl.ppomppu.processor.service.PpomppuFeedService;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.dto.APIResponse;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import reactor.core.publisher.Mono;
|
||||
|
||||
/**
|
||||
|
@ -27,22 +29,28 @@ public class CrawlAPIController {
|
|||
|
||||
private final PpomppuFeedService ppomppuRSSFeedService;
|
||||
private final PpomppuArticleService ppomppuArticleService;
|
||||
private final MessageSenderService messageSenderService;
|
||||
|
||||
public CrawlAPIController(PpomppuFeedService ppomppuRSSFeedService,
|
||||
PpomppuArticleService ppomppuArticleService) {
|
||||
PpomppuArticleService ppomppuArticleService,
|
||||
MessageSenderService messageSenderService) {
|
||||
this.ppomppuRSSFeedService = ppomppuRSSFeedService;
|
||||
this.ppomppuArticleService = ppomppuArticleService;
|
||||
this.messageSenderService = messageSenderService;
|
||||
}
|
||||
|
||||
@PostMapping("/boards/{boardName}")
|
||||
public Mono<APIResponse<FeedParsedResult>> crawlBoard(@PathVariable("boardName") PpomppuBoardName boardName) {
|
||||
log.info("got request... {}", boardName);
|
||||
FeedParsedResult result = FeedParsedResult.of(boardName);
|
||||
Mono<List<PpomppuArticle>> articles = ppomppuRSSFeedService.getArticles(boardName)
|
||||
.doOnNext(e -> ppomppuArticleService.filterOnlyNewArticles(boardName, e))
|
||||
.doOnNext(e -> ppomppuArticleService.save(boardName, e));
|
||||
|
||||
return articles.then(Mono.just(APIResponse.success(result.done())));
|
||||
Mono<String> publishedMessages =
|
||||
ppomppuRSSFeedService.getArticles(boardName)
|
||||
.map(e -> ppomppuArticleService.filterOnlyNewArticles(boardName, e))
|
||||
.map(e -> ppomppuArticleService.save(boardName, e))
|
||||
.flatMap(messageSenderService::sendMessageToSlack);
|
||||
|
||||
return publishedMessages.then(Mono.just(APIResponse.success(result.done())));
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -0,0 +1,79 @@
|
|||
package com.myoa.engineering.crawl.ppomppu.processor.dto;
|
||||
|
||||
import java.time.Instant;
|
||||
import java.time.ZoneId;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
|
||||
import org.jsoup.nodes.Element;
|
||||
import org.jsoup.select.Elements;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
|
||||
|
||||
/**
|
||||
* PpomppuArticleTransformer
|
||||
*
|
||||
* @author Shin Woo-jin (woozu.shin@kakaoent.com)
|
||||
* @since 2021-09-08
|
||||
*/
|
||||
public final class PpomppuArticleParser {
|
||||
|
||||
private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yy.MM.dd HH:mm:ss")
|
||||
.withZone(ZoneId.of("Asia/Seoul"));
|
||||
|
||||
private PpomppuArticleParser() {}
|
||||
|
||||
public static PpomppuArticle toArticle(Elements articleElement) {
|
||||
final long articleId = PpomppuArticleParser.parseArticleId(articleElement.get(0));
|
||||
final String title = PpomppuArticleParser.parseTitle(articleElement.get(2));
|
||||
final String articleUrl = PpomppuArticleParser.parseArticleUrl(articleElement.get(2));
|
||||
final int recommended = PpomppuArticleParser.parseRecommended(articleElement.get(6));
|
||||
final int hit = PpomppuArticleParser.parseHit(articleElement.get(7));
|
||||
final Instant registeredAt = PpomppuArticleParser.parseRegisteredAt(articleElement.get(5));
|
||||
|
||||
return PpomppuArticle.builder()
|
||||
.articleId(articleId)
|
||||
.title(title)
|
||||
.articleUrl(articleUrl)
|
||||
.recommended(recommended)
|
||||
.hit(hit)
|
||||
.registeredAt(registeredAt)
|
||||
.build();
|
||||
}
|
||||
|
||||
public static Long parseArticleId(Element td) {
|
||||
return Long.parseLong(td.text().trim());
|
||||
}
|
||||
|
||||
public static String parseTitle(Element td) {
|
||||
return td.getElementsByTag("a").text(); // TODO cdn image extracting
|
||||
}
|
||||
|
||||
public static String parseArticleUrl(Element td) {
|
||||
return PpomppuBoardName.ofViewPageUrl(td.getElementsByTag("a").attr("href"));
|
||||
}
|
||||
|
||||
public static Integer parseRecommended(Element td) {
|
||||
final String voteString = td.text();
|
||||
final int recommended;
|
||||
|
||||
if (voteString.isEmpty()) {
|
||||
recommended = 0;
|
||||
} else {
|
||||
final int voteUp = Integer.parseInt(td.text().split(" - ")[0]);
|
||||
final int voteDown = Integer.parseInt(td.text().split(" - ")[1]);
|
||||
recommended = voteUp - voteDown;
|
||||
}
|
||||
return recommended;
|
||||
}
|
||||
|
||||
public static Integer parseHit(Element td) {
|
||||
return Integer.parseInt(td.text());
|
||||
}
|
||||
|
||||
public static Instant parseRegisteredAt(Element td) {
|
||||
final String registeredAtString = td.attr("title");
|
||||
return DATE_TIME_FORMATTER.parse(registeredAtString, Instant::from);
|
||||
}
|
||||
|
||||
}
|
|
@ -1,77 +1,53 @@
|
|||
package com.myoa.engineering.crawl.ppomppu.processor.dto;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
|
||||
import java.time.Instant;
|
||||
import java.time.ZoneId;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import org.jsoup.nodes.Element;
|
||||
import org.jsoup.select.Elements;
|
||||
import java.util.List;
|
||||
import java.util.function.Function;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.dto.SimpleMessageDTO;
|
||||
|
||||
/**
|
||||
* PpomppuArticleTransformer
|
||||
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
|
||||
* @since 2021-11-21
|
||||
*
|
||||
* @author Shin Woo-jin (woozu.shin@kakaoent.com)
|
||||
* @since 2021-09-08
|
||||
*/
|
||||
public final class PpomppuArticleTransformer {
|
||||
|
||||
private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yy.MM.dd HH:mm:ss")
|
||||
.withZone(ZoneId.of("Asia/Seoul"));
|
||||
|
||||
private PpomppuArticleTransformer() {}
|
||||
|
||||
public static PpomppuArticle toArticle(Elements articleElement) {
|
||||
final long articleId = PpomppuArticleTransformer.toArticleId(articleElement.get(0));
|
||||
final String title = PpomppuArticleTransformer.toTitle(articleElement.get(2));
|
||||
final String articleUrl = PpomppuArticleTransformer.toArticleUrl(articleElement.get(2));
|
||||
final int recommended = PpomppuArticleTransformer.toRecommended(articleElement.get(6));
|
||||
final int hit = PpomppuArticleTransformer.toHit(articleElement.get(7));
|
||||
final Instant registeredAt = PpomppuArticleTransformer.toRegisteredAt(articleElement.get(5));
|
||||
private static final String MESSAGE_FORMAT_V1 = "%s)) `%s` <%s:LINK>";
|
||||
|
||||
return PpomppuArticle.builder()
|
||||
.articleId(articleId)
|
||||
.title(title)
|
||||
.articleUrl(articleUrl)
|
||||
.recommended(recommended)
|
||||
.hit(hit)
|
||||
.registeredAt(registeredAt)
|
||||
.build();
|
||||
private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")
|
||||
.withZone(ZoneId.of("Asia/Seoul"));
|
||||
|
||||
public static final Function<PpomppuArticle, SimpleMessageDTO> TRANSFORM_TO_MESSAGE_DTO = entity ->
|
||||
SimpleMessageDTO.builder()
|
||||
.requestedAt(Instant.now())
|
||||
.publishedAt(entity.getRegisteredAt())
|
||||
.title(String.format(MESSAGE_FORMAT_V1, entity.getBoardName().getMenuName(), entity.getTitle()))
|
||||
.body(entity.getArticleUrl())
|
||||
.build();
|
||||
|
||||
// https://stackoverflow.com/questions/24882927/using-streams-to-convert-a-list-of-objects-into-a-string-obtained-from-the-tostr
|
||||
public static SimpleMessageDTO transform(List<PpomppuArticle> articles) {
|
||||
Instant requestedAt = Instant.now();
|
||||
String body = articles.stream()
|
||||
.map(PpomppuArticleTransformer::convertToInlineMessage)
|
||||
.collect(Collectors.joining("\n\n"));
|
||||
return SimpleMessageDTO.builder()
|
||||
.requestedAt(requestedAt)
|
||||
.title(DATE_TIME_FORMATTER.format(requestedAt))
|
||||
.body(body)
|
||||
.build();
|
||||
}
|
||||
|
||||
public static Long toArticleId(Element td) {
|
||||
return Long.parseLong(td.text().trim());
|
||||
public static String convertToInlineMessage(PpomppuArticle article) {
|
||||
return String.format(MESSAGE_FORMAT_V1,
|
||||
article.getBoardName().getMenuName(), article.getTitle(), article.getArticleUrl());
|
||||
}
|
||||
|
||||
public static String toTitle(Element td) {
|
||||
return td.text();
|
||||
}
|
||||
|
||||
public static String toArticleUrl(Element td) {
|
||||
return PpomppuBoardName.ofViewPageUrl(td.getElementsByTag("a").attr("href"));
|
||||
}
|
||||
|
||||
public static Integer toRecommended(Element td) {
|
||||
final String voteString = td.text();
|
||||
final int recommended;
|
||||
|
||||
if (voteString.isEmpty()) {
|
||||
recommended = 0;
|
||||
} else {
|
||||
final int voteUp = Integer.parseInt(td.text().split(" - ")[0]);
|
||||
final int voteDown = Integer.parseInt(td.text().split(" - ")[1]);
|
||||
recommended = voteUp - voteDown;
|
||||
}
|
||||
return recommended;
|
||||
}
|
||||
|
||||
public static Integer toHit(Element td) {
|
||||
return Integer.parseInt(td.text());
|
||||
}
|
||||
|
||||
public static Instant toRegisteredAt(Element td) {
|
||||
final String registeredAtString = td.attr("title");
|
||||
return DATE_TIME_FORMATTER.parse(registeredAtString, Instant::from);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -0,0 +1,19 @@
|
|||
package com.myoa.engineering.crawl.ppomppu.processor.dto.constant;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Getter;
|
||||
|
||||
/**
|
||||
* WebClientPropertiesUnitName
|
||||
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
|
||||
* @since 2021-11-18
|
||||
*
|
||||
*/
|
||||
@Getter
|
||||
@AllArgsConstructor
|
||||
public enum WebClientPropertiesUnitName {
|
||||
PPOMPPU_NOTIFIER_SENDER_API("ppn-sender-api"),
|
||||
;
|
||||
|
||||
private String unitName;
|
||||
}
|
|
@ -0,0 +1,56 @@
|
|||
package com.myoa.engineering.crawl.ppomppu.processor.infrastructure.client;
|
||||
|
||||
import org.springframework.core.ParameterizedTypeReference;
|
||||
import org.springframework.http.HttpHeaders;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.web.reactive.function.client.WebClient;
|
||||
import org.springframework.web.reactive.function.client.WebClientRequestException;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.processor.dto.constant.WebClientPropertiesUnitName;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.dto.SimpleMessageDTO;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebClientFilterFactory;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebFluxExchangeStragiesFactory;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.webclient.properties.WebClientProperties;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.webclient.properties.WebClientProperties.WebClientPropertiesUnit;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import reactor.core.publisher.Mono;
|
||||
import reactor.core.scheduler.Schedulers;
|
||||
|
||||
/**
|
||||
* PpomppuNotifierSenderAPIClient
|
||||
*
|
||||
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
|
||||
* @since 2021-11-17
|
||||
*/
|
||||
@Slf4j
|
||||
@Component
|
||||
public class MessageSenderAPIClient {
|
||||
|
||||
private final WebClient webClient;
|
||||
|
||||
public MessageSenderAPIClient(WebClientProperties webClientProperties) {
|
||||
WebClientPropertiesUnit webClientPropertiesUnit =
|
||||
webClientProperties.find(WebClientPropertiesUnitName.PPOMPPU_NOTIFIER_SENDER_API.getUnitName());
|
||||
this.webClient = WebClient.builder()
|
||||
.baseUrl(webClientPropertiesUnit.getBaseUrl())
|
||||
.exchangeStrategies(WebFluxExchangeStragiesFactory.ofDefault())
|
||||
.defaultHeader(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_VALUE)
|
||||
// .filter(WebClientFilterFactory.logRequest())
|
||||
// .filter(WebClientFilterFactory.logResponse())
|
||||
.build();
|
||||
}
|
||||
|
||||
public Mono<String> sendMessageToSlack(SimpleMessageDTO dto) {
|
||||
return webClient.post()
|
||||
.uri("/api/v1/messages/sendMessage/messengers/slack")
|
||||
.bodyValue(dto)
|
||||
.exchangeToMono(e -> e.bodyToMono(new ParameterizedTypeReference<String>() {}))
|
||||
.publishOn(Schedulers.boundedElastic())
|
||||
.onErrorResume(WebClientRequestException.class, t -> {
|
||||
log.info("Exception occured, ignoring. : {}", t.getClass().getSimpleName());
|
||||
return Mono.empty();
|
||||
});
|
||||
}
|
||||
}
|
|
@ -1,9 +1,11 @@
|
|||
package com.myoa.engineering.crawl.ppomppu.processor.infrastructure.client;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.processor.configuration.factory.WebClientFilterFactory;
|
||||
import com.myoa.engineering.crawl.ppomppu.processor.configuration.factory.WebFluxExchangeStragiesFactory;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebClientFilterFactory;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebFluxExchangeStragiesFactory;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import org.springframework.core.ParameterizedTypeReference;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.web.reactive.function.client.WebClient;
|
||||
import org.springframework.web.reactive.function.client.WebClientRequestException;
|
||||
|
@ -38,8 +40,8 @@ public class PpomppuBoardFeedRetriever {
|
|||
.onErrorResume(WebClientRequestException.class, t -> {
|
||||
log.info("Exception occured, ignoring. : {}", t.getClass().getSimpleName());
|
||||
return Mono.empty();
|
||||
})
|
||||
.doOnNext(e -> log.info("[getHtml] {}", e));
|
||||
});
|
||||
// .doOnNext(e -> log.info("[getHtml] {}", e));
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -0,0 +1,38 @@
|
|||
package com.myoa.engineering.crawl.ppomppu.processor.service;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
|
||||
import com.myoa.engineering.crawl.ppomppu.processor.dto.PpomppuArticleTransformer;
|
||||
import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.client.MessageSenderAPIClient;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import reactor.core.publisher.Mono;
|
||||
|
||||
/**
|
||||
* MessageSenderService
|
||||
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
|
||||
* @since 2021-11-21
|
||||
*
|
||||
*/
|
||||
@Slf4j
|
||||
@Service
|
||||
public class MessageSenderService {
|
||||
|
||||
private final MessageSenderAPIClient messageSenderAPIClient;
|
||||
|
||||
public MessageSenderService(MessageSenderAPIClient messageSenderAPIClient) {
|
||||
this.messageSenderAPIClient = messageSenderAPIClient;
|
||||
}
|
||||
|
||||
public Mono<String> sendMessageToSlack(PpomppuArticle article) {
|
||||
return messageSenderAPIClient.sendMessageToSlack(PpomppuArticleTransformer.TRANSFORM_TO_MESSAGE_DTO.apply(article));
|
||||
}
|
||||
|
||||
public Mono<String> sendMessageToSlack(List<PpomppuArticle> articles) {
|
||||
return messageSenderAPIClient.sendMessageToSlack(PpomppuArticleTransformer.transform(articles));
|
||||
}
|
||||
|
||||
}
|
|
@ -1,16 +1,19 @@
|
|||
package com.myoa.engineering.crawl.ppomppu.processor.service;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
|
||||
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuBoardFeedStatus;
|
||||
import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository.PpomppuArticleRepository;
|
||||
import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository.PpomppuBoardFeedStatusRepository;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
|
@ -32,13 +35,14 @@ public class PpomppuArticleService {
|
|||
Long latestArticleId = boardFeedStatus.map(PpomppuBoardFeedStatus::getLatestParsedArticleId)
|
||||
.orElse(0L);
|
||||
|
||||
log.info("latestArticleId : {}", latestArticleId);
|
||||
return articles.stream()
|
||||
.filter(e -> e.getArticleId().compareTo(latestArticleId) > 0)
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public void save(PpomppuBoardName boardName, List<PpomppuArticle> articles) {
|
||||
public List<PpomppuArticle> save(PpomppuBoardName boardName, List<PpomppuArticle> articles) {
|
||||
Long latestArticleId = articles.stream()
|
||||
.map(PpomppuArticle::getArticleId)
|
||||
.max(Long::compareTo)
|
||||
|
@ -46,14 +50,17 @@ public class PpomppuArticleService {
|
|||
|
||||
// save PpomppuBoardFeedStatus
|
||||
Optional<PpomppuBoardFeedStatus> boardFeedStatus = ppomppuBoardFeedStatusRepository.findByBoardName(boardName);
|
||||
log.info("boardName: {}, isPresent?: {}", boardName, boardFeedStatus.isPresent());
|
||||
boardFeedStatus.ifPresentOrElse(e -> {
|
||||
e.updateArticleId(latestArticleId);
|
||||
ppomppuBoardFeedStatusRepository.save(e);
|
||||
if (latestArticleId.longValue() > 0L) {
|
||||
e.updateArticleId(latestArticleId);
|
||||
ppomppuBoardFeedStatusRepository.save(e);
|
||||
}
|
||||
},
|
||||
() -> ppomppuBoardFeedStatusRepository.save(PpomppuBoardFeedStatus.of(boardName,
|
||||
latestArticleId)));
|
||||
|
||||
// save real articles.
|
||||
ppomppuArticleRepository.saveAll(articles);
|
||||
return ppomppuArticleRepository.saveAll(articles);
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
package com.myoa.engineering.crawl.ppomppu.processor.service;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
|
||||
import com.myoa.engineering.crawl.ppomppu.processor.dto.PpomppuArticleTransformer;
|
||||
import com.myoa.engineering.crawl.ppomppu.processor.dto.PpomppuArticleParser;
|
||||
import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.client.PpomppuBoardFeedRetriever;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
|
||||
import java.util.List;
|
||||
|
@ -30,11 +30,11 @@ public class PpomppuFeedService {
|
|||
|
||||
public Mono<List<PpomppuArticle>> getArticles(PpomppuBoardName boardName) {
|
||||
final Mono<String> html = ppomppuBoardFeedRetriever.getHtml(boardName.getResourcePath());
|
||||
final Mono<Element> tbody = extractTbodyFromHtml(html)
|
||||
.doOnNext(e -> log.info("pre tbody - {}", e.html()));
|
||||
final Mono<Element> tbody = extractTbodyFromHtml(html);
|
||||
// .doOnNext(e -> log.info("pre tbody - {}", e.html()));
|
||||
return extractArticlesFromTbody(tbody).map(this::convertFromElement)
|
||||
.map(e -> e.updateBoardName(boardName))
|
||||
.doOnNext(e -> log.info("parsed Result: {}", e))
|
||||
// .doOnNext(e -> log.info("parsed Result: {}", e))
|
||||
.collectList();
|
||||
}
|
||||
|
||||
|
@ -42,7 +42,7 @@ public class PpomppuFeedService {
|
|||
return html.map(Jsoup::parse)
|
||||
.mapNotNull(e -> e.getElementById("revolution_main_table"))
|
||||
.map(e -> e.getElementsByTag("tbody"))
|
||||
.doOnNext(e -> log.info("tbody - {}", e.html()))
|
||||
// .doOnNext(e -> log.info("tbody - {}", e.html()))
|
||||
.map(e -> e.stream()
|
||||
.findFirst()
|
||||
.orElseThrow(() -> new IndexOutOfBoundsException("no tbody")));
|
||||
|
@ -54,6 +54,6 @@ public class PpomppuFeedService {
|
|||
}
|
||||
|
||||
private PpomppuArticle convertFromElement(Element element) {
|
||||
return PpomppuArticleTransformer.toArticle(element.getElementsByTag("td"));
|
||||
return PpomppuArticleParser.toArticle(element.getElementsByTag("td"));
|
||||
}
|
||||
}
|
||||
|
|
|
@ -3,6 +3,11 @@ spring:
|
|||
activate:
|
||||
on-profile: development
|
||||
import:
|
||||
- "configserver:http://192.168.0.100:11080"
|
||||
- classpath:/development/webclient.yml
|
||||
- classpath:/development/temppassword.yml
|
||||
- classpath:/development/database.yml
|
||||
|
||||
|
||||
server:
|
||||
port: 20081
|
||||
|
||||
# import: optional:configserver:http://localhost:11080 # can be start up even config server was not found.
|
|
@ -0,0 +1,12 @@
|
|||
spring:
|
||||
config:
|
||||
activate:
|
||||
on-profile: local
|
||||
import:
|
||||
- "configserver:http://localhost:20085"
|
||||
- classpath:/local/webclient.yml
|
||||
|
||||
server:
|
||||
port: 20081
|
||||
|
||||
# import: optional:configserver:http://localhost:11080 # can be start up even config server was not found.
|
|
@ -0,0 +1,6 @@
|
|||
spring:
|
||||
config:
|
||||
activate:
|
||||
on-profile: production
|
||||
import:
|
||||
- "configserver:http://192.168.0.100:11080"
|
|
@ -1,14 +1,25 @@
|
|||
spring:
|
||||
application:
|
||||
name: ppn-processor
|
||||
main:
|
||||
allow-bean-definition-overriding: true
|
||||
profiles:
|
||||
active: development
|
||||
active: ${SPRING_ACTIVE_PROFILE:local}
|
||||
group:
|
||||
local: "local,datasource-local"
|
||||
development: "development,datasource-development"
|
||||
production: "production, datasource-production"
|
||||
freemarker:
|
||||
enabled: false
|
||||
|
||||
|
||||
server:
|
||||
port: 20081
|
||||
port: 20080
|
||||
error:
|
||||
whitelabel:
|
||||
enabled: false
|
||||
|
||||
management:
|
||||
endpoints:
|
||||
web:
|
||||
exposure:
|
||||
include: refresh
|
||||
|
|
|
@ -1,5 +1,5 @@
|
|||
webclient:
|
||||
some: test
|
||||
init: true
|
||||
units:
|
||||
- unit-name: processor-api
|
||||
base-url: http://localhost:20081
|
||||
- unit-name: ppn-sender-api
|
||||
base-url: http://localhost:20082
|
|
@ -0,0 +1,5 @@
|
|||
webclient:
|
||||
init: true
|
||||
units:
|
||||
- unit-name: ppn-sender-api
|
||||
base-url: http://localhost:20082
|
|
@ -1,6 +1,10 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<configuration>
|
||||
<springProperty name="DEFAULT_LEVEL_CONFIG" source="log.defaultLevel" />
|
||||
<springProfile name="local">
|
||||
<include resource="logback/logback-development.xml" />
|
||||
<logger name="org.apache.kafka" level="INFO" />
|
||||
</springProfile>
|
||||
<springProfile name="development">
|
||||
<include resource="logback/logback-development.xml" />
|
||||
<logger name="org.apache.kafka" level="INFO" />
|
||||
|
|
|
@ -1,4 +1,5 @@
|
|||
webclient:
|
||||
init: true
|
||||
units:
|
||||
- unit-name: processor-api
|
||||
base-url: http://soundhoundfound-processor:20080
|
||||
- unit-name: ppn-sender-api
|
||||
base-url: http://ppn_sender:20082
|
|
@ -8,6 +8,7 @@ dependencies {
|
|||
// https://projectreactor.io/docs/core/release/reference/#debug-activate
|
||||
implementation 'org.springframework.boot:spring-boot-starter-webflux'
|
||||
implementation 'org.springframework.boot:spring-boot-configuration-processor'
|
||||
implementation 'org.springframework.cloud:spring-cloud-starter-config'
|
||||
implementation 'org.telegram:telegrambots:5.3.0'
|
||||
|
||||
annotationProcessor 'org.springframework.boot:spring-boot-configuration-processor'
|
||||
|
|
|
@ -3,8 +3,10 @@ package com.myoa.engineering.crawl.ppomppu.receiver;
|
|||
import org.springframework.boot.SpringApplication;
|
||||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||
import org.springframework.boot.context.properties.EnableConfigurationProperties;
|
||||
import org.springframework.context.annotation.Import;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.receiver.configuration.properties.TelegramBotProperties;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.webclient.PpomppuNotifierWebClientConfiguration;
|
||||
|
||||
/**
|
||||
* ReceiverApplication
|
||||
|
@ -12,8 +14,9 @@ import com.myoa.engineering.crawl.ppomppu.receiver.configuration.properties.Tele
|
|||
* @since 2021-08-20
|
||||
*
|
||||
*/
|
||||
@Import({ PpomppuNotifierWebClientConfiguration.class})
|
||||
@SpringBootApplication
|
||||
@EnableConfigurationProperties({ TelegramBotProperties.class })
|
||||
// @EnableConfigurationProperties({ TelegramBotProperties.class })
|
||||
public class ReceiverApplication {
|
||||
|
||||
public static void main(String[] args) {
|
||||
|
|
|
@ -9,6 +9,7 @@ import org.telegram.telegrambots.meta.exceptions.TelegramApiException;
|
|||
import org.telegram.telegrambots.updatesreceivers.DefaultBotSession;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.receiver.configuration.properties.TelegramBotProperties;
|
||||
import com.myoa.engineering.crawl.ppomppu.receiver.configuration.properties.TelegramBotProperties.TelegramBotPropertiesUnit;
|
||||
import com.myoa.engineering.crawl.ppomppu.receiver.dispatch.MessageDispatcher;
|
||||
import com.myoa.engineering.crawl.ppomppu.receiver.handler.message.MessageHandler;
|
||||
|
||||
|
@ -21,6 +22,7 @@ import com.myoa.engineering.crawl.ppomppu.receiver.handler.message.MessageHandle
|
|||
@Configuration
|
||||
public class TelegramBotConfiguration {
|
||||
|
||||
private static final String BOT_PROPERTIES_UNIT_NAME = "ppomppu_notify_bot";
|
||||
@Bean
|
||||
public TelegramBotsApi telegramBotsApi(MessageDispatcher messageDispatcher) throws TelegramApiException {
|
||||
TelegramBotsApi api = new TelegramBotsApi(DefaultBotSession.class);
|
||||
|
@ -30,7 +32,8 @@ public class TelegramBotConfiguration {
|
|||
|
||||
@Bean
|
||||
public MessageDispatcher messageDispatcher(List<MessageHandler> messageHandlers,
|
||||
TelegramBotProperties botProperties) {
|
||||
return new MessageDispatcher(messageHandlers, botProperties.getName(), botProperties.getToken());
|
||||
TelegramBotProperties telegramBotProperties) {
|
||||
TelegramBotPropertiesUnit propertiesUnit = telegramBotProperties.find(BOT_PROPERTIES_UNIT_NAME);
|
||||
return new MessageDispatcher(messageHandlers, propertiesUnit.getName(), propertiesUnit.getToken());
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1,9 +1,17 @@
|
|||
package com.myoa.engineering.crawl.ppomppu.receiver.configuration.properties;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
import org.springframework.boot.context.properties.ConstructorBinding;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.Setter;
|
||||
|
||||
/**
|
||||
* TelegramBotProperties
|
||||
|
@ -11,15 +19,26 @@ import lombok.Getter;
|
|||
* @since 2021-09-05
|
||||
*
|
||||
*/
|
||||
@Component
|
||||
@Setter
|
||||
@Getter
|
||||
@ConstructorBinding
|
||||
@ConfigurationProperties(prefix = "telegram.bot")
|
||||
@ConfigurationProperties(prefix = "infra.telegram.bot")
|
||||
public class TelegramBotProperties {
|
||||
private final String name;
|
||||
private final String token;
|
||||
|
||||
public TelegramBotProperties(final String name, final String token) {
|
||||
this.name = name;
|
||||
this.token = token;
|
||||
private List<TelegramBotPropertiesUnit> units = new ArrayList<>();
|
||||
|
||||
@Data
|
||||
public static class TelegramBotPropertiesUnit {
|
||||
private String unitName;
|
||||
private String name;
|
||||
private String token;
|
||||
}
|
||||
|
||||
public TelegramBotPropertiesUnit find(String unitName) {
|
||||
return units.stream()
|
||||
.filter(e -> e.getUnitName().equals(unitName))
|
||||
.findFirst()
|
||||
.orElseThrow(
|
||||
() -> new IllegalArgumentException(this.getClass().getName() + ": unitName Not found. " + unitName));
|
||||
}
|
||||
}
|
||||
|
|
|
@ -0,0 +1,19 @@
|
|||
package com.myoa.engineering.crawl.ppomppu.receiver.dto.constant;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Getter;
|
||||
|
||||
/**
|
||||
* WebClientPropertiesUnitName
|
||||
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
|
||||
* @since 2021-11-18
|
||||
*
|
||||
*/
|
||||
@Getter
|
||||
@AllArgsConstructor
|
||||
public enum WebClientPropertiesUnitName {
|
||||
PPOMPPU_NOTIFIER_PROCESSOR_API("ppn-processor-api"),
|
||||
;
|
||||
|
||||
private String unitName;
|
||||
}
|
|
@ -1,12 +1,15 @@
|
|||
package com.myoa.engineering.crawl.ppomppu.receiver.infrastructure.client;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.core.ParameterizedTypeReference;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.web.reactive.function.client.WebClient;
|
||||
import org.springframework.web.reactive.function.client.WebClientRequestException;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.receiver.dto.constant.WebClientPropertiesUnitName;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebClientFilterFactory;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.webclient.properties.WebClientProperties;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.webclient.properties.WebClientProperties.WebClientPropertiesUnit;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import reactor.core.publisher.Mono;
|
||||
|
@ -18,10 +21,14 @@ public class ProcessorAPIWebClient {
|
|||
|
||||
private final WebClient webClient;
|
||||
|
||||
public ProcessorAPIWebClient(WebClient.Builder webClientBuilder,
|
||||
@Value("${webclient.base-url}") String baseUrl) {
|
||||
this.webClient = webClientBuilder.baseUrl(baseUrl)
|
||||
.build();
|
||||
public ProcessorAPIWebClient(WebClientProperties webClientProperties) {
|
||||
WebClientPropertiesUnit webClientPropertiesUnit =
|
||||
webClientProperties.find(WebClientPropertiesUnitName.PPOMPPU_NOTIFIER_PROCESSOR_API.getUnitName());
|
||||
this.webClient = WebClient.builder()
|
||||
.baseUrl(webClientPropertiesUnit.getBaseUrl())
|
||||
.filter(WebClientFilterFactory.logRequest())
|
||||
.filter(WebClientFilterFactory.logResponse())
|
||||
.build();
|
||||
}
|
||||
|
||||
public Mono<String> emitParseEvent(PpomppuBoardName boardName) {
|
||||
|
|
|
@ -26,15 +26,11 @@ public class ParseEventEmitter {
|
|||
this.processorAPIService = processorAPIService;
|
||||
}
|
||||
|
||||
@Scheduled(fixedRate = 60 * 1000L)
|
||||
public void emitDomesticBoard() {
|
||||
log.info("[emitDomesticBoard] trigger fired!");
|
||||
processorAPIService.emitParseEvent(PpomppuBoardName.PPOMPPU_DOMESTIC_ETC).block();
|
||||
}
|
||||
|
||||
@Scheduled(fixedRate = 300 * 1000L)
|
||||
public void emitOverseaBoard() {
|
||||
log.info("[emitOverseaBoard] trigger fired!");
|
||||
processorAPIService.emitParseEvent(PpomppuBoardName.PPOMPPU_OVERSEA_ETC).block();
|
||||
public void emitBoards() {
|
||||
log.info("[emitDomesticBoard] trigger fired!");
|
||||
for (PpomppuBoardName boardName : PpomppuBoardName.values()) {
|
||||
processorAPIService.emitParseEvent(boardName).block();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -4,4 +4,4 @@ spring:
|
|||
on-profile: development
|
||||
import:
|
||||
- classpath:/development/webclient.yml
|
||||
- classpath:/development/temppassword.yml
|
||||
- "configserver:http://192.168.0.100:11080"
|
|
@ -0,0 +1,7 @@
|
|||
spring:
|
||||
config:
|
||||
activate:
|
||||
on-profile: local
|
||||
import:
|
||||
- classpath:/local/webclient.yml
|
||||
- "configserver:http://localhost:20085"
|
|
@ -3,4 +3,5 @@ spring:
|
|||
activate:
|
||||
on-profile: production
|
||||
import:
|
||||
- classpath:/production/webclient.yml
|
||||
- classpath:/production/webclient.yml
|
||||
- "configserver:http://192.168.0.100:11080"
|
|
@ -1,10 +1,15 @@
|
|||
spring:
|
||||
application:
|
||||
name: ppn-receiver
|
||||
main:
|
||||
allow-bean-definition-overriding: true
|
||||
profiles:
|
||||
active: development
|
||||
active: ${SPRING_ACTIVE_PROFILE:local}
|
||||
freemarker:
|
||||
enabled: false
|
||||
|
||||
server:
|
||||
port: 20080
|
||||
port: 20080
|
||||
error:
|
||||
whitelabel:
|
||||
enabled: false
|
|
@ -1,5 +1,5 @@
|
|||
webclient:
|
||||
base-url: http://localhost:20081
|
||||
init: true
|
||||
units:
|
||||
- unit-name: processor-api
|
||||
- unit-name: ppn-processor-api
|
||||
base-url: http://localhost:20081
|
|
@ -0,0 +1,5 @@
|
|||
webclient:
|
||||
init: true
|
||||
units:
|
||||
- unit-name: ppn-processor-api
|
||||
base-url: http://localhost:20081
|
|
@ -0,0 +1,15 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<configuration>
|
||||
<springProperty name="DEFAULT_LEVEL_CONFIG" source="log.defaultLevel" />
|
||||
<springProfile name="local">
|
||||
<include resource="logback/logback-development.xml" />
|
||||
<logger name="org.apache.kafka" level="INFO" />
|
||||
</springProfile>
|
||||
<springProfile name="development">
|
||||
<include resource="logback/logback-development.xml" />
|
||||
<logger name="org.apache.kafka" level="INFO" />
|
||||
</springProfile>
|
||||
<springProfile name="production">
|
||||
<include resource="logback/logback-production.xml" />
|
||||
</springProfile>
|
||||
</configuration>
|
|
@ -0,0 +1,23 @@
|
|||
<included>
|
||||
<property name="FILE_LOG_PATTERN"
|
||||
value="%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{45}:%L - %msg%n" />
|
||||
<property name="LOG_FILE_BASE" value="lcp-benefit-benefit-api" />
|
||||
<appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
|
||||
<file>${DIRECTORY}/${LOG_FILE_BASE}_log</file>
|
||||
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
|
||||
<fileNamePattern>${DIRECTORY}/${LOG_FILE_BASE}_log.%d{yyyyMMdd}.%i</fileNamePattern>
|
||||
<timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
|
||||
<maxFileSize>1000MB</maxFileSize>
|
||||
</timeBasedFileNamingAndTriggeringPolicy>
|
||||
<maxHistory>60</maxHistory>
|
||||
</rollingPolicy>
|
||||
<encoder>
|
||||
<pattern>${FILE_LOG_PATTERN}</pattern>
|
||||
<immediateFlush>${IMMEDIATE_FLUSH}</immediateFlush>
|
||||
</encoder>
|
||||
</appender>
|
||||
<appender name="ASYNC_FILE" class="ch.qos.logback.classic.AsyncAppender">
|
||||
<queueSize>1024</queueSize>
|
||||
<appender-ref ref="FILE" />
|
||||
</appender>
|
||||
</included>
|
|
@ -0,0 +1,19 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<included>
|
||||
<!-- =========== property BETA ========= -->
|
||||
<property name="DEFAULT_LEVEL" value="${DEFAULT_LEVEL_CONFIG:-INFO}"/>
|
||||
<!--file-->
|
||||
<property name="DIRECTORY" value="/home1/www/logs/supervisor"/>
|
||||
<property name="IMMEDIATE_FLUSH" value="true"/>
|
||||
<!--nelo2-->
|
||||
<property name="NELO2_LEVEL" value="WARN"/>
|
||||
<!-- =========== include appender =========== -->
|
||||
<include resource="org/springframework/boot/logging/logback/defaults.xml"/>
|
||||
<include resource="org/springframework/boot/logging/logback/console-appender.xml"/>
|
||||
<include resource="logback/component/logback-nelo2.xml"/>
|
||||
<include resource="logback/component/logback-datachain.xml"/>
|
||||
<!-- =========== root logger ============== -->
|
||||
<root level="${DEFAULT_LEVEL}">
|
||||
<appender-ref ref="CONSOLE"/>
|
||||
</root>
|
||||
</included>
|
|
@ -0,0 +1,19 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<included>
|
||||
<!-- =========== property RELEASE ========= -->
|
||||
<property name="DEFAULT_LEVEL" value="${DEFAULT_LEVEL_CONFIG:-INFO}"/>
|
||||
<!--file-->
|
||||
<property name="DIRECTORY" value="/home1/www/logs/supervisor"/>
|
||||
<property name="IMMEDIATE_FLUSH" value="true"/>
|
||||
<!--nelo2-->
|
||||
<property name="NELO2_LEVEL" value="WARN"/>
|
||||
<!-- =========== include appender =========== -->
|
||||
<include resource="org/springframework/boot/logging/logback/defaults.xml"/>
|
||||
<include resource="org/springframework/boot/logging/logback/console-appender.xml"/>
|
||||
<include resource="logback/component/logback-nelo2.xml"/>
|
||||
<include resource="logback/component/logback-datachain.xml"/>
|
||||
<!-- =========== root logger ============== -->
|
||||
<root level="${DEFAULT_LEVEL}">
|
||||
<appender-ref ref="CONSOLE"/>
|
||||
</root>
|
||||
</included>
|
|
@ -1,5 +0,0 @@
|
|||
webclient:
|
||||
base-url: http://ppomppu_notifier_processor:20080
|
||||
units:
|
||||
- unit-name: processor-api
|
||||
base-url: http://ppomppu_notifier_processor:20080
|
|
@ -0,0 +1,5 @@
|
|||
webclient:
|
||||
init: true
|
||||
units:
|
||||
- unit-name: ppn-processor-api
|
||||
base-url: http://ppn_processor:20080
|
|
@ -0,0 +1,23 @@
|
|||
dependencies {
|
||||
developmentOnly 'org.springframework.boot:spring-boot-devtools'
|
||||
runtimeOnly 'mysql:mysql-connector-java'
|
||||
compileOnly 'org.projectlombok:lombok'
|
||||
|
||||
implementation project(':support')
|
||||
// https://projectreactor.io/docs/core/release/reference/#debug-activate
|
||||
implementation 'org.springframework.boot:spring-boot-starter-webflux'
|
||||
implementation 'org.springframework.boot:spring-boot-starter-actuator'
|
||||
implementation 'org.springframework.cloud:spring-cloud-starter-config'
|
||||
|
||||
annotationProcessor 'org.springframework.boot:spring-boot-configuration-processor'
|
||||
annotationProcessor 'org.projectlombok:lombok'
|
||||
|
||||
testImplementation 'org.springframework.boot:spring-boot-starter-test'
|
||||
}
|
||||
|
||||
test {
|
||||
useJUnitPlatform()
|
||||
testLogging {
|
||||
events "passed", "skipped", "failed"
|
||||
}
|
||||
}
|
|
@ -1,10 +1,22 @@
|
|||
package com.myoa.engineering.crawl.ppomppu.sender;
|
||||
|
||||
import org.springframework.boot.SpringApplication;
|
||||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||
import org.springframework.context.annotation.Import;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.support.webclient.PpomppuNotifierWebClientConfiguration;
|
||||
|
||||
/**
|
||||
* SenderApplication
|
||||
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
|
||||
* @since 2021-08-20
|
||||
*
|
||||
*/
|
||||
@Import({ PpomppuNotifierWebClientConfiguration.class })
|
||||
@SpringBootApplication
|
||||
public class SenderApplication {
|
||||
|
||||
public static void main(String[] args) {
|
||||
SpringApplication.run(SenderApplication.class, args);
|
||||
}
|
||||
}
|
||||
|
|
|
@ -0,0 +1,34 @@
|
|||
package com.myoa.engineering.crawl.ppomppu.sender.configuration.properties;
|
||||
|
||||
import java.util.List;
|
||||
import lombok.Data;
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
@Getter
|
||||
@Setter
|
||||
@Component
|
||||
@ConfigurationProperties("infra.slack.bot")
|
||||
public class SlackSecretProperties {
|
||||
|
||||
private List<SlackSecretPropertiesUnit> units;
|
||||
|
||||
@Data
|
||||
public static class SlackSecretPropertiesUnit {
|
||||
|
||||
private String botName;
|
||||
private String username;
|
||||
private String iconEmoji;
|
||||
private String channel;
|
||||
private String token;
|
||||
}
|
||||
|
||||
public SlackSecretPropertiesUnit find(String botUnitName) {
|
||||
return units.stream()
|
||||
.filter(e -> e.getBotName().equals(botUnitName))
|
||||
.findFirst()
|
||||
.orElseThrow(() -> new IllegalArgumentException("not found bot unit name : " + botUnitName));
|
||||
}
|
||||
}
|
|
@ -0,0 +1,35 @@
|
|||
package com.myoa.engineering.crawl.ppomppu.sender.controller;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.sender.infrastructure.client.MongeShoppingBotSlackMessageSender;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.dto.APIResponse;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.dto.SimpleMessageDTO;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
import reactor.core.publisher.Mono;
|
||||
|
||||
/**
|
||||
* MessageSenderAPIController
|
||||
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
|
||||
* @since 2021-11-21
|
||||
*
|
||||
*/
|
||||
@Slf4j
|
||||
@RestController
|
||||
@RequestMapping("/api/v1")
|
||||
public class MessageSenderAPIController {
|
||||
|
||||
private final MongeShoppingBotSlackMessageSender sender;
|
||||
|
||||
public MessageSenderAPIController(MongeShoppingBotSlackMessageSender sender) {
|
||||
this.sender = sender;
|
||||
}
|
||||
|
||||
@PostMapping("/messages/sendMessage/messengers/slack")
|
||||
public Mono<APIResponse<SimpleMessageDTO>> sendMessageToSlack(@RequestBody SimpleMessageDTO dto) {
|
||||
return sender.sendMessage(sender.ofMessage(dto.getBody()))
|
||||
.then(Mono.just(APIResponse.success(dto)));
|
||||
}
|
||||
}
|
|
@ -0,0 +1,33 @@
|
|||
package com.myoa.engineering.crawl.ppomppu.sender.controller;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.sender.infrastructure.client.MongeShoppingBotSlackMessageSender;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import reactor.core.publisher.Mono;
|
||||
|
||||
/**
|
||||
* TestAPIController
|
||||
*
|
||||
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
|
||||
* @since 2021-11-15
|
||||
*/
|
||||
@Slf4j
|
||||
@RestController
|
||||
@RequestMapping("/api/v1")
|
||||
public class TestAPIController {
|
||||
|
||||
private final MongeShoppingBotSlackMessageSender sender;
|
||||
|
||||
public TestAPIController(MongeShoppingBotSlackMessageSender sender) {
|
||||
this.sender = sender;
|
||||
}
|
||||
|
||||
@GetMapping("/test")
|
||||
public Mono<String> test() {
|
||||
log.info("received!!!");
|
||||
return sender.sendMessage(sender.ofMessage("testtesttest!!!"));
|
||||
}
|
||||
}
|
|
@ -0,0 +1,12 @@
|
|||
package com.myoa.engineering.crawl.ppomppu.sender.dto;
|
||||
|
||||
import java.io.Serializable;
|
||||
|
||||
/**
|
||||
* MessageDTO
|
||||
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
|
||||
* @since 2021-11-14
|
||||
*
|
||||
*/
|
||||
public interface MessageDTO extends Serializable {
|
||||
}
|
|
@ -0,0 +1,39 @@
|
|||
package com.myoa.engineering.crawl.ppomppu.sender.dto;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
|
||||
import lombok.Builder;
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
/**
|
||||
* SlackMessageDTO
|
||||
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
|
||||
* @since 2021-11-14
|
||||
*
|
||||
*/
|
||||
@Getter
|
||||
@NoArgsConstructor
|
||||
public class SlackMessageDTO implements MessageDTO {
|
||||
|
||||
private final static long serialVersionUID = 4737608709660494713L;
|
||||
|
||||
private String text;
|
||||
private String channel;
|
||||
private String username;
|
||||
|
||||
@JsonProperty("icon_emoji")
|
||||
private String iconEmoji;
|
||||
|
||||
@Builder
|
||||
public SlackMessageDTO(String text, String channel, String username, String iconEmoji) {
|
||||
this.text = text;
|
||||
this.channel = channel;
|
||||
this.username = username;
|
||||
this.iconEmoji = iconEmoji;
|
||||
}
|
||||
|
||||
public void applyText(String text) {
|
||||
this.text = text;
|
||||
}
|
||||
}
|
|
@ -0,0 +1,17 @@
|
|||
package com.myoa.engineering.crawl.ppomppu.sender.infrastructure.client;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.sender.dto.MessageDTO;
|
||||
|
||||
import reactor.core.publisher.Mono;
|
||||
|
||||
/**
|
||||
* MessageSender
|
||||
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
|
||||
* @since 2021-11-14
|
||||
*
|
||||
*/
|
||||
public interface MessageSender<T extends MessageDTO> {
|
||||
|
||||
Mono<String> sendMessage(T message);
|
||||
|
||||
}
|
|
@ -0,0 +1,38 @@
|
|||
package com.myoa.engineering.crawl.ppomppu.sender.infrastructure.client;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.sender.configuration.properties.SlackSecretProperties;
|
||||
import com.myoa.engineering.crawl.ppomppu.sender.configuration.properties.SlackSecretProperties.SlackSecretPropertiesUnit;
|
||||
import com.myoa.engineering.crawl.ppomppu.sender.dto.SlackMessageDTO;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
@Slf4j
|
||||
@Component
|
||||
public class MongeShoppingBotSlackMessageSender extends SlackMessageSender {
|
||||
|
||||
private static final String SLACK_SECRET_UNIT_NAME = "monge_shopping_bot";
|
||||
|
||||
private final SlackSecretPropertiesUnit slackProperties;
|
||||
|
||||
public MongeShoppingBotSlackMessageSender(SlackSecretProperties slackSecretProperties) {
|
||||
super(slackSecretProperties.find(SLACK_SECRET_UNIT_NAME).getToken());
|
||||
this.slackProperties = slackSecretProperties.find(SLACK_SECRET_UNIT_NAME);
|
||||
}
|
||||
|
||||
public SlackMessageDTO ofMessageTemplate() {
|
||||
return SlackMessageDTO.builder()
|
||||
.channel(slackProperties.getChannel())
|
||||
.iconEmoji(slackProperties.getIconEmoji())
|
||||
.username(slackProperties.getUsername())
|
||||
.build();
|
||||
}
|
||||
|
||||
public SlackMessageDTO ofMessage(String text) {
|
||||
return SlackMessageDTO.builder()
|
||||
.channel(slackProperties.getChannel())
|
||||
.iconEmoji(slackProperties.getIconEmoji())
|
||||
.username(slackProperties.getUsername())
|
||||
.text(text)
|
||||
.build();
|
||||
}
|
||||
}
|
|
@ -0,0 +1,53 @@
|
|||
package com.myoa.engineering.crawl.ppomppu.sender.infrastructure.client;
|
||||
|
||||
import org.springframework.http.HttpHeaders;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.web.reactive.function.client.WebClient;
|
||||
import org.springframework.web.reactive.function.client.WebClientRequestException;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.sender.dto.SlackMessageDTO;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebClientFilterFactory;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebFluxExchangeStragiesFactory;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import reactor.core.publisher.Mono;
|
||||
import reactor.core.scheduler.Schedulers;
|
||||
|
||||
/**
|
||||
* SlackMessageSender
|
||||
*
|
||||
* @author Shin Woo-jin (woozu.shin@kakaoent.com)
|
||||
* @since 2021-09-08
|
||||
*/
|
||||
@Slf4j
|
||||
public class SlackMessageSender implements MessageSender<SlackMessageDTO> {
|
||||
|
||||
private static final String SLACK_API_URL = "https://slack.com/api";
|
||||
|
||||
private final WebClient webClient;
|
||||
|
||||
public SlackMessageSender(String apiSecret) {
|
||||
this.webClient = WebClient.builder()
|
||||
.baseUrl(SLACK_API_URL)
|
||||
.defaultHeader(HttpHeaders.AUTHORIZATION, "Bearer " + apiSecret)
|
||||
.exchangeStrategies(WebFluxExchangeStragiesFactory.ofDefault())
|
||||
.defaultHeader(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_UTF8_VALUE)
|
||||
.defaultHeader(HttpHeaders.ACCEPT_CHARSET, "UTF-8")
|
||||
.filter(WebClientFilterFactory.logResponse())
|
||||
.build();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Mono<String> sendMessage(SlackMessageDTO message) {
|
||||
return webClient.post()
|
||||
.uri("/chat.postMessage")
|
||||
.bodyValue(message)
|
||||
.exchangeToMono(e -> e.bodyToMono(String.class))
|
||||
.publishOn(Schedulers.boundedElastic())
|
||||
.onErrorResume(WebClientRequestException.class, t -> {
|
||||
log.info("Exception occured, ignoring. : {}", t.getClass().getSimpleName());
|
||||
return Mono.empty();
|
||||
});
|
||||
}
|
||||
|
||||
}
|
|
@ -0,0 +1,10 @@
|
|||
spring:
|
||||
config:
|
||||
activate:
|
||||
on-profile: development
|
||||
import:
|
||||
- "configserver:http://192.168.0.100:11080"
|
||||
# import: optional:configserver:http://localhost:11080 # can be start up even config server was not found.
|
||||
|
||||
server:
|
||||
port: 20082
|
|
@ -0,0 +1,10 @@
|
|||
spring:
|
||||
config:
|
||||
activate:
|
||||
on-profile: local
|
||||
import:
|
||||
- "configserver:http://localhost:20085"
|
||||
# import: optional:configserver:http://localhost:11080 # can be start up even config server was not found.
|
||||
|
||||
server:
|
||||
port: 20082
|
|
@ -0,0 +1,6 @@
|
|||
spring:
|
||||
config:
|
||||
activate:
|
||||
on-profile: production
|
||||
import:
|
||||
- "configserver:http://192.168.0.100:11080"
|
|
@ -0,0 +1,25 @@
|
|||
spring:
|
||||
application:
|
||||
name: ppn-sender
|
||||
main:
|
||||
allow-bean-definition-overriding: true
|
||||
profiles:
|
||||
active: ${SPRING_ACTIVE_PROFILE:local}
|
||||
group:
|
||||
local: local, slackapi-local
|
||||
development: development, slackapi-development
|
||||
production: production, slackapi-production
|
||||
freemarker:
|
||||
enabled: false
|
||||
|
||||
server:
|
||||
port: 20080
|
||||
error:
|
||||
whitelabel:
|
||||
enabled: false
|
||||
|
||||
management:
|
||||
endpoints:
|
||||
web:
|
||||
exposure:
|
||||
include: refresh, env
|
|
@ -0,0 +1,15 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<configuration>
|
||||
<springProfile name="local">
|
||||
<include resource="logback/logback-development.xml" />
|
||||
<logger name="org.apache.kafka" level="INFO" />
|
||||
</springProfile>
|
||||
<springProperty name="DEFAULT_LEVEL_CONFIG" source="log.defaultLevel" />
|
||||
<springProfile name="development">
|
||||
<include resource="logback/logback-development.xml" />
|
||||
<logger name="org.apache.kafka" level="INFO" />
|
||||
</springProfile>
|
||||
<springProfile name="production">
|
||||
<include resource="logback/logback-production.xml" />
|
||||
</springProfile>
|
||||
</configuration>
|
|
@ -0,0 +1,23 @@
|
|||
<included>
|
||||
<property name="FILE_LOG_PATTERN"
|
||||
value="%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{45}:%L - %msg%n" />
|
||||
<property name="LOG_FILE_BASE" value="lcp-benefit-benefit-api" />
|
||||
<appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
|
||||
<file>${DIRECTORY}/${LOG_FILE_BASE}_log</file>
|
||||
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
|
||||
<fileNamePattern>${DIRECTORY}/${LOG_FILE_BASE}_log.%d{yyyyMMdd}.%i</fileNamePattern>
|
||||
<timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
|
||||
<maxFileSize>1000MB</maxFileSize>
|
||||
</timeBasedFileNamingAndTriggeringPolicy>
|
||||
<maxHistory>60</maxHistory>
|
||||
</rollingPolicy>
|
||||
<encoder>
|
||||
<pattern>${FILE_LOG_PATTERN}</pattern>
|
||||
<immediateFlush>${IMMEDIATE_FLUSH}</immediateFlush>
|
||||
</encoder>
|
||||
</appender>
|
||||
<appender name="ASYNC_FILE" class="ch.qos.logback.classic.AsyncAppender">
|
||||
<queueSize>1024</queueSize>
|
||||
<appender-ref ref="FILE" />
|
||||
</appender>
|
||||
</included>
|
|
@ -0,0 +1,19 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<included>
|
||||
<!-- =========== property BETA ========= -->
|
||||
<property name="DEFAULT_LEVEL" value="${DEFAULT_LEVEL_CONFIG:-INFO}"/>
|
||||
<!--file-->
|
||||
<property name="DIRECTORY" value="/home1/www/logs/supervisor"/>
|
||||
<property name="IMMEDIATE_FLUSH" value="true"/>
|
||||
<!--nelo2-->
|
||||
<property name="NELO2_LEVEL" value="WARN"/>
|
||||
<!-- =========== include appender =========== -->
|
||||
<include resource="org/springframework/boot/logging/logback/defaults.xml"/>
|
||||
<include resource="org/springframework/boot/logging/logback/console-appender.xml"/>
|
||||
<include resource="logback/component/logback-nelo2.xml"/>
|
||||
<include resource="logback/component/logback-datachain.xml"/>
|
||||
<!-- =========== root logger ============== -->
|
||||
<root level="${DEFAULT_LEVEL}">
|
||||
<appender-ref ref="CONSOLE"/>
|
||||
</root>
|
||||
</included>
|
|
@ -0,0 +1,19 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<included>
|
||||
<!-- =========== property RELEASE ========= -->
|
||||
<property name="DEFAULT_LEVEL" value="${DEFAULT_LEVEL_CONFIG:-INFO}"/>
|
||||
<!--file-->
|
||||
<property name="DIRECTORY" value="/home1/www/logs/supervisor"/>
|
||||
<property name="IMMEDIATE_FLUSH" value="true"/>
|
||||
<!--nelo2-->
|
||||
<property name="NELO2_LEVEL" value="WARN"/>
|
||||
<!-- =========== include appender =========== -->
|
||||
<include resource="org/springframework/boot/logging/logback/defaults.xml"/>
|
||||
<include resource="org/springframework/boot/logging/logback/console-appender.xml"/>
|
||||
<include resource="logback/component/logback-nelo2.xml"/>
|
||||
<include resource="logback/component/logback-datachain.xml"/>
|
||||
<!-- =========== root logger ============== -->
|
||||
<root level="${DEFAULT_LEVEL}">
|
||||
<appender-ref ref="CONSOLE"/>
|
||||
</root>
|
||||
</included>
|
|
@ -1,11 +1,9 @@
|
|||
dependencies {
|
||||
runtimeOnly 'mysql:mysql-connector-java'
|
||||
compileOnly 'org.projectlombok:lombok'
|
||||
|
||||
// https://projectreactor.io/docs/core/release/reference/#debug-activate
|
||||
|
||||
annotationProcessor 'org.springframework.boot:spring-boot-configuration-processor'
|
||||
annotationProcessor 'org.projectlombok:lombok'
|
||||
implementation 'org.springframework.boot:spring-boot-starter-webflux'
|
||||
|
||||
}
|
||||
|
||||
|
|
|
@ -0,0 +1,31 @@
|
|||
{
|
||||
"blocks": [
|
||||
{
|
||||
"type": "section",
|
||||
"text": {
|
||||
"type": "mrkdwn",
|
||||
"text": "기타)) `[쿠팡] 타푸코 이중 진공 스텐 세라믹 코팅 텀블러 (화이트만 11,130원/로켓와우무료) 4 [기타]`<https://www.ppomppu.co.kr/zboardview.php?id=ppomppu&page=1&divpage=69&category=1&&no=404126|LINK>"
|
||||
},
|
||||
"accessory": {
|
||||
"type": "image",
|
||||
"image_url": "cdn.ppomppu.co.kr/zboard/data3/2021/1121/m_20211121184835_zfyahnow.png",
|
||||
"alt_text": "alt text for image"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "divider"
|
||||
},
|
||||
{
|
||||
"type": "section",
|
||||
"text": {
|
||||
"type": "mrkdwn",
|
||||
"text": "기타)) `[롯데온] 1매 149원 올국산 KF94새부리마스크 대형 200매 (29,920원/무료배송) 58 [기타]`<https://www.ppomppu.co.kr/zboardview.php?id=ppomppu&page=1&divpage=69&category=1&&no=404113|LINK>"
|
||||
},
|
||||
"accessory": {
|
||||
"type": "image",
|
||||
"image_url": "https://s3-media3.fl.yelpcdn.com/bphoto/c7ed05m9lC2EmA3Aruue7A/o.jpg",
|
||||
"alt_text": "alt text for image"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
|
@ -0,0 +1,37 @@
|
|||
package com.myoa.engineering.crawl.ppomppu.support.dto;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.time.Instant;
|
||||
|
||||
import lombok.Builder;
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
/**
|
||||
* SimpleMessageDTO
|
||||
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
|
||||
* @since 2021-11-21
|
||||
*
|
||||
*/
|
||||
@Getter
|
||||
@NoArgsConstructor
|
||||
public class SimpleMessageDTO implements Serializable {
|
||||
|
||||
private static final long serialVersionUID = 2203955567672404428L;
|
||||
|
||||
private String title;
|
||||
private String body;
|
||||
private String url;
|
||||
private Instant publishedAt;
|
||||
private Instant requestedAt;
|
||||
|
||||
@Builder
|
||||
public SimpleMessageDTO(String title, String body, String url, Instant publishedAt, Instant requestedAt) {
|
||||
this.title = title;
|
||||
this.body = body;
|
||||
this.url = url;
|
||||
this.publishedAt = publishedAt;
|
||||
this.requestedAt = requestedAt;
|
||||
}
|
||||
|
||||
}
|
|
@ -12,40 +12,40 @@ import lombok.Getter;
|
|||
@Getter
|
||||
@AllArgsConstructor
|
||||
public enum PpomppuBoardName {
|
||||
PPOMPPU_DOMESTIC_ALL("zboard/zboard.php?id=ppomppu", "전체", true),
|
||||
PPOMPPU_DOMESTIC_ETC("zboard/zboard.php?id=ppomppu&category=1", "기타", true),
|
||||
PPOMPPU_DOMESTIC_COMPUTER("zboard/zboard.php?id=ppomppu&category=4", "컴퓨터", true),
|
||||
PPOMPPU_DOMESTIC_DIGITAL("zboard/zboard.php?id=ppomppu&category=5", "디지털", true),
|
||||
PPOMPPU_DOMESTIC_FOOD("zboard/zboard.php?id=ppomppu&category=6", "식품/건강", true),
|
||||
PPOMPPU_DOMESTIC_BOOK("zboard/zboard.php?id=ppomppu&category=8", "서적", true),
|
||||
PPOMPPU_DOMESTIC_APPLIANCES("zboard/zboard.php?id=ppomppu&category=9", "가전/가구", true),
|
||||
PPOMPPU_DOMESTIC_PARENTING("zboard/zboard.php?id=ppomppu&category=10", "육아", true),
|
||||
PPOMPPU_DOMESTIC_GIFTCARD("zboard/zboard.php?id=ppomppu&category=11", "상품권", true),
|
||||
PPOMPPU_DOMESTIC_CLOTHES("zboard/zboard.php?id=ppomppu&category=12", "의류/잡화", true),
|
||||
PPOMPPU_DOMESTIC_COSMETIC("zboard/zboard.php?id=ppomppu&category=13", "화장품", true),
|
||||
PPOMPPU_DOMESTIC_OUTDOOR("zboard/zboard.php?id=ppomppu&category=15", "등산/캠핑", true),
|
||||
PPOMPPU_OVERSEA_ALL("zboard/zboard.php?id=ppomppu4", "전체", true),
|
||||
PPOMPPU_OVERSEA_ETC("zboard/zboard.php?id=ppomppu4&category=1", "기타", true),
|
||||
PPOMPPU_OVERSEA_APPLIANCES("zboard/zboard.php?id=ppomppu4&category=7", "가전", true),
|
||||
PPOMPPU_OVERSEA_TVAV("zboard/zboard.php?id=ppomppu4&category=8", "TV/영상", true),
|
||||
PPOMPPU_OVERSEA_COMPUTER("zboard/zboard.php?id=ppomppu4&category=3", "컴퓨터", true),
|
||||
PPOMPPU_OVERSEA_DIGITAL("zboard/zboard.php?id=ppomppu4&category=4", "디지털", true),
|
||||
PPOMPPU_OVERSEA_MOBILEACCESSORY("zboard/zboard.php?id=ppomppu4&category=9", "액세서리", true),
|
||||
PPOMPPU_OVERSEA_CLOTHES("zboard/zboard.php?id=ppomppu4&category=5", "의류/잡화", true),
|
||||
PPOMPPU_OVERSEA_WATCH("zboard/zboard.php?id=ppomppu4&category=2", "시계", true),
|
||||
PPOMPPU_OVERSEA_SHOES("zboard/zboard.php?id=ppomppu4&category=11", "신발", true),
|
||||
PPOMPPU_OVERSEA_FOOD("zboard/zboard.php?id=ppomppu4&category=10", "식품/건강", true),
|
||||
PPOMPPU_OVERSEA_PARENTING("zboard/zboard.php?id=ppomppu4&category=6", "육아", true),
|
||||
PPOMPPU_DOMESTIC_ALL("/zboard/zboard.php?id=ppomppu", "전체", false),
|
||||
PPOMPPU_DOMESTIC_ETC("/zboard/zboard.php?id=ppomppu&category=1", "기타", true),
|
||||
PPOMPPU_DOMESTIC_COMPUTER("/zboard/zboard.php?id=ppomppu&category=4", "컴퓨터", true),
|
||||
PPOMPPU_DOMESTIC_DIGITAL("/zboard/zboard.php?id=ppomppu&category=5", "디지털", true),
|
||||
PPOMPPU_DOMESTIC_FOOD("/zboard/zboard.php?id=ppomppu&category=6", "식품/건강", true),
|
||||
PPOMPPU_DOMESTIC_BOOK("/zboard/zboard.php?id=ppomppu&category=8", "서적", true),
|
||||
PPOMPPU_DOMESTIC_APPLIANCES("/zboard/zboard.php?id=ppomppu&category=9", "가전/가구", true),
|
||||
PPOMPPU_DOMESTIC_PARENTING("/zboard/zboard.php?id=ppomppu&category=10", "육아", true),
|
||||
PPOMPPU_DOMESTIC_GIFTCARD("/zboard/zboard.php?id=ppomppu&category=11", "상품권", true),
|
||||
PPOMPPU_DOMESTIC_CLOTHES("/zboard/zboard.php?id=ppomppu&category=12", "의류/잡화", true),
|
||||
PPOMPPU_DOMESTIC_COSMETIC("/zboard/zboard.php?id=ppomppu&category=13", "화장품", true),
|
||||
PPOMPPU_DOMESTIC_OUTDOOR("/zboard/zboard.php?id=ppomppu&category=15", "등산/캠핑", true),
|
||||
PPOMPPU_OVERSEA_ALL("/zboard/zboard.php?id=ppomppu4", "전체", false),
|
||||
PPOMPPU_OVERSEA_ETC("/zboard/zboard.php?id=ppomppu4&category=1", "기타", true),
|
||||
PPOMPPU_OVERSEA_APPLIANCES("/zboard/zboard.php?id=ppomppu4&category=7", "가전", true),
|
||||
PPOMPPU_OVERSEA_TVAV("/zboard/zboard.php?id=ppomppu4&category=8", "TV/영상", true),
|
||||
PPOMPPU_OVERSEA_COMPUTER("/zboard/zboard.php?id=ppomppu4&category=3", "컴퓨터", true),
|
||||
PPOMPPU_OVERSEA_DIGITAL("/zboard/zboard.php?id=ppomppu4&category=4", "디지털", true),
|
||||
PPOMPPU_OVERSEA_MOBILEACCESSORY("/zboard/zboard.php?id=ppomppu4&category=9", "액세서리", true),
|
||||
PPOMPPU_OVERSEA_CLOTHES("/zboard/zboard.php?id=ppomppu4&category=5", "의류/잡화", true),
|
||||
PPOMPPU_OVERSEA_WATCH("/zboard/zboard.php?id=ppomppu4&category=2", "시계", true),
|
||||
PPOMPPU_OVERSEA_SHOES("/zboard/zboard.php?id=ppomppu4&category=11", "신발", true),
|
||||
PPOMPPU_OVERSEA_FOOD("/zboard/zboard.php?id=ppomppu4&category=10", "식품/건강", true),
|
||||
PPOMPPU_OVERSEA_PARENTING("/zboard/zboard.php?id=ppomppu4&category=6", "육아", true),
|
||||
;
|
||||
|
||||
private String resourcePath;
|
||||
private String menuName;
|
||||
private boolean crawlWithDefaultTimer;
|
||||
|
||||
public static final String PPOMPPU_URL = "https://www.ppomppu.co.kr/";
|
||||
public static final String PPOMPPU_URL = "https://www.ppomppu.co.kr";
|
||||
|
||||
public static String ofViewPageUrl(String articleUrl) {
|
||||
return PPOMPPU_URL + "zboard/" + articleUrl;
|
||||
return PPOMPPU_URL + "/zboard/" + articleUrl;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
package com.myoa.engineering.crawl.ppomppu.processor.util;
|
||||
package com.myoa.engineering.crawl.ppomppu.support.util;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonGenerator;
|
||||
import com.fasterxml.jackson.core.JsonParser.Feature;
|
|
@ -0,0 +1,19 @@
|
|||
package com.myoa.engineering.crawl.ppomppu.support.webclient;
|
||||
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
import org.springframework.context.annotation.ComponentScan;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
|
||||
/**
|
||||
* PpomppuNotifierWebClientConfiguration
|
||||
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
|
||||
* @since 2021-11-18
|
||||
*
|
||||
*/
|
||||
@Configuration
|
||||
@ConditionalOnProperty(value = "webclient.init", havingValue = "true")
|
||||
@ComponentScan(basePackageClasses = WebClientBaseScan.class)
|
||||
public class PpomppuNotifierWebClientConfiguration {
|
||||
|
||||
|
||||
}
|
|
@ -1,4 +1,4 @@
|
|||
package com.myoa.engineering.crawl.ppomppu.processor.configuration.factory;
|
||||
package com.myoa.engineering.crawl.ppomppu.support.webclient.factory;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.web.reactive.function.client.ClientRequest;
|
||||
|
@ -27,8 +27,8 @@ public final class WebClientFilterFactory {
|
|||
|
||||
private static Mono<ClientRequest> writeRequest(ClientRequest clientRequest) {
|
||||
try {
|
||||
log.info("[WEBCLIENT REQUEST] uri : {} method : {} headers : {}",
|
||||
clientRequest.url(), clientRequest.method(), clientRequest.headers());
|
||||
log.info("[WEBCLIENT REQUEST] uri : {} method : {} headers : {}, body: {}",
|
||||
clientRequest.url(), clientRequest.method(), clientRequest.headers(), clientRequest.body());
|
||||
} catch (Exception e) {
|
||||
log.error("[WEBCLIENT REQUEST] write request failed", e);
|
||||
}
|
||||
|
@ -37,7 +37,7 @@ public final class WebClientFilterFactory {
|
|||
|
||||
private static Mono<ClientResponse> writeResponse(ClientResponse clientResponse) {
|
||||
try {
|
||||
log.info("[WEBCLIENT REQUEST] statusCode : {} headers : {}",
|
||||
log.info("[WEBCLIENT RESPONSE] statusCode : {} headers : {}",
|
||||
clientResponse.rawStatusCode(), clientResponse.headers().asHttpHeaders());
|
||||
} catch (Exception e) {
|
||||
log.error("[WEBCLIENT RESPONSE] write response failed", e);
|
|
@ -1,7 +1,8 @@
|
|||
package com.myoa.engineering.crawl.ppomppu.processor.configuration.factory;
|
||||
package com.myoa.engineering.crawl.ppomppu.support.webclient.factory;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.myoa.engineering.crawl.ppomppu.processor.util.ObjectMapperFactory;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.util.ObjectMapperFactory;
|
||||
|
||||
import org.springframework.http.codec.json.Jackson2JsonDecoder;
|
||||
import org.springframework.http.codec.json.Jackson2JsonEncoder;
|
||||
import org.springframework.util.MimeTypeUtils;
|
|
@ -0,0 +1,44 @@
|
|||
package com.myoa.engineering.crawl.ppomppu.support.webclient.properties;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
import org.springframework.lang.NonNull;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import lombok.Data;
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.Setter;
|
||||
|
||||
/**
|
||||
* WebClientPropertiesUnit
|
||||
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
|
||||
* @since 2021-11-18
|
||||
*
|
||||
*/
|
||||
@NoArgsConstructor
|
||||
@Setter
|
||||
@Getter
|
||||
@Component
|
||||
@ConfigurationProperties(prefix = "webclient")
|
||||
public class WebClientProperties {
|
||||
|
||||
private List<WebClientPropertiesUnit> units = new ArrayList<>();
|
||||
|
||||
@Data
|
||||
public static class WebClientPropertiesUnit {
|
||||
private String unitName;
|
||||
private String baseUrl;
|
||||
// TODO headers
|
||||
}
|
||||
|
||||
public WebClientPropertiesUnit find(@NonNull String unitName) {
|
||||
return units.stream()
|
||||
.filter(x -> x.getUnitName().equals(unitName))
|
||||
.findFirst()
|
||||
.orElseThrow(
|
||||
() -> new IllegalArgumentException("Not found properties unit. unitName : " + unitName));
|
||||
}
|
||||
}
|
Loading…
Reference in New Issue