MyoaEngineering
/
ShoppingCrawler
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request '[PPN-211113] Implement common sender component' (#12) from feature/PPN-211113 into develop 

Reviewed-on: OutworldDestroyer/PpomppuNotifier#12
This commit is contained in:
nthfuncx 2021-11-21 22:39:22 +09:00
parent 0afa52aa53 a0c3962e0d
commit 26520fba79
70 changed files with 1131 additions and 181 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
build.gradle
View File

@ -32,6 +32,17 @@ allprojects {
}
}
ext {
set('springCloudVersion', "2020.0.4")
}
dependencyManagement {
imports {
mavenBom "org.springframework.cloud:spring-cloud-dependencies:${springCloudVersion}"
}
}
/* task initSourceFolders {
sourceSets*.java.srcDirs*.each {
if( !it.exists() ) {

3
processor/build.gradle
View File

@ -1,3 +1,4 @@
dependencies {
developmentOnly 'org.springframework.boot:spring-boot-devtools'
runtimeOnly 'com.h2database:h2'
@ -9,6 +10,8 @@ dependencies {
implementation 'org.springframework.boot:spring-boot-starter-webflux'
implementation 'org.springframework.boot:spring-boot-starter-data-jpa'
implementation 'org.springframework.boot:spring-boot-starter-jdbc'
implementation 'org.springframework.cloud:spring-cloud-starter-config'
implementation 'org.springframework.boot:spring-boot-starter-actuator'
implementation 'com.rometools:rome:1.16.0'
implementation 'org.jsoup:jsoup:1.14.2'
implementation 'com.h2database:h2:1.4.200'

4
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/ProcessorApplication.java
View File

@ -2,6 +2,9 @@ package com.myoa.engineering.crawl.ppomppu.processor;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.context.annotation.Import;
import com.myoa.engineering.crawl.ppomppu.support.webclient.PpomppuNotifierWebClientConfiguration;
/**
* ProcessorApplication
@ -9,6 +12,7 @@ import org.springframework.boot.autoconfigure.SpringBootApplication;
* @since 2021-08-20
*
*/
@Import({ PpomppuNotifierWebClientConfiguration.class })
@SpringBootApplication
public class ProcessorApplication {

2
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/H2ConsoleConfiguration.java
View File

@ -11,7 +11,7 @@ import org.springframework.context.event.ContextRefreshedEvent;
import org.springframework.context.event.EventListener;
@Slf4j
@Profile("development")
@Profile({"datasource-local", "datasource-development"})
@Configuration
public class H2ConsoleConfiguration {

7
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/PpomppuDatasourceConfiguration.java
View File

@ -32,7 +32,7 @@ import org.springframework.transaction.PlatformTransactionManager;
)
public class PpomppuDatasourceConfiguration {
private static final String DATA_SOURCE_UNIT_NAME = "ppomppu";
private static final String DATA_SOURCE_UNIT_NAME = "ppn_mysql";
private final DatasourceProperties dataSourceProeprties;
private final HikariProperties hikariProperties;
@ -52,8 +52,9 @@ public class PpomppuDatasourceConfiguration {
final HikariConfig hikariConfig = new HikariConfig();
hikariConfig.setJdbcUrl(dataSourcePropertiesUnit.toCompletedJdbcUrl());
hikariConfig.setUsername("sa");
hikariConfig.setPassword("sa");
hikariConfig.setDriverClassName(dataSourcePropertiesUnit.getDriverClassName());
hikariConfig.setUsername(dataSourcePropertiesUnit.getUsername());
hikariConfig.setPassword(dataSourcePropertiesUnit.getPassword());
hikariConfig.setAutoCommit(hikariProperties.getAutoCommit());
hikariConfig.setMaximumPoolSize(hikariProperties.getMaximumPoolSize());
hikariConfig.setMinimumIdle(hikariProperties.getMinimumIdle());

16
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/properties/DatasourceProperties.java
View File

@ -2,6 +2,7 @@ package com.myoa.engineering.crawl.ppomppu.processor.configuration.properties;
import com.myoa.engineering.crawl.ppomppu.support.util.ObjectUtil;
import java.util.List;
import lombok.Getter;
import lombok.Setter;
import org.springframework.boot.context.properties.ConfigurationProperties;
@ -10,7 +11,7 @@ import org.springframework.stereotype.Component;
@Component
@Setter
@Getter
@ConfigurationProperties(prefix = "datasource")
@ConfigurationProperties(prefix = "infra.database")
public class DatasourceProperties {
private List<DataSourcePropertiesUnit> units;
@ -22,14 +23,17 @@ public class DatasourceProperties {
private String unitName;
private String schemaName;
private String connectionParameters;
private String dbConnectionUrl;
private Boolean simpleConnectionUrl;
private String datasourceUrl;
private Boolean isSimpleConnectionUrl;
private String username;
private String password;
private String driverClassName;
public String toCompletedJdbcUrl() {
if (ObjectUtil.isEmpty(simpleConnectionUrl) || simpleConnectionUrl == false) {
return String.format("%s/%s?%s", dbConnectionUrl, schemaName, connectionParameters);
if (ObjectUtil.isEmpty(isSimpleConnectionUrl) || isSimpleConnectionUrl == false) {
return String.format("%s/%s?%s", datasourceUrl, schemaName, connectionParameters);
}
return dbConnectionUrl;
return datasourceUrl;
}
}

3
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/properties/HibernateProperties.java
View File

@ -3,6 +3,9 @@ package com.myoa.engineering.crawl.ppomppu.processor.configuration.properties;
import java.util.List;
import lombok.Getter;
import lombok.Setter;
import org.hibernate.dialect.MySQL8Dialect;
import org.hibernate.dialect.MySQLDialect;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;

34
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/controller/CrawlAPIController.java
View File

@ -1,17 +1,19 @@
package com.myoa.engineering.crawl.ppomppu.processor.controller;
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
import com.myoa.engineering.crawl.ppomppu.processor.dto.FeedParsedResult;
import com.myoa.engineering.crawl.ppomppu.processor.service.PpomppuArticleService;
import com.myoa.engineering.crawl.ppomppu.processor.service.PpomppuFeedService;
import com.myoa.engineering.crawl.ppomppu.support.dto.APIResponse;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import java.util.List;
import lombok.extern.slf4j.Slf4j;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import com.myoa.engineering.crawl.ppomppu.processor.dto.FeedParsedResult;
import com.myoa.engineering.crawl.ppomppu.processor.service.MessageSenderService;
import com.myoa.engineering.crawl.ppomppu.processor.service.PpomppuArticleService;
import com.myoa.engineering.crawl.ppomppu.processor.service.PpomppuFeedService;
import com.myoa.engineering.crawl.ppomppu.support.dto.APIResponse;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import lombok.extern.slf4j.Slf4j;
import reactor.core.publisher.Mono;
/**
@ -27,22 +29,28 @@ public class CrawlAPIController {
private final PpomppuFeedService ppomppuRSSFeedService;
private final PpomppuArticleService ppomppuArticleService;
private final MessageSenderService messageSenderService;
public CrawlAPIController(PpomppuFeedService ppomppuRSSFeedService,
PpomppuArticleService ppomppuArticleService) {
PpomppuArticleService ppomppuArticleService,
MessageSenderService messageSenderService) {
this.ppomppuRSSFeedService = ppomppuRSSFeedService;
this.ppomppuArticleService = ppomppuArticleService;
this.messageSenderService = messageSenderService;
}
@PostMapping("/boards/{boardName}")
public Mono<APIResponse<FeedParsedResult>> crawlBoard(@PathVariable("boardName") PpomppuBoardName boardName) {
log.info("got request... {}", boardName);
FeedParsedResult result = FeedParsedResult.of(boardName);
Mono<List<PpomppuArticle>> articles = ppomppuRSSFeedService.getArticles(boardName)
.doOnNext(e -> ppomppuArticleService.filterOnlyNewArticles(boardName, e))
.doOnNext(e -> ppomppuArticleService.save(boardName, e));
return articles.then(Mono.just(APIResponse.success(result.done())));
Mono<String> publishedMessages =
ppomppuRSSFeedService.getArticles(boardName)
.map(e -> ppomppuArticleService.filterOnlyNewArticles(boardName, e))
.map(e -> ppomppuArticleService.save(boardName, e))
.flatMap(messageSenderService::sendMessageToSlack);
return publishedMessages.then(Mono.just(APIResponse.success(result.done())));
}
}

79
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/PpomppuArticleParser.java Normal file
View File

@ -0,0 +1,79 @@
package com.myoa.engineering.crawl.ppomppu.processor.dto;
import java.time.Instant;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
/**
* PpomppuArticleTransformer
*
* @author Shin Woo-jin (woozu.shin@kakaoent.com)
* @since 2021-09-08
*/
public final class PpomppuArticleParser {
private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yy.MM.dd HH:mm:ss")
.withZone(ZoneId.of("Asia/Seoul"));
private PpomppuArticleParser() {}
public static PpomppuArticle toArticle(Elements articleElement) {
final long articleId = PpomppuArticleParser.parseArticleId(articleElement.get(0));
final String title = PpomppuArticleParser.parseTitle(articleElement.get(2));
final String articleUrl = PpomppuArticleParser.parseArticleUrl(articleElement.get(2));
final int recommended = PpomppuArticleParser.parseRecommended(articleElement.get(6));
final int hit = PpomppuArticleParser.parseHit(articleElement.get(7));
final Instant registeredAt = PpomppuArticleParser.parseRegisteredAt(articleElement.get(5));
return PpomppuArticle.builder()
.articleId(articleId)
.title(title)
.articleUrl(articleUrl)
.recommended(recommended)
.hit(hit)
.registeredAt(registeredAt)
.build();
}
public static Long parseArticleId(Element td) {
return Long.parseLong(td.text().trim());
}
public static String parseTitle(Element td) {
return td.getElementsByTag("a").text(); // TODO cdn image extracting
}
public static String parseArticleUrl(Element td) {
return PpomppuBoardName.ofViewPageUrl(td.getElementsByTag("a").attr("href"));
}
public static Integer parseRecommended(Element td) {
final String voteString = td.text();
final int recommended;
if (voteString.isEmpty()) {
recommended = 0;
} else {
final int voteUp = Integer.parseInt(td.text().split(" - ")[0]);
final int voteDown = Integer.parseInt(td.text().split(" - ")[1]);
recommended = voteUp - voteDown;
}
return recommended;
}
public static Integer parseHit(Element td) {
return Integer.parseInt(td.text());
}
public static Instant parseRegisteredAt(Element td) {
final String registeredAtString = td.attr("title");
return DATE_TIME_FORMATTER.parse(registeredAtString, Instant::from);
}
}

92
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/PpomppuArticleTransformer.java
View File

@ -1,77 +1,53 @@
package com.myoa.engineering.crawl.ppomppu.processor.dto;
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import java.time.Instant;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import java.util.List;
import java.util.function.Function;
import java.util.stream.Collectors;
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
import com.myoa.engineering.crawl.ppomppu.support.dto.SimpleMessageDTO;
/**
* PpomppuArticleTransformer
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-11-21
*
* @author Shin Woo-jin (woozu.shin@kakaoent.com)
* @since 2021-09-08
*/
public final class PpomppuArticleTransformer {
private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yy.MM.dd HH:mm:ss")
.withZone(ZoneId.of("Asia/Seoul"));
private PpomppuArticleTransformer() {}
public static PpomppuArticle toArticle(Elements articleElement) {
final long articleId = PpomppuArticleTransformer.toArticleId(articleElement.get(0));
final String title = PpomppuArticleTransformer.toTitle(articleElement.get(2));
final String articleUrl = PpomppuArticleTransformer.toArticleUrl(articleElement.get(2));
final int recommended = PpomppuArticleTransformer.toRecommended(articleElement.get(6));
final int hit = PpomppuArticleTransformer.toHit(articleElement.get(7));
final Instant registeredAt = PpomppuArticleTransformer.toRegisteredAt(articleElement.get(5));
private static final String MESSAGE_FORMAT_V1 = "%s)) `%s` <%s:LINK>";
return PpomppuArticle.builder()
.articleId(articleId)
.title(title)
.articleUrl(articleUrl)
.recommended(recommended)
.hit(hit)
.registeredAt(registeredAt)
.build();
private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")
.withZone(ZoneId.of("Asia/Seoul"));
public static final Function<PpomppuArticle, SimpleMessageDTO> TRANSFORM_TO_MESSAGE_DTO = entity ->
SimpleMessageDTO.builder()
.requestedAt(Instant.now())
.publishedAt(entity.getRegisteredAt())
.title(String.format(MESSAGE_FORMAT_V1, entity.getBoardName().getMenuName(), entity.getTitle()))
.body(entity.getArticleUrl())
.build();
// https://stackoverflow.com/questions/24882927/using-streams-to-convert-a-list-of-objects-into-a-string-obtained-from-the-tostr
public static SimpleMessageDTO transform(List<PpomppuArticle> articles) {
Instant requestedAt = Instant.now();
String body = articles.stream()
.map(PpomppuArticleTransformer::convertToInlineMessage)
.collect(Collectors.joining("\n\n"));
return SimpleMessageDTO.builder()
.requestedAt(requestedAt)
.title(DATE_TIME_FORMATTER.format(requestedAt))
.body(body)
.build();
}
public static Long toArticleId(Element td) {
return Long.parseLong(td.text().trim());
public static String convertToInlineMessage(PpomppuArticle article) {
return String.format(MESSAGE_FORMAT_V1,
article.getBoardName().getMenuName(), article.getTitle(), article.getArticleUrl());
}
public static String toTitle(Element td) {
return td.text();
}
public static String toArticleUrl(Element td) {
return PpomppuBoardName.ofViewPageUrl(td.getElementsByTag("a").attr("href"));
}
public static Integer toRecommended(Element td) {
final String voteString = td.text();
final int recommended;
if (voteString.isEmpty()) {
recommended = 0;
} else {
final int voteUp = Integer.parseInt(td.text().split(" - ")[0]);
final int voteDown = Integer.parseInt(td.text().split(" - ")[1]);
recommended = voteUp - voteDown;
}
return recommended;
}
public static Integer toHit(Element td) {
return Integer.parseInt(td.text());
}
public static Instant toRegisteredAt(Element td) {
final String registeredAtString = td.attr("title");
return DATE_TIME_FORMATTER.parse(registeredAtString, Instant::from);
}
}

19
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/constant/WebClientPropertiesUnitName.java Normal file
View File

@ -0,0 +1,19 @@
package com.myoa.engineering.crawl.ppomppu.processor.dto.constant;
import lombok.AllArgsConstructor;
import lombok.Getter;
/**
* WebClientPropertiesUnitName
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-11-18
*
*/
@Getter
@AllArgsConstructor
public enum WebClientPropertiesUnitName {
PPOMPPU_NOTIFIER_SENDER_API("ppn-sender-api"),
;
private String unitName;
}

56
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/client/MessageSenderAPIClient.java Normal file
View File

@ -0,0 +1,56 @@
package com.myoa.engineering.crawl.ppomppu.processor.infrastructure.client;
import org.springframework.core.ParameterizedTypeReference;
import org.springframework.http.HttpHeaders;
import org.springframework.http.MediaType;
import org.springframework.stereotype.Component;
import org.springframework.web.reactive.function.client.WebClient;
import org.springframework.web.reactive.function.client.WebClientRequestException;
import com.myoa.engineering.crawl.ppomppu.processor.dto.constant.WebClientPropertiesUnitName;
import com.myoa.engineering.crawl.ppomppu.support.dto.SimpleMessageDTO;
import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebClientFilterFactory;
import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebFluxExchangeStragiesFactory;
import com.myoa.engineering.crawl.ppomppu.support.webclient.properties.WebClientProperties;
import com.myoa.engineering.crawl.ppomppu.support.webclient.properties.WebClientProperties.WebClientPropertiesUnit;
import lombok.extern.slf4j.Slf4j;
import reactor.core.publisher.Mono;
import reactor.core.scheduler.Schedulers;
/**
* PpomppuNotifierSenderAPIClient
*
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-11-17
*/
@Slf4j
@Component
public class MessageSenderAPIClient {
private final WebClient webClient;
public MessageSenderAPIClient(WebClientProperties webClientProperties) {
WebClientPropertiesUnit webClientPropertiesUnit =
webClientProperties.find(WebClientPropertiesUnitName.PPOMPPU_NOTIFIER_SENDER_API.getUnitName());
this.webClient = WebClient.builder()
.baseUrl(webClientPropertiesUnit.getBaseUrl())
.exchangeStrategies(WebFluxExchangeStragiesFactory.ofDefault())
.defaultHeader(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_VALUE)
// .filter(WebClientFilterFactory.logRequest())
// .filter(WebClientFilterFactory.logResponse())
.build();
}
public Mono<String> sendMessageToSlack(SimpleMessageDTO dto) {
return webClient.post()
.uri("/api/v1/messages/sendMessage/messengers/slack")
.bodyValue(dto)
.exchangeToMono(e -> e.bodyToMono(new ParameterizedTypeReference<String>() {}))
.publishOn(Schedulers.boundedElastic())
.onErrorResume(WebClientRequestException.class, t -> {
log.info("Exception occured, ignoring. : {}", t.getClass().getSimpleName());
return Mono.empty();
});
}
}

10
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/client/PpomppuBoardFeedRetriever.java
View File

@ -1,9 +1,11 @@
package com.myoa.engineering.crawl.ppomppu.processor.infrastructure.client;
import com.myoa.engineering.crawl.ppomppu.processor.configuration.factory.WebClientFilterFactory;
import com.myoa.engineering.crawl.ppomppu.processor.configuration.factory.WebFluxExchangeStragiesFactory;
import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebClientFilterFactory;
import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebFluxExchangeStragiesFactory;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import lombok.extern.slf4j.Slf4j;
import org.springframework.core.ParameterizedTypeReference;
import org.springframework.stereotype.Component;
import org.springframework.web.reactive.function.client.WebClient;
import org.springframework.web.reactive.function.client.WebClientRequestException;
@ -38,8 +40,8 @@ public class PpomppuBoardFeedRetriever {
.onErrorResume(WebClientRequestException.class, t -> {
log.info("Exception occured, ignoring. : {}", t.getClass().getSimpleName());
return Mono.empty();
})
.doOnNext(e -> log.info("[getHtml] {}", e));
});
// .doOnNext(e -> log.info("[getHtml] {}", e));
}
}

38
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/MessageSenderService.java Normal file
View File

@ -0,0 +1,38 @@
package com.myoa.engineering.crawl.ppomppu.processor.service;
import java.util.List;
import org.springframework.stereotype.Service;
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
import com.myoa.engineering.crawl.ppomppu.processor.dto.PpomppuArticleTransformer;
import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.client.MessageSenderAPIClient;
import lombok.extern.slf4j.Slf4j;
import reactor.core.publisher.Mono;
/**
* MessageSenderService
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-11-21
*
*/
@Slf4j
@Service
public class MessageSenderService {
private final MessageSenderAPIClient messageSenderAPIClient;
public MessageSenderService(MessageSenderAPIClient messageSenderAPIClient) {
this.messageSenderAPIClient = messageSenderAPIClient;
}
public Mono<String> sendMessageToSlack(PpomppuArticle article) {
return messageSenderAPIClient.sendMessageToSlack(PpomppuArticleTransformer.TRANSFORM_TO_MESSAGE_DTO.apply(article));
}
public Mono<String> sendMessageToSlack(List<PpomppuArticle> articles) {
return messageSenderAPIClient.sendMessageToSlack(PpomppuArticleTransformer.transform(articles));
}
}

25
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/PpomppuArticleService.java
View File

@ -1,16 +1,19 @@
package com.myoa.engineering.crawl.ppomppu.processor.service;
import java.util.List;
import java.util.Optional;
import java.util.stream.Collectors;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuBoardFeedStatus;
import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository.PpomppuArticleRepository;
import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository.PpomppuBoardFeedStatusRepository;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import java.util.List;
import java.util.Optional;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
@Slf4j
@Service
@ -32,13 +35,14 @@ public class PpomppuArticleService {
Long latestArticleId = boardFeedStatus.map(PpomppuBoardFeedStatus::getLatestParsedArticleId)
.orElse(0L);
log.info("latestArticleId : {}", latestArticleId);
return articles.stream()
.filter(e -> e.getArticleId().compareTo(latestArticleId) > 0)
.collect(Collectors.toList());
}
@Transactional
public void save(PpomppuBoardName boardName, List<PpomppuArticle> articles) {
public List<PpomppuArticle> save(PpomppuBoardName boardName, List<PpomppuArticle> articles) {
Long latestArticleId = articles.stream()
.map(PpomppuArticle::getArticleId)
.max(Long::compareTo)
@ -46,14 +50,17 @@ public class PpomppuArticleService {
// save PpomppuBoardFeedStatus
Optional<PpomppuBoardFeedStatus> boardFeedStatus = ppomppuBoardFeedStatusRepository.findByBoardName(boardName);
log.info("boardName: {}, isPresent?: {}", boardName, boardFeedStatus.isPresent());
boardFeedStatus.ifPresentOrElse(e -> {
e.updateArticleId(latestArticleId);
ppomppuBoardFeedStatusRepository.save(e);
if (latestArticleId.longValue() > 0L) {
e.updateArticleId(latestArticleId);
ppomppuBoardFeedStatusRepository.save(e);
}
},
() -> ppomppuBoardFeedStatusRepository.save(PpomppuBoardFeedStatus.of(boardName,
latestArticleId)));
// save real articles.
ppomppuArticleRepository.saveAll(articles);
return ppomppuArticleRepository.saveAll(articles);
}
}

12
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/PpomppuFeedService.java
View File

@ -1,7 +1,7 @@
package com.myoa.engineering.crawl.ppomppu.processor.service;
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
import com.myoa.engineering.crawl.ppomppu.processor.dto.PpomppuArticleTransformer;
import com.myoa.engineering.crawl.ppomppu.processor.dto.PpomppuArticleParser;
import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.client.PpomppuBoardFeedRetriever;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import java.util.List;
@ -30,11 +30,11 @@ public class PpomppuFeedService {
public Mono<List<PpomppuArticle>> getArticles(PpomppuBoardName boardName) {
final Mono<String> html = ppomppuBoardFeedRetriever.getHtml(boardName.getResourcePath());
final Mono<Element> tbody = extractTbodyFromHtml(html)
.doOnNext(e -> log.info("pre tbody - {}", e.html()));
final Mono<Element> tbody = extractTbodyFromHtml(html);
// .doOnNext(e -> log.info("pre tbody - {}", e.html()));
return extractArticlesFromTbody(tbody).map(this::convertFromElement)
.map(e -> e.updateBoardName(boardName))
.doOnNext(e -> log.info("parsed Result: {}", e))
// .doOnNext(e -> log.info("parsed Result: {}", e))
.collectList();
}
@ -42,7 +42,7 @@ public class PpomppuFeedService {
return html.map(Jsoup::parse)
.mapNotNull(e -> e.getElementById("revolution_main_table"))
.map(e -> e.getElementsByTag("tbody"))
.doOnNext(e -> log.info("tbody - {}", e.html()))
// .doOnNext(e -> log.info("tbody - {}", e.html()))
.map(e -> e.stream()
.findFirst()
.orElseThrow(() -> new IndexOutOfBoundsException("no tbody")));
@ -54,6 +54,6 @@ public class PpomppuFeedService {
}
private PpomppuArticle convertFromElement(Element element) {
return PpomppuArticleTransformer.toArticle(element.getElementsByTag("td"));
return PpomppuArticleParser.toArticle(element.getElementsByTag("td"));
}
}

9
processor/src/main/resources/application-development.yml
View File

@ -3,6 +3,11 @@ spring:
activate:
on-profile: development
import:
- "configserver:http://192.168.0.100:11080"
- classpath:/development/webclient.yml
- classpath:/development/temppassword.yml
- classpath:/development/database.yml
server:
port: 20081
# import: optional:configserver:http://localhost:11080 # can be start up even config server was not found.

12
processor/src/main/resources/application-local.yml Normal file
View File

@ -0,0 +1,12 @@
spring:
config:
activate:
on-profile: local
import:
- "configserver:http://localhost:20085"
- classpath:/local/webclient.yml
server:
port: 20081
# import: optional:configserver:http://localhost:11080 # can be start up even config server was not found.

6
processor/src/main/resources/application-production.yml Normal file
View File

@ -0,0 +1,6 @@
spring:
config:
activate:
on-profile: production
import:
- "configserver:http://192.168.0.100:11080"

17
processor/src/main/resources/application.yml
View File

@ -1,14 +1,25 @@
spring:
application:
name: ppn-processor
main:
allow-bean-definition-overriding: true
profiles:
active: development
active: ${SPRING_ACTIVE_PROFILE:local}
group:
local: "local,datasource-local"
development: "development,datasource-development"
production: "production, datasource-production"
freemarker:
enabled: false
server:
port: 20081
port: 20080
error:
whitelabel:
enabled: false
management:
endpoints:
web:
exposure:
include: refresh

6
processor/src/main/resources/development/webclient.yml
View File

@ -1,5 +1,5 @@
webclient:
some: test
init: true
units:
- unit-name: processor-api
base-url: http://localhost:20081
- unit-name: ppn-sender-api
base-url: http://localhost:20082

5
processor/src/main/resources/local/webclient.yml Normal file
View File

@ -0,0 +1,5 @@
webclient:
init: true
units:
- unit-name: ppn-sender-api
base-url: http://localhost:20082

4
processor/src/main/resources/logback-spring.xml
View File

@ -1,6 +1,10 @@
<?xml version="1.0" encoding="UTF-8"?>
<configuration>
<springProperty name="DEFAULT_LEVEL_CONFIG" source="log.defaultLevel" />
<springProfile name="local">
<include resource="logback/logback-development.xml" />
<logger name="org.apache.kafka" level="INFO" />
</springProfile>
<springProfile name="development">
<include resource="logback/logback-development.xml" />
<logger name="org.apache.kafka" level="INFO" />

5
processor/src/main/resources/production/webclient.yml
View File

@ -1,4 +1,5 @@
webclient:
init: true
units:
- unit-name: processor-api
base-url: http://soundhoundfound-processor:20080
- unit-name: ppn-sender-api
base-url: http://ppn_sender:20082

1
receiver/build.gradle
View File

@ -8,6 +8,7 @@ dependencies {
// https://projectreactor.io/docs/core/release/reference/#debug-activate
implementation 'org.springframework.boot:spring-boot-starter-webflux'
implementation 'org.springframework.boot:spring-boot-configuration-processor'
implementation 'org.springframework.cloud:spring-cloud-starter-config'
implementation 'org.telegram:telegrambots:5.3.0'
annotationProcessor 'org.springframework.boot:spring-boot-configuration-processor'

5
receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/ReceiverApplication.java
View File

@ -3,8 +3,10 @@ package com.myoa.engineering.crawl.ppomppu.receiver;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.annotation.Import;
import com.myoa.engineering.crawl.ppomppu.receiver.configuration.properties.TelegramBotProperties;
import com.myoa.engineering.crawl.ppomppu.support.webclient.PpomppuNotifierWebClientConfiguration;
/**
* ReceiverApplication
@ -12,8 +14,9 @@ import com.myoa.engineering.crawl.ppomppu.receiver.configuration.properties.Tele
* @since 2021-08-20
*
*/
@Import({ PpomppuNotifierWebClientConfiguration.class})
@SpringBootApplication
@EnableConfigurationProperties({ TelegramBotProperties.class })
// @EnableConfigurationProperties({ TelegramBotProperties.class })
public class ReceiverApplication {
public static void main(String[] args) {

7
receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/configuration/TelegramBotConfiguration.java
View File

@ -9,6 +9,7 @@ import org.telegram.telegrambots.meta.exceptions.TelegramApiException;
import org.telegram.telegrambots.updatesreceivers.DefaultBotSession;
import com.myoa.engineering.crawl.ppomppu.receiver.configuration.properties.TelegramBotProperties;
import com.myoa.engineering.crawl.ppomppu.receiver.configuration.properties.TelegramBotProperties.TelegramBotPropertiesUnit;
import com.myoa.engineering.crawl.ppomppu.receiver.dispatch.MessageDispatcher;
import com.myoa.engineering.crawl.ppomppu.receiver.handler.message.MessageHandler;
@ -21,6 +22,7 @@ import com.myoa.engineering.crawl.ppomppu.receiver.handler.message.MessageHandle
@Configuration
public class TelegramBotConfiguration {
private static final String BOT_PROPERTIES_UNIT_NAME = "ppomppu_notify_bot";
@Bean
public TelegramBotsApi telegramBotsApi(MessageDispatcher messageDispatcher) throws TelegramApiException {
TelegramBotsApi api = new TelegramBotsApi(DefaultBotSession.class);
@ -30,7 +32,8 @@ public class TelegramBotConfiguration {
@Bean
public MessageDispatcher messageDispatcher(List<MessageHandler> messageHandlers,
TelegramBotProperties botProperties) {
return new MessageDispatcher(messageHandlers, botProperties.getName(), botProperties.getToken());
TelegramBotProperties telegramBotProperties) {
TelegramBotPropertiesUnit propertiesUnit = telegramBotProperties.find(BOT_PROPERTIES_UNIT_NAME);
return new MessageDispatcher(messageHandlers, propertiesUnit.getName(), propertiesUnit.getToken());
}
}

33
receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/configuration/properties/TelegramBotProperties.java
View File

@ -1,9 +1,17 @@
package com.myoa.engineering.crawl.ppomppu.receiver.configuration.properties;
import java.util.ArrayList;
import java.util.List;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.boot.context.properties.ConstructorBinding;
import org.springframework.stereotype.Component;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
/**
* TelegramBotProperties
@ -11,15 +19,26 @@ import lombok.Getter;
* @since 2021-09-05
*
*/
@Component
@Setter
@Getter
@ConstructorBinding
@ConfigurationProperties(prefix = "telegram.bot")
@ConfigurationProperties(prefix = "infra.telegram.bot")
public class TelegramBotProperties {
private final String name;
private final String token;
public TelegramBotProperties(final String name, final String token) {
this.name = name;
this.token = token;
private List<TelegramBotPropertiesUnit> units = new ArrayList<>();
@Data
public static class TelegramBotPropertiesUnit {
private String unitName;
private String name;
private String token;
}
public TelegramBotPropertiesUnit find(String unitName) {
return units.stream()
.filter(e -> e.getUnitName().equals(unitName))
.findFirst()
.orElseThrow(
() -> new IllegalArgumentException(this.getClass().getName() + ": unitName Not found. " + unitName));
}
}

19
receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/dto/constant/WebClientPropertiesUnitName.java Normal file
View File

@ -0,0 +1,19 @@
package com.myoa.engineering.crawl.ppomppu.receiver.dto.constant;
import lombok.AllArgsConstructor;
import lombok.Getter;
/**
* WebClientPropertiesUnitName
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-11-18
*
*/
@Getter
@AllArgsConstructor
public enum WebClientPropertiesUnitName {
PPOMPPU_NOTIFIER_PROCESSOR_API("ppn-processor-api"),
;
private String unitName;
}

17
receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/infrastructure/client/ProcessorAPIWebClient.java
View File

@ -1,12 +1,15 @@
package com.myoa.engineering.crawl.ppomppu.receiver.infrastructure.client;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.core.ParameterizedTypeReference;
import org.springframework.stereotype.Component;
import org.springframework.web.reactive.function.client.WebClient;
import org.springframework.web.reactive.function.client.WebClientRequestException;
import com.myoa.engineering.crawl.ppomppu.receiver.dto.constant.WebClientPropertiesUnitName;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebClientFilterFactory;
import com.myoa.engineering.crawl.ppomppu.support.webclient.properties.WebClientProperties;
import com.myoa.engineering.crawl.ppomppu.support.webclient.properties.WebClientProperties.WebClientPropertiesUnit;
import lombok.extern.slf4j.Slf4j;
import reactor.core.publisher.Mono;
@ -18,10 +21,14 @@ public class ProcessorAPIWebClient {
private final WebClient webClient;
public ProcessorAPIWebClient(WebClient.Builder webClientBuilder,
@Value("${webclient.base-url}") String baseUrl) {
this.webClient = webClientBuilder.baseUrl(baseUrl)
.build();
public ProcessorAPIWebClient(WebClientProperties webClientProperties) {
WebClientPropertiesUnit webClientPropertiesUnit =
webClientProperties.find(WebClientPropertiesUnitName.PPOMPPU_NOTIFIER_PROCESSOR_API.getUnitName());
this.webClient = WebClient.builder()
.baseUrl(webClientPropertiesUnit.getBaseUrl())
.filter(WebClientFilterFactory.logRequest())
.filter(WebClientFilterFactory.logResponse())
.build();
}
public Mono<String> emitParseEvent(PpomppuBoardName boardName) {

14
receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/scheduler/ParseEventEmitter.java
View File

@ -26,15 +26,11 @@ public class ParseEventEmitter {
this.processorAPIService = processorAPIService;
}
@Scheduled(fixedRate = 60 * 1000L)
public void emitDomesticBoard() {
log.info("[emitDomesticBoard] trigger fired!");
processorAPIService.emitParseEvent(PpomppuBoardName.PPOMPPU_DOMESTIC_ETC).block();
}
@Scheduled(fixedRate = 300 * 1000L)
public void emitOverseaBoard() {
log.info("[emitOverseaBoard] trigger fired!");
processorAPIService.emitParseEvent(PpomppuBoardName.PPOMPPU_OVERSEA_ETC).block();
public void emitBoards() {
log.info("[emitDomesticBoard] trigger fired!");
for (PpomppuBoardName boardName : PpomppuBoardName.values()) {
processorAPIService.emitParseEvent(boardName).block();
}
}
}

2
receiver/src/main/resources/application-development.yml
View File

@ -4,4 +4,4 @@ spring:
on-profile: development
import:
- classpath:/development/webclient.yml
- classpath:/development/temppassword.yml
- "configserver:http://192.168.0.100:11080"

7
receiver/src/main/resources/application-local.yml Normal file
View File

@ -0,0 +1,7 @@
spring:
config:
activate:
on-profile: local
import:
- classpath:/local/webclient.yml
- "configserver:http://localhost:20085"

1
receiver/src/main/resources/application-production.yml
View File

@ -4,3 +4,4 @@ spring:
on-profile: production
import:
- classpath:/production/webclient.yml
- "configserver:http://192.168.0.100:11080"

7
receiver/src/main/resources/application.yml
View File

@ -1,10 +1,15 @@
spring:
application:
name: ppn-receiver
main:
allow-bean-definition-overriding: true
profiles:
active: development
active: ${SPRING_ACTIVE_PROFILE:local}
freemarker:
enabled: false
server:
port: 20080
error:
whitelabel:
enabled: false

4
receiver/src/main/resources/development/webclient.yml
View File

@ -1,5 +1,5 @@
webclient:
base-url: http://localhost:20081
init: true
units:
- unit-name: processor-api
- unit-name: ppn-processor-api
base-url: http://localhost:20081

5
receiver/src/main/resources/local/webclient.yml Normal file
View File

@ -0,0 +1,5 @@
webclient:
init: true
units:
- unit-name: ppn-processor-api
base-url: http://localhost:20081

15
receiver/src/main/resources/logback-spring.xml Normal file
View File

@ -0,0 +1,15 @@
<?xml version="1.0" encoding="UTF-8"?>
<configuration>
<springProperty name="DEFAULT_LEVEL_CONFIG" source="log.defaultLevel" />
<springProfile name="local">
<include resource="logback/logback-development.xml" />
<logger name="org.apache.kafka" level="INFO" />
</springProfile>
<springProfile name="development">
<include resource="logback/logback-development.xml" />
<logger name="org.apache.kafka" level="INFO" />
</springProfile>
<springProfile name="production">
<include resource="logback/logback-production.xml" />
</springProfile>
</configuration>

23
receiver/src/main/resources/logback/component/logback-file.xml Normal file
View File

@ -0,0 +1,23 @@
<included>
<property name="FILE_LOG_PATTERN"
value="%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{45}:%L - %msg%n" />
<property name="LOG_FILE_BASE" value="lcp-benefit-benefit-api" />
<appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${DIRECTORY}/${LOG_FILE_BASE}_log</file>
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<fileNamePattern>${DIRECTORY}/${LOG_FILE_BASE}_log.%d{yyyyMMdd}.%i</fileNamePattern>
<timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
<maxFileSize>1000MB</maxFileSize>
</timeBasedFileNamingAndTriggeringPolicy>
<maxHistory>60</maxHistory>
</rollingPolicy>
<encoder>
<pattern>${FILE_LOG_PATTERN}</pattern>
<immediateFlush>${IMMEDIATE_FLUSH}</immediateFlush>
</encoder>
</appender>
<appender name="ASYNC_FILE" class="ch.qos.logback.classic.AsyncAppender">
<queueSize>1024</queueSize>
<appender-ref ref="FILE" />
</appender>
</included>

19
receiver/src/main/resources/logback/logback-development.xml Normal file
View File

@ -0,0 +1,19 @@
<?xml version="1.0" encoding="UTF-8"?>
<included>
<!-- =========== property BETA ========= -->
<property name="DEFAULT_LEVEL" value="${DEFAULT_LEVEL_CONFIG:-INFO}"/>
<!--file-->
<property name="DIRECTORY" value="/home1/www/logs/supervisor"/>
<property name="IMMEDIATE_FLUSH" value="true"/>
<!--nelo2-->
<property name="NELO2_LEVEL" value="WARN"/>
<!-- =========== include appender =========== -->
<include resource="org/springframework/boot/logging/logback/defaults.xml"/>
<include resource="org/springframework/boot/logging/logback/console-appender.xml"/>
<include resource="logback/component/logback-nelo2.xml"/>
<include resource="logback/component/logback-datachain.xml"/>
<!-- =========== root logger ============== -->
<root level="${DEFAULT_LEVEL}">
<appender-ref ref="CONSOLE"/>
</root>
</included>

19
receiver/src/main/resources/logback/logback-production.xml Normal file
View File

@ -0,0 +1,19 @@
<?xml version="1.0" encoding="UTF-8"?>
<included>
<!-- =========== property RELEASE ========= -->
<property name="DEFAULT_LEVEL" value="${DEFAULT_LEVEL_CONFIG:-INFO}"/>
<!--file-->
<property name="DIRECTORY" value="/home1/www/logs/supervisor"/>
<property name="IMMEDIATE_FLUSH" value="true"/>
<!--nelo2-->
<property name="NELO2_LEVEL" value="WARN"/>
<!-- =========== include appender =========== -->
<include resource="org/springframework/boot/logging/logback/defaults.xml"/>
<include resource="org/springframework/boot/logging/logback/console-appender.xml"/>
<include resource="logback/component/logback-nelo2.xml"/>
<include resource="logback/component/logback-datachain.xml"/>
<!-- =========== root logger ============== -->
<root level="${DEFAULT_LEVEL}">
<appender-ref ref="CONSOLE"/>
</root>
</included>

5
receiver/src/main/resources/production/production.yml
View File

@ -1,5 +0,0 @@
webclient:
base-url: http://ppomppu_notifier_processor:20080
units:
- unit-name: processor-api
base-url: http://ppomppu_notifier_processor:20080

5
receiver/src/main/resources/production/webclient.yml Normal file
View File

@ -0,0 +1,5 @@
webclient:
init: true
units:
- unit-name: ppn-processor-api
base-url: http://ppn_processor:20080

23
sender/build.gradle Normal file
View File

@ -0,0 +1,23 @@
dependencies {
developmentOnly 'org.springframework.boot:spring-boot-devtools'
runtimeOnly 'mysql:mysql-connector-java'
compileOnly 'org.projectlombok:lombok'
implementation project(':support')
// https://projectreactor.io/docs/core/release/reference/#debug-activate
implementation 'org.springframework.boot:spring-boot-starter-webflux'
implementation 'org.springframework.boot:spring-boot-starter-actuator'
implementation 'org.springframework.cloud:spring-cloud-starter-config'
annotationProcessor 'org.springframework.boot:spring-boot-configuration-processor'
annotationProcessor 'org.projectlombok:lombok'
testImplementation 'org.springframework.boot:spring-boot-starter-test'
}
test {
useJUnitPlatform()
testLogging {
events "passed", "skipped", "failed"
}
}

12
sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/SenderApplication.java
View File

@ -1,10 +1,22 @@
package com.myoa.engineering.crawl.ppomppu.sender;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.context.annotation.Import;
import com.myoa.engineering.crawl.ppomppu.support.webclient.PpomppuNotifierWebClientConfiguration;
/**
* SenderApplication
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-08-20
*
*/
@Import({ PpomppuNotifierWebClientConfiguration.class })
@SpringBootApplication
public class SenderApplication {
public static void main(String[] args) {
SpringApplication.run(SenderApplication.class, args);
}
}

34
sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/configuration/properties/SlackSecretProperties.java Normal file
View File

@ -0,0 +1,34 @@
package com.myoa.engineering.crawl.ppomppu.sender.configuration.properties;
import java.util.List;
import lombok.Data;
import lombok.Getter;
import lombok.Setter;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;
@Getter
@Setter
@Component
@ConfigurationProperties("infra.slack.bot")
public class SlackSecretProperties {
private List<SlackSecretPropertiesUnit> units;
@Data
public static class SlackSecretPropertiesUnit {
private String botName;
private String username;
private String iconEmoji;
private String channel;
private String token;
}
public SlackSecretPropertiesUnit find(String botUnitName) {
return units.stream()
.filter(e -> e.getBotName().equals(botUnitName))
.findFirst()
.orElseThrow(() -> new IllegalArgumentException("not found bot unit name : " + botUnitName));
}
}

35
sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/controller/MessageSenderAPIController.java Normal file
View File

@ -0,0 +1,35 @@
package com.myoa.engineering.crawl.ppomppu.sender.controller;
import com.myoa.engineering.crawl.ppomppu.sender.infrastructure.client.MongeShoppingBotSlackMessageSender;
import com.myoa.engineering.crawl.ppomppu.support.dto.APIResponse;
import com.myoa.engineering.crawl.ppomppu.support.dto.SimpleMessageDTO;
import lombok.extern.slf4j.Slf4j;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import reactor.core.publisher.Mono;
/**
* MessageSenderAPIController
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-11-21
*
*/
@Slf4j
@RestController
@RequestMapping("/api/v1")
public class MessageSenderAPIController {
private final MongeShoppingBotSlackMessageSender sender;
public MessageSenderAPIController(MongeShoppingBotSlackMessageSender sender) {
this.sender = sender;
}
@PostMapping("/messages/sendMessage/messengers/slack")
public Mono<APIResponse<SimpleMessageDTO>> sendMessageToSlack(@RequestBody SimpleMessageDTO dto) {
return sender.sendMessage(sender.ofMessage(dto.getBody()))
.then(Mono.just(APIResponse.success(dto)));
}
}

33
sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/controller/TestAPIController.java Normal file
View File

@ -0,0 +1,33 @@
package com.myoa.engineering.crawl.ppomppu.sender.controller;
import com.myoa.engineering.crawl.ppomppu.sender.infrastructure.client.MongeShoppingBotSlackMessageSender;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import lombok.extern.slf4j.Slf4j;
import reactor.core.publisher.Mono;
/**
* TestAPIController
*
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-11-15
*/
@Slf4j
@RestController
@RequestMapping("/api/v1")
public class TestAPIController {
private final MongeShoppingBotSlackMessageSender sender;
public TestAPIController(MongeShoppingBotSlackMessageSender sender) {
this.sender = sender;
}
@GetMapping("/test")
public Mono<String> test() {
log.info("received!!!");
return sender.sendMessage(sender.ofMessage("testtesttest!!!"));
}
}

12
sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/dto/MessageDTO.java Normal file
View File

@ -0,0 +1,12 @@
package com.myoa.engineering.crawl.ppomppu.sender.dto;
import java.io.Serializable;
/**
* MessageDTO
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-11-14
*
*/
public interface MessageDTO extends Serializable {
}

39
sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/dto/SlackMessageDTO.java Normal file
View File

@ -0,0 +1,39 @@
package com.myoa.engineering.crawl.ppomppu.sender.dto;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.Builder;
import lombok.Getter;
import lombok.NoArgsConstructor;
/**
* SlackMessageDTO
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-11-14
*
*/
@Getter
@NoArgsConstructor
public class SlackMessageDTO implements MessageDTO {
private final static long serialVersionUID = 4737608709660494713L;
private String text;
private String channel;
private String username;
@JsonProperty("icon_emoji")
private String iconEmoji;
@Builder
public SlackMessageDTO(String text, String channel, String username, String iconEmoji) {
this.text = text;
this.channel = channel;
this.username = username;
this.iconEmoji = iconEmoji;
}
public void applyText(String text) {
this.text = text;
}
}

17
sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/infrastructure/client/MessageSender.java Normal file
View File

@ -0,0 +1,17 @@
package com.myoa.engineering.crawl.ppomppu.sender.infrastructure.client;
import com.myoa.engineering.crawl.ppomppu.sender.dto.MessageDTO;
import reactor.core.publisher.Mono;
/**
* MessageSender
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-11-14
*
*/
public interface MessageSender<T extends MessageDTO> {
Mono<String> sendMessage(T message);
}

38
sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/infrastructure/client/MongeShoppingBotSlackMessageSender.java Normal file
View File

@ -0,0 +1,38 @@
package com.myoa.engineering.crawl.ppomppu.sender.infrastructure.client;
import com.myoa.engineering.crawl.ppomppu.sender.configuration.properties.SlackSecretProperties;
import com.myoa.engineering.crawl.ppomppu.sender.configuration.properties.SlackSecretProperties.SlackSecretPropertiesUnit;
import com.myoa.engineering.crawl.ppomppu.sender.dto.SlackMessageDTO;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
@Slf4j
@Component
public class MongeShoppingBotSlackMessageSender extends SlackMessageSender {
private static final String SLACK_SECRET_UNIT_NAME = "monge_shopping_bot";
private final SlackSecretPropertiesUnit slackProperties;
public MongeShoppingBotSlackMessageSender(SlackSecretProperties slackSecretProperties) {
super(slackSecretProperties.find(SLACK_SECRET_UNIT_NAME).getToken());
this.slackProperties = slackSecretProperties.find(SLACK_SECRET_UNIT_NAME);
}
public SlackMessageDTO ofMessageTemplate() {
return SlackMessageDTO.builder()
.channel(slackProperties.getChannel())
.iconEmoji(slackProperties.getIconEmoji())
.username(slackProperties.getUsername())
.build();
}
public SlackMessageDTO ofMessage(String text) {
return SlackMessageDTO.builder()
.channel(slackProperties.getChannel())
.iconEmoji(slackProperties.getIconEmoji())
.username(slackProperties.getUsername())
.text(text)
.build();
}
}

53
sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/infrastructure/client/SlackMessageSender.java Normal file
View File

@ -0,0 +1,53 @@
package com.myoa.engineering.crawl.ppomppu.sender.infrastructure.client;
import org.springframework.http.HttpHeaders;
import org.springframework.http.MediaType;
import org.springframework.web.reactive.function.client.WebClient;
import org.springframework.web.reactive.function.client.WebClientRequestException;
import com.myoa.engineering.crawl.ppomppu.sender.dto.SlackMessageDTO;
import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebClientFilterFactory;
import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebFluxExchangeStragiesFactory;
import lombok.extern.slf4j.Slf4j;
import reactor.core.publisher.Mono;
import reactor.core.scheduler.Schedulers;
/**
* SlackMessageSender
*
* @author Shin Woo-jin (woozu.shin@kakaoent.com)
* @since 2021-09-08
*/
@Slf4j
public class SlackMessageSender implements MessageSender<SlackMessageDTO> {
private static final String SLACK_API_URL = "https://slack.com/api";
private final WebClient webClient;
public SlackMessageSender(String apiSecret) {
this.webClient = WebClient.builder()
.baseUrl(SLACK_API_URL)
.defaultHeader(HttpHeaders.AUTHORIZATION, "Bearer " + apiSecret)
.exchangeStrategies(WebFluxExchangeStragiesFactory.ofDefault())
.defaultHeader(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_UTF8_VALUE)
.defaultHeader(HttpHeaders.ACCEPT_CHARSET, "UTF-8")
.filter(WebClientFilterFactory.logResponse())
.build();
}
@Override
public Mono<String> sendMessage(SlackMessageDTO message) {
return webClient.post()
.uri("/chat.postMessage")
.bodyValue(message)
.exchangeToMono(e -> e.bodyToMono(String.class))
.publishOn(Schedulers.boundedElastic())
.onErrorResume(WebClientRequestException.class, t -> {
log.info("Exception occured, ignoring. : {}", t.getClass().getSimpleName());
return Mono.empty();
});
}
}

10
sender/src/main/resources/application-development.yml Normal file
View File

@ -0,0 +1,10 @@
spring:
config:
activate:
on-profile: development
import:
- "configserver:http://192.168.0.100:11080"
# import: optional:configserver:http://localhost:11080 # can be start up even config server was not found.
server:
port: 20082

10
sender/src/main/resources/application-local.yml Normal file
View File

@ -0,0 +1,10 @@
spring:
config:
activate:
on-profile: local
import:
- "configserver:http://localhost:20085"
# import: optional:configserver:http://localhost:11080 # can be start up even config server was not found.
server:
port: 20082

6
sender/src/main/resources/application-production.yml Normal file
View File

@ -0,0 +1,6 @@
spring:
config:
activate:
on-profile: production
import:
- "configserver:http://192.168.0.100:11080"

25
sender/src/main/resources/application.yml
View File

@ -0,0 +1,25 @@
spring:
application:
name: ppn-sender
main:
allow-bean-definition-overriding: true
profiles:
active: ${SPRING_ACTIVE_PROFILE:local}
group:
local: local, slackapi-local
development: development, slackapi-development
production: production, slackapi-production
freemarker:
enabled: false
server:
port: 20080
error:
whitelabel:
enabled: false
management:
endpoints:
web:
exposure:
include: refresh, env

15
sender/src/main/resources/logback-spring.xml Normal file
View File

@ -0,0 +1,15 @@
<?xml version="1.0" encoding="UTF-8"?>
<configuration>
<springProfile name="local">
<include resource="logback/logback-development.xml" />
<logger name="org.apache.kafka" level="INFO" />
</springProfile>
<springProperty name="DEFAULT_LEVEL_CONFIG" source="log.defaultLevel" />
<springProfile name="development">
<include resource="logback/logback-development.xml" />
<logger name="org.apache.kafka" level="INFO" />
</springProfile>
<springProfile name="production">
<include resource="logback/logback-production.xml" />
</springProfile>
</configuration>

23
sender/src/main/resources/logback/component/logback-file.xml Normal file
View File

@ -0,0 +1,23 @@
<included>
<property name="FILE_LOG_PATTERN"
value="%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{45}:%L - %msg%n" />
<property name="LOG_FILE_BASE" value="lcp-benefit-benefit-api" />
<appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${DIRECTORY}/${LOG_FILE_BASE}_log</file>
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<fileNamePattern>${DIRECTORY}/${LOG_FILE_BASE}_log.%d{yyyyMMdd}.%i</fileNamePattern>
<timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
<maxFileSize>1000MB</maxFileSize>
</timeBasedFileNamingAndTriggeringPolicy>
<maxHistory>60</maxHistory>
</rollingPolicy>
<encoder>
<pattern>${FILE_LOG_PATTERN}</pattern>
<immediateFlush>${IMMEDIATE_FLUSH}</immediateFlush>
</encoder>
</appender>
<appender name="ASYNC_FILE" class="ch.qos.logback.classic.AsyncAppender">
<queueSize>1024</queueSize>
<appender-ref ref="FILE" />
</appender>
</included>

19
sender/src/main/resources/logback/logback-development.xml Normal file
View File

@ -0,0 +1,19 @@
<?xml version="1.0" encoding="UTF-8"?>
<included>
<!-- =========== property BETA ========= -->
<property name="DEFAULT_LEVEL" value="${DEFAULT_LEVEL_CONFIG:-INFO}"/>
<!--file-->
<property name="DIRECTORY" value="/home1/www/logs/supervisor"/>
<property name="IMMEDIATE_FLUSH" value="true"/>
<!--nelo2-->
<property name="NELO2_LEVEL" value="WARN"/>
<!-- =========== include appender =========== -->
<include resource="org/springframework/boot/logging/logback/defaults.xml"/>
<include resource="org/springframework/boot/logging/logback/console-appender.xml"/>
<include resource="logback/component/logback-nelo2.xml"/>
<include resource="logback/component/logback-datachain.xml"/>
<!-- =========== root logger ============== -->
<root level="${DEFAULT_LEVEL}">
<appender-ref ref="CONSOLE"/>
</root>
</included>

19
sender/src/main/resources/logback/logback-production.xml Normal file
View File

@ -0,0 +1,19 @@
<?xml version="1.0" encoding="UTF-8"?>
<included>
<!-- =========== property RELEASE ========= -->
<property name="DEFAULT_LEVEL" value="${DEFAULT_LEVEL_CONFIG:-INFO}"/>
<!--file-->
<property name="DIRECTORY" value="/home1/www/logs/supervisor"/>
<property name="IMMEDIATE_FLUSH" value="true"/>
<!--nelo2-->
<property name="NELO2_LEVEL" value="WARN"/>
<!-- =========== include appender =========== -->
<include resource="org/springframework/boot/logging/logback/defaults.xml"/>
<include resource="org/springframework/boot/logging/logback/console-appender.xml"/>
<include resource="logback/component/logback-nelo2.xml"/>
<include resource="logback/component/logback-datachain.xml"/>
<!-- =========== root logger ============== -->
<root level="${DEFAULT_LEVEL}">
<appender-ref ref="CONSOLE"/>
</root>
</included>

4
support/build.gradle
View File

@ -1,11 +1,9 @@
dependencies {
runtimeOnly 'mysql:mysql-connector-java'
compileOnly 'org.projectlombok:lombok'
// https://projectreactor.io/docs/core/release/reference/#debug-activate
annotationProcessor 'org.springframework.boot:spring-boot-configuration-processor'
annotationProcessor 'org.projectlombok:lombok'
implementation 'org.springframework.boot:spring-boot-starter-webflux'
}

31
support/next_message_tempalte(block_kit_builder).json Normal file
View File

@ -0,0 +1,31 @@
{
"blocks": [
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": "기타)) `[쿠팡] 타푸코 이중 진공 스텐 세라믹 코팅 텀블러 (화이트만 11,130원/로켓와우무료) 4 [기타]`<https://www.ppomppu.co.kr/zboardview.php?id=ppomppu&page=1&divpage=69&category=1&&no=404126|LINK>"
},
"accessory": {
"type": "image",
"image_url": "cdn.ppomppu.co.kr/zboard/data3/2021/1121/m_20211121184835_zfyahnow.png",
"alt_text": "alt text for image"
}
},
{
"type": "divider"
},
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": "기타)) `[롯데온] 1매 149원 올국산 KF94새부리마스크 대형 200매 (29,920원/무료배송) 58 [기타]`<https://www.ppomppu.co.kr/zboardview.php?id=ppomppu&page=1&divpage=69&category=1&&no=404113|LINK>"
},
"accessory": {
"type": "image",
"image_url": "https://s3-media3.fl.yelpcdn.com/bphoto/c7ed05m9lC2EmA3Aruue7A/o.jpg",
"alt_text": "alt text for image"
}
}
]
}

37
support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/dto/SimpleMessageDTO.java Normal file
View File

@ -0,0 +1,37 @@
package com.myoa.engineering.crawl.ppomppu.support.dto;
import java.io.Serializable;
import java.time.Instant;
import lombok.Builder;
import lombok.Getter;
import lombok.NoArgsConstructor;
/**
* SimpleMessageDTO
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-11-21
*
*/
@Getter
@NoArgsConstructor
public class SimpleMessageDTO implements Serializable {
private static final long serialVersionUID = 2203955567672404428L;
private String title;
private String body;
private String url;
private Instant publishedAt;
private Instant requestedAt;
@Builder
public SimpleMessageDTO(String title, String body, String url, Instant publishedAt, Instant requestedAt) {
this.title = title;
this.body = body;
this.url = url;
this.publishedAt = publishedAt;
this.requestedAt = requestedAt;
}
}

52
support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/dto/code/PpomppuBoardName.java
View File

@ -12,40 +12,40 @@ import lombok.Getter;
@Getter
@AllArgsConstructor
public enum PpomppuBoardName {
PPOMPPU_DOMESTIC_ALL("zboard/zboard.php?id=ppomppu", "전체", true),
PPOMPPU_DOMESTIC_ETC("zboard/zboard.php?id=ppomppu&category=1", "기타", true),
PPOMPPU_DOMESTIC_COMPUTER("zboard/zboard.php?id=ppomppu&category=4", "컴퓨터", true),
PPOMPPU_DOMESTIC_DIGITAL("zboard/zboard.php?id=ppomppu&category=5", "디지털", true),
PPOMPPU_DOMESTIC_FOOD("zboard/zboard.php?id=ppomppu&category=6", "식품/건강", true),
PPOMPPU_DOMESTIC_BOOK("zboard/zboard.php?id=ppomppu&category=8", "서적", true),
PPOMPPU_DOMESTIC_APPLIANCES("zboard/zboard.php?id=ppomppu&category=9", "가전/가구", true),
PPOMPPU_DOMESTIC_PARENTING("zboard/zboard.php?id=ppomppu&category=10", "육아", true),
PPOMPPU_DOMESTIC_GIFTCARD("zboard/zboard.php?id=ppomppu&category=11", "상품권", true),
PPOMPPU_DOMESTIC_CLOTHES("zboard/zboard.php?id=ppomppu&category=12", "의류/잡화", true),
PPOMPPU_DOMESTIC_COSMETIC("zboard/zboard.php?id=ppomppu&category=13", "화장품", true),
PPOMPPU_DOMESTIC_OUTDOOR("zboard/zboard.php?id=ppomppu&category=15", "등산/캠핑", true),
PPOMPPU_OVERSEA_ALL("zboard/zboard.php?id=ppomppu4", "전체", true),
PPOMPPU_OVERSEA_ETC("zboard/zboard.php?id=ppomppu4&category=1", "기타", true),
PPOMPPU_OVERSEA_APPLIANCES("zboard/zboard.php?id=ppomppu4&category=7", "가전", true),
PPOMPPU_OVERSEA_TVAV("zboard/zboard.php?id=ppomppu4&category=8", "TV/영상", true),
PPOMPPU_OVERSEA_COMPUTER("zboard/zboard.php?id=ppomppu4&category=3", "컴퓨터", true),
PPOMPPU_OVERSEA_DIGITAL("zboard/zboard.php?id=ppomppu4&category=4", "디지털", true),
PPOMPPU_OVERSEA_MOBILEACCESSORY("zboard/zboard.php?id=ppomppu4&category=9", "액세서리", true),
PPOMPPU_OVERSEA_CLOTHES("zboard/zboard.php?id=ppomppu4&category=5", "의류/잡화", true),
PPOMPPU_OVERSEA_WATCH("zboard/zboard.php?id=ppomppu4&category=2", "시계", true),
PPOMPPU_OVERSEA_SHOES("zboard/zboard.php?id=ppomppu4&category=11", "신발", true),
PPOMPPU_OVERSEA_FOOD("zboard/zboard.php?id=ppomppu4&category=10", "식품/건강", true),
PPOMPPU_OVERSEA_PARENTING("zboard/zboard.php?id=ppomppu4&category=6", "육아", true),
PPOMPPU_DOMESTIC_ALL("/zboard/zboard.php?id=ppomppu", "전체", false),
PPOMPPU_DOMESTIC_ETC("/zboard/zboard.php?id=ppomppu&category=1", "기타", true),
PPOMPPU_DOMESTIC_COMPUTER("/zboard/zboard.php?id=ppomppu&category=4", "컴퓨터", true),
PPOMPPU_DOMESTIC_DIGITAL("/zboard/zboard.php?id=ppomppu&category=5", "디지털", true),
PPOMPPU_DOMESTIC_FOOD("/zboard/zboard.php?id=ppomppu&category=6", "식품/건강", true),
PPOMPPU_DOMESTIC_BOOK("/zboard/zboard.php?id=ppomppu&category=8", "서적", true),
PPOMPPU_DOMESTIC_APPLIANCES("/zboard/zboard.php?id=ppomppu&category=9", "가전/가구", true),
PPOMPPU_DOMESTIC_PARENTING("/zboard/zboard.php?id=ppomppu&category=10", "육아", true),
PPOMPPU_DOMESTIC_GIFTCARD("/zboard/zboard.php?id=ppomppu&category=11", "상품권", true),
PPOMPPU_DOMESTIC_CLOTHES("/zboard/zboard.php?id=ppomppu&category=12", "의류/잡화", true),
PPOMPPU_DOMESTIC_COSMETIC("/zboard/zboard.php?id=ppomppu&category=13", "화장품", true),
PPOMPPU_DOMESTIC_OUTDOOR("/zboard/zboard.php?id=ppomppu&category=15", "등산/캠핑", true),
PPOMPPU_OVERSEA_ALL("/zboard/zboard.php?id=ppomppu4", "전체", false),
PPOMPPU_OVERSEA_ETC("/zboard/zboard.php?id=ppomppu4&category=1", "기타", true),
PPOMPPU_OVERSEA_APPLIANCES("/zboard/zboard.php?id=ppomppu4&category=7", "가전", true),
PPOMPPU_OVERSEA_TVAV("/zboard/zboard.php?id=ppomppu4&category=8", "TV/영상", true),
PPOMPPU_OVERSEA_COMPUTER("/zboard/zboard.php?id=ppomppu4&category=3", "컴퓨터", true),
PPOMPPU_OVERSEA_DIGITAL("/zboard/zboard.php?id=ppomppu4&category=4", "디지털", true),
PPOMPPU_OVERSEA_MOBILEACCESSORY("/zboard/zboard.php?id=ppomppu4&category=9", "액세서리", true),
PPOMPPU_OVERSEA_CLOTHES("/zboard/zboard.php?id=ppomppu4&category=5", "의류/잡화", true),
PPOMPPU_OVERSEA_WATCH("/zboard/zboard.php?id=ppomppu4&category=2", "시계", true),
PPOMPPU_OVERSEA_SHOES("/zboard/zboard.php?id=ppomppu4&category=11", "신발", true),
PPOMPPU_OVERSEA_FOOD("/zboard/zboard.php?id=ppomppu4&category=10", "식품/건강", true),
PPOMPPU_OVERSEA_PARENTING("/zboard/zboard.php?id=ppomppu4&category=6", "육아", true),
;
private String resourcePath;
private String menuName;
private boolean crawlWithDefaultTimer;
public static final String PPOMPPU_URL = "https://www.ppomppu.co.kr/";
public static final String PPOMPPU_URL = "https://www.ppomppu.co.kr";
public static String ofViewPageUrl(String articleUrl) {
return PPOMPPU_URL + "zboard/" + articleUrl;
return PPOMPPU_URL + "/zboard/" + articleUrl;
}
}

2
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/util/ObjectMapperFactory.java → support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/util/ObjectMapperFactory.java
View File

@ -1,4 +1,4 @@
package com.myoa.engineering.crawl.ppomppu.processor.util;
package com.myoa.engineering.crawl.ppomppu.support.util;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.core.JsonParser.Feature;

19
support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/webclient/PpomppuNotifierWebClientConfiguration.java Normal file
View File

@ -0,0 +1,19 @@
package com.myoa.engineering.crawl.ppomppu.support.webclient;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.Configuration;
/**
* PpomppuNotifierWebClientConfiguration
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-11-18
*
*/
@Configuration
@ConditionalOnProperty(value = "webclient.init", havingValue = "true")
@ComponentScan(basePackageClasses = WebClientBaseScan.class)
public class PpomppuNotifierWebClientConfiguration {
}

8
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/factory/WebClientFilterFactory.java → support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/webclient/factory/WebClientFilterFactory.java
View File

@ -1,4 +1,4 @@
package com.myoa.engineering.crawl.ppomppu.processor.configuration.factory;
package com.myoa.engineering.crawl.ppomppu.support.webclient.factory;
import lombok.extern.slf4j.Slf4j;
import org.springframework.web.reactive.function.client.ClientRequest;
@ -27,8 +27,8 @@ public final class WebClientFilterFactory {
private static Mono<ClientRequest> writeRequest(ClientRequest clientRequest) {
try {
log.info("[WEBCLIENT REQUEST] uri : {} method : {} headers : {}",
clientRequest.url(), clientRequest.method(), clientRequest.headers());
log.info("[WEBCLIENT REQUEST] uri : {} method : {} headers : {}, body: {}",
clientRequest.url(), clientRequest.method(), clientRequest.headers(), clientRequest.body());
} catch (Exception e) {
log.error("[WEBCLIENT REQUEST] write request failed", e);
}
@ -37,7 +37,7 @@ public final class WebClientFilterFactory {
private static Mono<ClientResponse> writeResponse(ClientResponse clientResponse) {
try {
log.info("[WEBCLIENT REQUEST] statusCode : {} headers : {}",
log.info("[WEBCLIENT RESPONSE] statusCode : {} headers : {}",
clientResponse.rawStatusCode(), clientResponse.headers().asHttpHeaders());
} catch (Exception e) {
log.error("[WEBCLIENT RESPONSE] write response failed", e);

5
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/factory/WebFluxExchangeStragiesFactory.java → support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/webclient/factory/WebFluxExchangeStragiesFactory.java
View File

@ -1,7 +1,8 @@
package com.myoa.engineering.crawl.ppomppu.processor.configuration.factory;
package com.myoa.engineering.crawl.ppomppu.support.webclient.factory;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.myoa.engineering.crawl.ppomppu.processor.util.ObjectMapperFactory;
import com.myoa.engineering.crawl.ppomppu.support.util.ObjectMapperFactory;
import org.springframework.http.codec.json.Jackson2JsonDecoder;
import org.springframework.http.codec.json.Jackson2JsonEncoder;
import org.springframework.util.MimeTypeUtils;

44
support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/webclient/properties/WebClientProperties.java Normal file
View File

@ -0,0 +1,44 @@
package com.myoa.engineering.crawl.ppomppu.support.webclient.properties;
import java.util.ArrayList;
import java.util.List;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.lang.NonNull;
import org.springframework.stereotype.Component;
import lombok.Data;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
/**
* WebClientPropertiesUnit
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-11-18
*
*/
@NoArgsConstructor
@Setter
@Getter
@Component
@ConfigurationProperties(prefix = "webclient")
public class WebClientProperties {
private List<WebClientPropertiesUnit> units = new ArrayList<>();
@Data
public static class WebClientPropertiesUnit {
private String unitName;
private String baseUrl;
// TODO headers
}
public WebClientPropertiesUnit find(@NonNull String unitName) {
return units.stream()
.filter(x -> x.getUnitName().equals(unitName))
.findFirst()
.orElseThrow(
() -> new IllegalArgumentException("Not found properties unit. unitName : " + unitName));
}
}