MyoaEngineering
/
ShoppingCrawler
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

[PPN-211113] Implement common sender component #12

Merged
nthfuncx merged 6 commits from feature/PPN-211113 into develop 2021-11-21 13:39:22 +00:00
Conversation 0 Commits 6 Files Changed 70 +175 -74
24 changed files with 175 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit c97c8dc01f - Show all commits

2
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/H2ConsoleConfiguration.java
View File

@ -11,7 +11,7 @@ import org.springframework.context.event.ContextRefreshedEvent;
import org.springframework.context.event.EventListener;
@Slf4j
@Profile("datasource-development")
@Profile({"datasource-local", "datasource-development"})
@Configuration
public class H2ConsoleConfiguration {

4
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/controller/CrawlAPIController.java
View File

@ -48,8 +48,8 @@ public class CrawlAPIController {
Mono<List<PpomppuArticle>> articles =
ppomppuRSSFeedService.getArticles(boardName)
.doOnNext(e -> ppomppuArticleService.filterOnlyNewArticles(boardName, e))
.doOnNext(e -> ppomppuArticleService.save(boardName, e))
.doOnNext(messageSenderService::sendMessageToSlack);
.doOnNext(e -> messageSenderService.sendMessageToSlack(e))
.doOnNext(e -> ppomppuArticleService.save(boardName, e));
return articles.then(Mono.just(APIResponse.success(result.done())));
}

21
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/client/MessageSenderAPIClient.java
View File

@ -1,24 +1,26 @@
package com.myoa.engineering.crawl.ppomppu.processor.infrastructure.client;
import org.springframework.stereotype.Component;
import org.springframework.web.reactive.function.client.WebClient;
import org.springframework.web.reactive.function.client.WebClientRequestException;
import com.myoa.engineering.crawl.ppomppu.processor.dto.constant.WebClientPropertiesUnitName;
import com.myoa.engineering.crawl.ppomppu.support.dto.SimpleMessageDTO;
import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebClientFilterFactory;
import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebFluxExchangeStragiesFactory;
import com.myoa.engineering.crawl.ppomppu.support.webclient.properties.WebClientProperties;
import com.myoa.engineering.crawl.ppomppu.support.webclient.properties.WebClientProperties.WebClientPropertiesUnit;
import lombok.extern.slf4j.Slf4j;
import org.springframework.core.ParameterizedTypeReference;
import org.springframework.http.HttpHeaders;
import org.springframework.http.MediaType;
import org.springframework.stereotype.Component;
import org.springframework.web.reactive.function.client.WebClient;
import org.springframework.web.reactive.function.client.WebClientRequestException;
import reactor.core.publisher.Mono;
import reactor.core.scheduler.Schedulers;
/**
* PpomppuNotifierSenderAPIClient
*
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-11-17
*
*/
@Slf4j
@Component
@ -31,6 +33,8 @@ public class MessageSenderAPIClient {
webClientProperties.find(WebClientPropertiesUnitName.PPOMPPU_NOTIFIER_SENDER_API.getUnitName());
this.webClient = WebClient.builder()
.baseUrl(webClientPropertiesUnit.getBaseUrl())
.exchangeStrategies(WebFluxExchangeStragiesFactory.ofDefault())
.defaultHeader(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_VALUE)
.filter(WebClientFilterFactory.logRequest())
.filter(WebClientFilterFactory.logResponse())
.build();
@ -40,11 +44,12 @@ public class MessageSenderAPIClient {
return webClient.post()
.uri("/api/v1/messages/sendMessage/messengers/slack")
.bodyValue(dto)
.exchangeToMono(e -> e.bodyToMono(String.class))
.exchangeToMono(e -> e.bodyToMono(new ParameterizedTypeReference<String>() {}))
.publishOn(Schedulers.boundedElastic())
.onErrorResume(WebClientRequestException.class, t -> {
log.info("Exception occured, ignoring. : {}", t.getClass().getSimpleName());
return Mono.empty();
});
})
.doOnNext(e -> log.info("response: {} ", e));
}
}

9
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/MessageSenderService.java
View File

@ -9,6 +9,7 @@ import com.myoa.engineering.crawl.ppomppu.processor.dto.PpomppuArticleTransforme
import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.client.MessageSenderAPIClient;
import lombok.extern.slf4j.Slf4j;
import reactor.core.publisher.Mono;
/**
* MessageSenderService
@ -26,12 +27,12 @@ public class MessageSenderService {
this.messageSenderAPIClient = messageSenderAPIClient;
}
public void sendMessageToSlack(PpomppuArticle article) {
messageSenderAPIClient.sendMessageToSlack(PpomppuArticleTransformer.TRANSFORM_TO_MESSAGE_DTO.apply(article));
public Mono<String> sendMessageToSlack(PpomppuArticle article) {
return messageSenderAPIClient.sendMessageToSlack(PpomppuArticleTransformer.TRANSFORM_TO_MESSAGE_DTO.apply(article));
}
public void sendMessageToSlack(List<PpomppuArticle> articles) {
messageSenderAPIClient.sendMessageToSlack(PpomppuArticleTransformer.transform(articles));
public Mono<String> sendMessageToSlack(List<PpomppuArticle> articles) {
return messageSenderAPIClient.sendMessageToSlack(PpomppuArticleTransformer.transform(articles));
}
}

4
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/PpomppuArticleService.java
View File

@ -38,7 +38,7 @@ public class PpomppuArticleService {
}
@Transactional
public void save(PpomppuBoardName boardName, List<PpomppuArticle> articles) {
public List<PpomppuArticle> save(PpomppuBoardName boardName, List<PpomppuArticle> articles) {
Long latestArticleId = articles.stream()
.map(PpomppuArticle::getArticleId)
.max(Long::compareTo)
@ -54,6 +54,6 @@ public class PpomppuArticleService {
latestArticleId)));
// save real articles.
ppomppuArticleRepository.saveAll(articles);
return ppomppuArticleRepository.saveAll(articles);
}
}

12
processor/src/main/resources/application-local.yml Normal file
View File

@ -0,0 +1,12 @@
spring:
config:
activate:
on-profile: local
import:
- "configserver:http://localhost:20085"
- classpath:/local/webclient.yml
server:
port: 20081
# import: optional:configserver:http://localhost:11080 # can be start up even config server was not found.

3
processor/src/main/resources/application.yml
View File

@ -4,8 +4,9 @@ spring:
main:
allow-bean-definition-overriding: true
profiles:
active: ${SPRING_ACTIVE_PROFILE:development}
active: ${SPRING_ACTIVE_PROFILE:local}
group:
local: "local,datasource-local"
development: "development,datasource-development"
production: "production, datasource-production"
freemarker:

2
processor/src/main/resources/development/webclient.yml
View File

@ -2,4 +2,4 @@ webclient:
init: true
units:
- unit-name: ppn-sender-api
base-url: http://localhost:20081
base-url: http://localhost:20082

5
processor/src/main/resources/local/webclient.yml Normal file
View File

@ -0,0 +1,5 @@
webclient:
init: true
units:
- unit-name: ppn-sender-api
base-url: http://localhost:20082

4
processor/src/main/resources/logback-spring.xml
View File

@ -1,6 +1,10 @@
<?xml version="1.0" encoding="UTF-8"?>
<configuration>
<springProperty name="DEFAULT_LEVEL_CONFIG" source="log.defaultLevel" />
<springProfile name="local">
<include resource="logback/logback-development.xml" />
<logger name="org.apache.kafka" level="INFO" />
</springProfile>
<springProfile name="development">
<include resource="logback/logback-development.xml" />
<logger name="org.apache.kafka" level="INFO" />

2
processor/src/main/resources/production/webclient.yml
View File

@ -2,4 +2,4 @@ webclient:
init: true
units:
- unit-name: ppn-sender-api
base-url: http://ppn_sender:20081
base-url: http://ppn_sender:20082

7
receiver/src/main/resources/application-local.yml Normal file
View File

@ -0,0 +1,7 @@
spring:
config:
activate:
on-profile: local
import:
- classpath:/local/webclient.yml
- "configserver:http://localhost:20085"

2
receiver/src/main/resources/application.yml
View File

@ -4,7 +4,7 @@ spring:
main:
allow-bean-definition-overriding: true
profiles:
active: ${SPRING_ACTIVE_PROFILE:development}
active: ${SPRING_ACTIVE_PROFILE:local}
freemarker:
enabled: false

5
receiver/src/main/resources/local/webclient.yml Normal file
View File

@ -0,0 +1,5 @@
webclient:
init: true
units:
- unit-name: ppn-processor-api
base-url: http://localhost:20081

4
receiver/src/main/resources/logback-spring.xml
View File

@ -1,6 +1,10 @@
<?xml version="1.0" encoding="UTF-8"?>
<configuration>
<springProperty name="DEFAULT_LEVEL_CONFIG" source="log.defaultLevel" />
<springProfile name="local">
<include resource="logback/logback-development.xml" />
<logger name="org.apache.kafka" level="INFO" />
</springProfile>
<springProfile name="development">
<include resource="logback/logback-development.xml" />
<logger name="org.apache.kafka" level="INFO" />

34
sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/configuration/properties/SlackSecretProperties.java Normal file
View File

@ -0,0 +1,34 @@
package com.myoa.engineering.crawl.ppomppu.sender.configuration.properties;
import java.util.List;
import lombok.Data;
import lombok.Getter;
import lombok.Setter;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;
@Getter
@Setter
@Component
@ConfigurationProperties("infra.slack.bot")
public class SlackSecretProperties {
private List<SlackSecretPropertiesUnit> units;
@Data
public static class SlackSecretPropertiesUnit {
private String botName;
private String username;
private String iconEmoji;
private String channel;
private String token;
}
public SlackSecretPropertiesUnit find(String botUnitName) {
return units.stream()
.filter(e -> e.getBotName().equals(botUnitName))
.findFirst()
.orElseThrow(() -> new IllegalArgumentException("not found bot unit name : " + botUnitName));
}
}

19
sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/controller/MessageSenderAPIController.java
View File

@ -1,13 +1,12 @@
package com.myoa.engineering.crawl.ppomppu.sender.controller;
import com.myoa.engineering.crawl.ppomppu.sender.infrastructure.client.MongeShoppingBotSlackMessageSender;
import com.myoa.engineering.crawl.ppomppu.support.dto.APIResponse;
import com.myoa.engineering.crawl.ppomppu.support.dto.SimpleMessageDTO;
import lombok.extern.slf4j.Slf4j;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import com.myoa.engineering.crawl.ppomppu.support.dto.APIResponse;
import com.myoa.engineering.crawl.ppomppu.support.dto.SimpleMessageDTO;
import lombok.extern.slf4j.Slf4j;
import reactor.core.publisher.Mono;
/**
@ -21,11 +20,17 @@ import reactor.core.publisher.Mono;
@RequestMapping("/api/v1")
public class MessageSenderAPIController {
private final MongeShoppingBotSlackMessageSender sender;
public MessageSenderAPIController(MongeShoppingBotSlackMessageSender sender) {
this.sender = sender;
}
@PostMapping("/messages/sendMessage/messengers/slack")
public Mono<APIResponse<SimpleMessageDTO>> sendMessageToSlack(SimpleMessageDTO dto) {
log.info("received : {}, \nbody: {}", dto.getTitle(), dto.getBody());
// TODO transform
return Mono.just(APIResponse.success(dto));
return sender.sendMessage(sender.ofMessage(dto.getBody()))
.then(Mono.just(APIResponse.success(dto)));
}
}

20
sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/controller/TestAPIController.java
View File

@ -1,37 +1,29 @@
package com.myoa.engineering.crawl.ppomppu.sender.controller;
import com.myoa.engineering.crawl.ppomppu.sender.infrastructure.client.MongeShoppingBotSlackMessageSender;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import com.myoa.engineering.crawl.ppomppu.sender.dto.SlackMessageDTO;
import com.myoa.engineering.crawl.ppomppu.sender.infrastructure.client.SlackMessageSender;
import reactor.core.publisher.Mono;
/**
* TestAPIController
*
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-11-15
*
*/
@RestController
@RequestMapping("/api/v1")
public class TestAPIController {
private final SlackMessageSender sender;
private final MongeShoppingBotSlackMessageSender sender;
public TestAPIController() {
this.sender = new SlackMessageSender("xoxb-2688454277126-2695026012277-K2Ib13lKokmTiBSnSMrc0Bp2");
public TestAPIController(MongeShoppingBotSlackMessageSender sender) {
this.sender = sender;
}
@GetMapping("/test")
public Mono<String> test() {
return sender.sendMessage(SlackMessageDTO.builder()
.text("test!")
.iconEmoji("monge")
.channel("notify_shopping")
.username("shopping notifier")
.build());
return sender.sendMessage(sender.ofMessage("testtesttest!!!"));
}
}

3
sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/dto/SlackMessageDTO.java
View File

@ -33,4 +33,7 @@ public class SlackMessageDTO implements MessageDTO {
this.iconEmoji = iconEmoji;
}
public void applyText(String text) {
this.text = text;
}
}

39
sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/infrastructure/client/MongeShoppingBotSlackMessageSender.java Normal file
View File

@ -0,0 +1,39 @@
package com.myoa.engineering.crawl.ppomppu.sender.infrastructure.client;
import com.myoa.engineering.crawl.ppomppu.sender.configuration.properties.SlackSecretProperties;
import com.myoa.engineering.crawl.ppomppu.sender.configuration.properties.SlackSecretProperties.SlackSecretPropertiesUnit;
import com.myoa.engineering.crawl.ppomppu.sender.dto.SlackMessageDTO;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
@Slf4j
@Component
public class MongeShoppingBotSlackMessageSender extends SlackMessageSender {
private static final String SLACK_SECRET_UNIT_NAME = "monge_shopping_bot";
private final SlackSecretPropertiesUnit slackProperties;
// private final SlackMessageSender slackMessageSender;
public MongeShoppingBotSlackMessageSender(SlackSecretProperties slackSecretProperties) {
super(slackSecretProperties.find(SLACK_SECRET_UNIT_NAME).getToken());
this.slackProperties = slackSecretProperties.find(SLACK_SECRET_UNIT_NAME);
}
public SlackMessageDTO ofMessageTemplate() {
return SlackMessageDTO.builder()
.channel(slackProperties.getChannel())
.iconEmoji(slackProperties.getIconEmoji())
.username(slackProperties.getUsername())
.build();
}
public SlackMessageDTO ofMessage(String text) {
return SlackMessageDTO.builder()
.channel(slackProperties.getChannel())
.iconEmoji(slackProperties.getIconEmoji())
.username(slackProperties.getUsername())
.text(text)
.build();
}
}

31
sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/infrastructure/client/SlackMessageSender.java
View File

@ -53,34 +53,3 @@ public class SlackMessageSender implements MessageSender<SlackMessageDTO> {
}
}
/*
import requests
import json
# woozuyeni_watchtower
SLACK_TOKEN="xoxb-2688454277126-2695026012277-K2Ib13lKokmTiBSnSMrc0Bp2"
CHANNEL = "notify_watchtower"
CHANNEL = "common"
POST_API = "https://slack.com/api/chat.postMessage"
HEADER = "Authorization: Bearer"
headers = {
"Content-Type" : "application/json",
"Authorization" : f"Bearer {SLACK_TOKEN}"
}
data = {}
data["text"] = "몽몽! :monge_big:"
data["channel"] = CHANNEL
data["username"] = "몽이봇"
data["icon_emoji"] = ":monge_big:"
# requests.post(POST_API, data=json.dumps(data), headers=headers)
attachments = [{"title": "Cat", "image_url": image_url}]
image_url = "https://myoa-universe.com/assets/love1.png"
*/

10
sender/src/main/resources/application-local.yml Normal file
View File

@ -0,0 +1,10 @@
spring:
config:
activate:
on-profile: local
import:
- "configserver:http://localhost:20085"
# import: optional:configserver:http://localhost:11080 # can be start up even config server was not found.
server:
port: 20082

3
sender/src/main/resources/application.yml
View File

@ -4,8 +4,9 @@ spring:
main:
allow-bean-definition-overriding: true
profiles:
active: ${SPRING_ACTIVE_PROFILE:development}
active: ${SPRING_ACTIVE_PROFILE:local}
group:
local: local, slackapi-local
development: development, slackapi-development
production: production, slackapi-production
freemarker:

4
sender/src/main/resources/logback-spring.xml
View File

@ -1,5 +1,9 @@
<?xml version="1.0" encoding="UTF-8"?>
<configuration>
<springProfile name="local">
<include resource="logback/logback-development.xml" />
<logger name="org.apache.kafka" level="INFO" />
</springProfile>
<springProperty name="DEFAULT_LEVEL_CONFIG" source="log.defaultLevel" />
<springProfile name="development">
<include resource="logback/logback-development.xml" />