3 Commits

Author SHA1 Message Date
woozu-shin
7b015e8093 [NO-ISSUE] Remove reader, processor, writer 2024-05-09 08:49:11 +09:00
woozu-shin
b4accbc2c0 [NO-ISSUE] Initialize v2 2024-05-09 08:48:36 +09:00
woozu-shin
0c4be3cc05 [NO-ISSUE] Implement v2 2024-04-30 22:34:49 +09:00
166 changed files with 3025 additions and 2287 deletions

3
.gitignore vendored
View File

@@ -37,3 +37,6 @@ out/
.vscode/ .vscode/
temppassword.yml temppassword.yml
data.sql
**/src/main/resources/slack
**/src/main/resources/datasource

View File

@@ -1,13 +1,14 @@
plugins { plugins {
id 'org.springframework.boot' version '2.5.4'
id 'io.spring.dependency-management' version '1.0.11.RELEASE'
id 'java' id 'java'
id 'idea' id 'idea'
id 'org.springframework.boot' version '3.2.5'
id 'io.spring.dependency-management' version '1.1.4'
} }
group = 'com.myoa.engineering.crawl.ppomppu' group = 'com.myoa.engineering.crawl.ppomppu'
version = '1.1.1' version = '1.0.1'
sourceCompatibility = '11' sourceCompatibility = JavaVersion.VERSION_21
targetCompatibility = JavaVersion.VERSION_21
configurations { configurations {
compileOnly { compileOnly {
@@ -20,8 +21,8 @@ repositories {
} }
allprojects { allprojects {
group = 'com.myoa.engineering.crawl.ppomppu' group = 'com.myoa.engineering.crawl.shopping'
version = '1.1.1' version = '2.0.0'
apply plugin: 'java' apply plugin: 'java'
apply plugin: 'idea' apply plugin: 'idea'
@@ -36,7 +37,7 @@ allprojects {
} }
ext { ext {
set('springCloudVersion', "2020.0.4") set('springCloudVersion', "2023.0.1")
} }
dependencyManagement { dependencyManagement {

View File

@@ -1,3 +0,0 @@
xcopy /y .\processor\build\libs\*.jar .\
xcopy /y .\receiver\build\libs\*.jar .\
xcopy /y .\sender\build\libs\*.jar .\

View File

@@ -1,5 +1,6 @@
#Sun Apr 28 23:47:38 KST 2024
distributionBase=GRADLE_USER_HOME distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists distributionPath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-7.1.1-bin.zip distributionUrl=https\://services.gradle.org/distributions/gradle-8.5-bin.zip
zipStoreBase=GRADLE_USER_HOME zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists zipStorePath=wrapper/dists

0
gradlew vendored Normal file → Executable file
View File

View File

@@ -1,30 +0,0 @@
dependencies {
developmentOnly 'org.springframework.boot:spring-boot-devtools'
runtimeOnly 'com.h2database:h2'
runtimeOnly 'mysql:mysql-connector-java'
compileOnly 'org.projectlombok:lombok'
implementation project(':support')
// https://projectreactor.io/docs/core/release/reference/#debug-activate
implementation 'org.springframework.boot:spring-boot-starter-webflux'
implementation 'org.springframework.boot:spring-boot-starter-data-jpa'
implementation 'org.springframework.boot:spring-boot-starter-jdbc'
implementation 'org.springframework.cloud:spring-cloud-starter-config'
implementation 'org.springframework.boot:spring-boot-starter-actuator'
implementation 'com.rometools:rome:1.16.0'
implementation 'org.jsoup:jsoup:1.14.2'
implementation 'com.h2database:h2:1.4.200'
annotationProcessor 'org.springframework.boot:spring-boot-configuration-processor'
annotationProcessor 'org.projectlombok:lombok'
testImplementation 'org.springframework.boot:spring-boot-starter-test'
}
test {
useJUnitPlatform()
testLogging {
events "passed", "skipped", "failed"
}
}

View File

@@ -1,23 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.processor;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.context.annotation.Import;
import com.myoa.engineering.crawl.ppomppu.support.webclient.PpomppuNotifierWebClientConfiguration;
/**
* ProcessorApplication
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-08-20
*
*/
@Import({ PpomppuNotifierWebClientConfiguration.class })
@SpringBootApplication
public class ProcessorApplication {
public static void main(String[] args) {
SpringApplication.run(ProcessorApplication.class, args);
}
}

View File

@@ -1,70 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.processor.controller;
import java.util.List;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
import com.myoa.engineering.crawl.ppomppu.processor.dto.FeedParsedResult;
import com.myoa.engineering.crawl.ppomppu.processor.service.MessageSenderService;
import com.myoa.engineering.crawl.ppomppu.processor.service.PpomppuArticleService;
import com.myoa.engineering.crawl.ppomppu.processor.service.PpomppuFeedService;
import com.myoa.engineering.crawl.ppomppu.support.dto.APIResponse;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import lombok.extern.slf4j.Slf4j;
import reactor.core.publisher.Mono;
/**
* CrawlAPIController
*
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-09-05
*/
@Slf4j
@RestController
@RequestMapping("/api/v1/crawl")
public class CrawlAPIController {
private final PpomppuFeedService ppomppuRSSFeedService;
private final PpomppuArticleService ppomppuArticleService;
private final MessageSenderService messageSenderService;
public CrawlAPIController(PpomppuFeedService ppomppuRSSFeedService,
PpomppuArticleService ppomppuArticleService,
MessageSenderService messageSenderService) {
this.ppomppuRSSFeedService = ppomppuRSSFeedService;
this.ppomppuArticleService = ppomppuArticleService;
this.messageSenderService = messageSenderService;
}
@PostMapping("/boards/{boardName}")
public Mono<APIResponse<FeedParsedResult>> crawlBoard(@PathVariable("boardName") PpomppuBoardName boardName) {
log.info("got request... {}", boardName);
FeedParsedResult result = FeedParsedResult.of(boardName);
Mono<String> publishedMessages =
ppomppuRSSFeedService.getArticles(boardName)
.map(e -> ppomppuArticleService.filterOnlyNewArticles(boardName, e))
.map(e -> ppomppuArticleService.save(boardName, e))
.filter(e -> !e.isEmpty())
.flatMap(e -> messageSenderService.sendBlockMessageToSlack(boardName, e));
return publishedMessages.then(Mono.just(APIResponse.success(result.done())));
}
@PostMapping("/exploit/boards/{boardName}")
public Mono<APIResponse<String>> crawlBoardDryRun(
@PathVariable("boardName") PpomppuBoardName boardName) {
log.info("got request... {}", boardName);
Mono<String> publishedMessages =
ppomppuRSSFeedService.getArticles(boardName)
.flatMap(e -> messageSenderService.sendBlockMessageToSlack(boardName, e));
return publishedMessages.map(APIResponse::success);
}
}

View File

@@ -1,32 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.processor.domain;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.EnumType;
import javax.persistence.Enumerated;
import javax.persistence.GeneratedValue;
import javax.persistence.GenerationType;
import javax.persistence.Id;
import javax.persistence.Table;
import lombok.Getter;
import lombok.NoArgsConstructor;
@Getter
@NoArgsConstructor
@Entity
@Table(name = "subscribed_board")
public class SubscribedBoard extends Auditable {
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
private Long id;
@Column
private Long userId;
@Column
@Enumerated(EnumType.STRING)
private PpomppuBoardName boardName;
}

View File

@@ -1,66 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.processor.dto;
import java.time.Instant;
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
import com.myoa.engineering.crawl.ppomppu.support.util.DateUtil;
import lombok.Builder;
import lombok.Getter;
import lombok.NoArgsConstructor;
/**
* PpomppuArticleParseDTO
*
* @author Shin Woo-jin (woozu.shin@kakaoent.com)
* @since 2021-09-08
*/
@Getter
@NoArgsConstructor
public class PpomppuArticleParseDTO {
private String id;
private String articleId;
private String boardName;
private String articleUrl;
private String thumbnailUrl;
private String title;
private String hit;
private Integer recommended;
private String registeredAt;
@Builder
public PpomppuArticleParseDTO(String id, String articleId, String boardName, String articleUrl,
String thumbnailUrl, String title, String hit, Integer recommended,
String registeredAt) {
this.id = id;
this.articleId = articleId;
this.boardName = boardName;
this.articleUrl = articleUrl;
this.thumbnailUrl = thumbnailUrl;
this.title = title;
this.hit = hit;
this.recommended = recommended;
this.registeredAt = registeredAt;
}
public boolean isInValidated() {
return articleId == null || articleId.isEmpty()
|| hit == null || hit.isEmpty();
}
public PpomppuArticle convert() {
if (isInValidated()) {
throw new IllegalArgumentException("PpomppuArticleParseDTO was invalidated");
}
return PpomppuArticle.builder()
.articleId(Long.parseLong(articleId))
.title(title)
.articleUrl(articleUrl)
.thumbnailUrl(thumbnailUrl)
.recommended(recommended)
.hit(Integer.parseInt(hit))
.registeredAt(DateUtil.DATE_TIME_FORMATTER.parse(registeredAt, Instant::from))
.build();
}
}

View File

@@ -1,78 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.processor.dto;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
/**
* PpomppuArticleTransformer
*
* @author Shin Woo-jin (woozu.shin@kakaoent.com)
* @since 2021-09-08
*/
public final class PpomppuArticleParser {
private PpomppuArticleParser() {}
public static PpomppuArticle toArticle(Elements articleElement) {
final String articleIdString = PpomppuArticleParser.parseArticleId(articleElement.get(0));
final String title = PpomppuArticleParser.parseTitle(articleElement.get(2));
final String articleUrl = PpomppuArticleParser.parseArticleUrl(articleElement.get(2));
final String thumbnailUrl = PpomppuArticleParser.parseThumbnailUrl(articleElement.get(3));
final Integer recommended = PpomppuArticleParser.parseRecommended(articleElement.get(6));
final String hitString = PpomppuArticleParser.parseHit(articleElement.get(7));
final String registeredAtString = PpomppuArticleParser.parseRegisteredAt(articleElement.get(5));
return PpomppuArticleParseDTO.builder()
.articleId(articleIdString)
.title(title)
.articleUrl(articleUrl)
.thumbnailUrl(thumbnailUrl)
.recommended(recommended)
.hit(hitString)
.registeredAt(registeredAtString)
.build()
.convert();
}
public static String parseArticleId(Element td) {
return td.text().trim();
}
public static String parseTitle(Element td) {
return td.getElementsByTag("a").text();
}
public static String parseArticleUrl(Element td) {
return PpomppuBoardName.ofViewPageUrl(td.getElementsByTag("a").attr("href"));
}
public static String parseThumbnailUrl(Element td) {
return "https:" + td.getElementsByTag("img").get(0).attr("src");
}
public static Integer parseRecommended(Element td) {
final String voteString = td.text();
final int recommended;
if (voteString.isEmpty()) {
recommended = 0;
} else {
final int voteUp = Integer.parseInt(td.text().split(" - ")[0]);
final int voteDown = Integer.parseInt(td.text().split(" - ")[1]);
recommended = voteUp - voteDown;
}
return recommended;
}
public static String parseHit(Element td) {
return td.text();
}
public static String parseRegisteredAt(Element td) {
return td.attr("title");
}
}

View File

@@ -1,70 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.processor.dto;
import java.time.Instant;
import java.util.List;
import java.util.function.Function;
import java.util.stream.Collectors;
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
import com.myoa.engineering.crawl.ppomppu.support.dto.BlockMessageDTO;
import com.myoa.engineering.crawl.ppomppu.support.dto.SimpleMessageDTO;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import com.myoa.engineering.crawl.ppomppu.support.util.DateUtil;
/**
* PpomppuArticleTransformer
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-11-21
*
*/
public final class PpomppuArticleTransformer {
private PpomppuArticleTransformer() {}
private static final String MESSAGE_FORMAT_V1 = "%s)) <%s|LINK> `%s` ";
private static final String MESSAGE_FORMAT_V2 = "%s *<%s|LINK>*\n%s";
private static final String TITLE_FORMAT_V1 = "_*:hearts: %s | %s*_";
public static final Function<PpomppuArticle, SimpleMessageDTO> TRANSFORM_TO_MESSAGE_DTO = article ->
SimpleMessageDTO.builder()
.requestedAt(Instant.now())
.publishedAt(article.getRegisteredAt())
.title(String.format(MESSAGE_FORMAT_V1,
article.getBoardName().getMenuName(), article.getArticleUrl(),
article.getTitle()))
.body(article.getArticleUrl())
.build();
// https://stackoverflow.com/questions/24882927/using-streams-to-convert-a-list-of-objects-into-a-string-obtained-from-the-tostr
public static SimpleMessageDTO transformToSimpleMessage(List<PpomppuArticle> articles) {
Instant requestedAt = Instant.now();
String body = articles.stream()
.map(PpomppuArticleTransformer::convertToInlineMessage)
.collect(Collectors.joining("\n\n"));
return SimpleMessageDTO.builder()
.requestedAt(requestedAt)
.title(DateUtil.DATE_TIME_FORMATTER.format(requestedAt))
.body(body)
.build();
}
public static BlockMessageDTO transformToBlockMessage(PpomppuBoardName boardName, List<PpomppuArticle> articles) {
Instant requestedAt = Instant.now();
List<BlockMessageDTO.Block> body = articles.stream()
.map(e -> BlockMessageDTO.createBlock(convertToInlineMessage(e),
e.getThumbnailUrl()))
.collect(Collectors.toList());
return BlockMessageDTO.builder()
.requestedAt(requestedAt)
.title(String.format(TITLE_FORMAT_V1,
boardName.getMenuName(),
DateUtil.DATE_TIME_FORMATTER.format(requestedAt)))
.blocks(body)
.build();
}
public static String convertToInlineMessage(PpomppuArticle article) {
return String.format(MESSAGE_FORMAT_V2,
article.getBoardName().getMenuName(), article.getArticleUrl(), article.getTitle());
}
}

View File

@@ -1,70 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.processor.infrastructure.client;
import org.springframework.core.ParameterizedTypeReference;
import org.springframework.http.HttpHeaders;
import org.springframework.http.MediaType;
import org.springframework.stereotype.Component;
import org.springframework.web.reactive.function.client.WebClient;
import org.springframework.web.reactive.function.client.WebClientRequestException;
import com.myoa.engineering.crawl.ppomppu.processor.dto.constant.WebClientPropertiesUnitName;
import com.myoa.engineering.crawl.ppomppu.support.dto.BlockMessageDTO;
import com.myoa.engineering.crawl.ppomppu.support.dto.SimpleMessageDTO;
import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebClientFilterFactory;
import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebFluxExchangeStragiesFactory;
import com.myoa.engineering.crawl.ppomppu.support.webclient.properties.WebClientProperties;
import com.myoa.engineering.crawl.ppomppu.support.webclient.properties.WebClientProperties.WebClientPropertiesUnit;
import lombok.extern.slf4j.Slf4j;
import reactor.core.publisher.Mono;
import reactor.core.scheduler.Schedulers;
/**
* PpomppuNotifierSenderAPIClient
*
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-11-17
*/
@Slf4j
@Component
public class MessageSenderAPIClient {
private final WebClient webClient;
public MessageSenderAPIClient(WebClientProperties webClientProperties) {
WebClientPropertiesUnit webClientPropertiesUnit =
webClientProperties.find(WebClientPropertiesUnitName.PPOMPPU_NOTIFIER_SENDER_API.getUnitName());
this.webClient = WebClient.builder()
.baseUrl(webClientPropertiesUnit.getBaseUrl())
.exchangeStrategies(WebFluxExchangeStragiesFactory.ofDefault())
.defaultHeader(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_VALUE)
// .filter(WebClientFilterFactory.logRequest())
// .filter(WebClientFilterFactory.logResponse())
.build();
}
public Mono<String> sendSimpleMessageToSlack(SimpleMessageDTO dto) {
return webClient.post()
.uri("/api/v1/messages/sendSimpleMessage/messengers/slack")
.bodyValue(dto)
.exchangeToMono(e -> e.bodyToMono(new ParameterizedTypeReference<String>() {}))
.publishOn(Schedulers.boundedElastic())
.onErrorResume(WebClientRequestException.class, t -> {
log.info("Exception occured, ignoring. : {}", t.getClass().getSimpleName());
return Mono.empty();
});
}
public Mono<String> sendBlockMessageToSlack(BlockMessageDTO dto) {
return webClient.post()
.uri("/api/v1/messages/sendBlockMessage/messengers/slack")
.bodyValue(dto)
.exchangeToMono(e -> e.bodyToMono(new ParameterizedTypeReference<String>() {}))
.publishOn(Schedulers.boundedElastic())
.onErrorResume(WebClientRequestException.class, t -> {
log.info("Exception occured, ignoring. : {}", t.getClass().getSimpleName());
return Mono.empty();
});
}
}

View File

@@ -1,4 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository;
public interface BaseScanRepository {
}

View File

@@ -1,44 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.processor.service;
import java.util.List;
import org.springframework.stereotype.Service;
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
import com.myoa.engineering.crawl.ppomppu.processor.dto.PpomppuArticleTransformer;
import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.client.MessageSenderAPIClient;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import lombok.extern.slf4j.Slf4j;
import reactor.core.publisher.Mono;
/**
* MessageSenderService
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-11-21
*
*/
@Slf4j
@Service
public class MessageSenderService {
private final MessageSenderAPIClient messageSenderAPIClient;
public MessageSenderService(MessageSenderAPIClient messageSenderAPIClient) {
this.messageSenderAPIClient = messageSenderAPIClient;
}
public Mono<String> sendSimpleMessageToSlack(PpomppuArticle article) {
return messageSenderAPIClient.sendSimpleMessageToSlack(PpomppuArticleTransformer.TRANSFORM_TO_MESSAGE_DTO.apply(article));
}
public Mono<String> sendSimpleMessageToSlack(List<PpomppuArticle> articles) {
return messageSenderAPIClient.sendSimpleMessageToSlack(PpomppuArticleTransformer.transformToSimpleMessage(articles));
}
public Mono<String> sendBlockMessageToSlack(PpomppuBoardName boardName, List<PpomppuArticle> articles) {
return messageSenderAPIClient.sendBlockMessageToSlack(
PpomppuArticleTransformer.transformToBlockMessage(boardName, articles));
}
}

View File

@@ -1,11 +0,0 @@
spring:
config:
activate:
on-profile: local
import:
- "configserver:http://localhost:20085"
server:
port: 20081
# import: optional:configserver:http://localhost:11080 # can be start up even config server was not found.

View File

@@ -1,26 +0,0 @@
dependencies {
developmentOnly 'org.springframework.boot:spring-boot-devtools'
runtimeOnly 'com.h2database:h2'
runtimeOnly 'mysql:mysql-connector-java'
compileOnly 'org.projectlombok:lombok'
implementation project(':support')
// https://projectreactor.io/docs/core/release/reference/#debug-activate
implementation 'org.springframework.boot:spring-boot-starter-webflux'
implementation 'org.springframework.boot:spring-boot-configuration-processor'
implementation 'org.springframework.cloud:spring-cloud-starter-config'
implementation 'org.springframework.boot:spring-boot-starter-actuator'
implementation 'org.telegram:telegrambots:5.3.0'
annotationProcessor 'org.springframework.boot:spring-boot-configuration-processor'
annotationProcessor 'org.projectlombok:lombok'
testImplementation 'org.springframework.boot:spring-boot-starter-test'
}
test {
useJUnitPlatform()
testLogging {
events "passed", "skipped", "failed"
}
}

View File

@@ -1,25 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.receiver;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.annotation.Import;
import com.myoa.engineering.crawl.ppomppu.receiver.configuration.properties.TelegramBotProperties;
import com.myoa.engineering.crawl.ppomppu.support.webclient.PpomppuNotifierWebClientConfiguration;
/**
* ReceiverApplication
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-08-20
*
*/
@Import({ PpomppuNotifierWebClientConfiguration.class})
@SpringBootApplication
// @EnableConfigurationProperties({ TelegramBotProperties.class })
public class ReceiverApplication {
public static void main(String[] args) {
SpringApplication.run(ReceiverApplication.class, args);
}
}

View File

@@ -1,39 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.receiver.configuration;
import java.util.List;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.telegram.telegrambots.meta.TelegramBotsApi;
import org.telegram.telegrambots.meta.exceptions.TelegramApiException;
import org.telegram.telegrambots.updatesreceivers.DefaultBotSession;
import com.myoa.engineering.crawl.ppomppu.receiver.configuration.properties.TelegramBotProperties;
import com.myoa.engineering.crawl.ppomppu.receiver.configuration.properties.TelegramBotProperties.TelegramBotPropertiesUnit;
import com.myoa.engineering.crawl.ppomppu.receiver.dispatch.MessageDispatcher;
import com.myoa.engineering.crawl.ppomppu.receiver.handler.message.MessageHandler;
/**
* TelegramBotConfiguration
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-08-21
*
*/
@Configuration
public class TelegramBotConfiguration {
private static final String BOT_PROPERTIES_UNIT_NAME = "ppomppu_notify_bot";
@Bean
public TelegramBotsApi telegramBotsApi(MessageDispatcher messageDispatcher) throws TelegramApiException {
TelegramBotsApi api = new TelegramBotsApi(DefaultBotSession.class);
api.registerBot(messageDispatcher);
return api;
}
@Bean
public MessageDispatcher messageDispatcher(List<MessageHandler> messageHandlers,
TelegramBotProperties telegramBotProperties) {
TelegramBotPropertiesUnit propertiesUnit = telegramBotProperties.find(BOT_PROPERTIES_UNIT_NAME);
return new MessageDispatcher(messageHandlers, propertiesUnit.getName(), propertiesUnit.getToken());
}
}

View File

@@ -1,44 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.receiver.configuration.properties;
import java.util.ArrayList;
import java.util.List;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.boot.context.properties.ConstructorBinding;
import org.springframework.stereotype.Component;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
/**
* TelegramBotProperties
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-09-05
*
*/
@Component
@Setter
@Getter
@ConfigurationProperties(prefix = "infra.telegram.bot")
public class TelegramBotProperties {
private List<TelegramBotPropertiesUnit> units = new ArrayList<>();
@Data
public static class TelegramBotPropertiesUnit {
private String unitName;
private String name;
private String token;
}
public TelegramBotPropertiesUnit find(String unitName) {
return units.stream()
.filter(e -> e.getUnitName().equals(unitName))
.findFirst()
.orElseThrow(
() -> new IllegalArgumentException(this.getClass().getName() + ": unitName Not found. " + unitName));
}
}

View File

@@ -1,33 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.receiver.controller.v1;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import com.myoa.engineering.crawl.ppomppu.receiver.service.ProcessorAPIService;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import reactor.core.publisher.Mono;
/**
* EventAPIController
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-09-05
*
*/
@RestController
@RequestMapping("/api/v1/")
public class EventAPIController {
private final ProcessorAPIService processorAPIService;
public EventAPIController(ProcessorAPIService processorAPIService) {
this.processorAPIService = processorAPIService;
}
@PostMapping("/exploit/parsePpomppuRSS/{boardName}")
public Mono<String> parsePpomppuRSS(@PathVariable("boardName") PpomppuBoardName boardName) {
return processorAPIService.emitParseEvent(boardName);
}
}

View File

@@ -1,50 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.receiver.dispatch;
import java.util.List;
import org.telegram.telegrambots.bots.TelegramLongPollingBot;
import org.telegram.telegrambots.meta.api.objects.Message;
import org.telegram.telegrambots.meta.api.objects.Update;
import com.myoa.engineering.crawl.ppomppu.receiver.handler.message.MessageHandler;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class MessageDispatcher extends TelegramLongPollingBot {
private final List<MessageHandler> messageHandlers;
private final String botName;
private final String botToken;
public MessageDispatcher(List<MessageHandler> messageHandlers, String botName, String botToken) {
this.messageHandlers = messageHandlers;
this.botName = botName;
this.botToken = botToken;
}
@Override
public String getBotToken() {
return botToken;
}
@Override
public void onUpdateReceived(Update update) {
Message message = update.getMessage();
MessageHandler handler = getMessageHandler(message);
handler.handle(message);
}
private MessageHandler getMessageHandler(Message message) {
return messageHandlers.stream()
.filter(e -> e.isApplicable(message))
.findFirst()
.orElseThrow(() -> new IllegalArgumentException("Can not found applicable handler"));
}
@Override
public String getBotUsername() {
return botName;
}
}

View File

@@ -1,25 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.receiver.dto;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import java.io.Serializable;
import java.time.Instant;
import lombok.Getter;
import lombok.NoArgsConstructor;
/**
* FeedParsedResult
*
* @author Shin Woo-jin (woozu.shin@kakaoent.com)
* @since 2021-09-08
*/
@Getter
@NoArgsConstructor
public class FeedParsedResult implements Serializable {
private static final long serialVersionUID = -3771310078623481348L;
private PpomppuBoardName boardName;
private Instant requestedAt;
private Instant processedAt;
}

View File

@@ -1,19 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.receiver.dto.constant;
import lombok.AllArgsConstructor;
import lombok.Getter;
/**
* WebClientPropertiesUnitName
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-11-18
*
*/
@Getter
@AllArgsConstructor
public enum WebClientPropertiesUnitName {
PPOMPPU_NOTIFIER_PROCESSOR_API("ppn-processor-api"),
;
private String unitName;
}

View File

@@ -1,21 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.receiver.handler.message;
import com.myoa.engineering.crawl.ppomppu.support.util.ObjectUtil;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import org.telegram.telegrambots.meta.api.objects.Message;
@Slf4j
@Component
public class HelloWorldMessageHandler implements MessageHandler {
@Override
public boolean isApplicable(Message message) {
return ObjectUtil.isEmpty(message);
}
@Override
public void handle(Message message) {
// skip empty event message.
}
}

View File

@@ -1,12 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.receiver.handler.message;
/**
* ImageHandler
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-08-21
*
*/
public interface ImageMessageHandler extends MessageHandler {
}

View File

@@ -1,17 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.receiver.handler.message;
import org.telegram.telegrambots.meta.api.objects.Message;
/**
* MessageHandler
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-08-21
*
*/
public interface MessageHandler {
boolean isApplicable(Message message);
void handle(Message message);
}

View File

@@ -1,18 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.receiver.handler.message;
import com.myoa.engineering.crawl.ppomppu.support.util.ObjectUtil;
import org.telegram.telegrambots.meta.api.objects.Message;
/**
* TextMessageHandler
*
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-08-21
*/
public interface TextMessageHandler extends MessageHandler {
@Override
default boolean isApplicable(Message message) {
return ObjectUtil.isNotEmpty(message) && message.isUserMessage() && message.hasText();
}
}

View File

@@ -1,48 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.receiver.handler.message.text;
import java.util.List;
import org.springframework.stereotype.Component;
import org.telegram.telegrambots.meta.api.objects.Message;
import com.myoa.engineering.crawl.ppomppu.receiver.handler.message.TextMessageHandler;
import lombok.extern.slf4j.Slf4j;
/**
* CommandHandler
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-08-21
*
*/
@Slf4j
@Component
public class CommandHandler implements TextMessageHandler {
private final List<TextCommandProcessor> processors;
public CommandHandler(List<TextCommandProcessor> processors) {
this.processors = processors;
}
@Override
public boolean isApplicable(Message message) {
return TextMessageHandler.super.isApplicable(message)
&& message.isCommand(); // && message.getText().startsWith("/");
}
@Override
public void handle(Message message) {
log.info("CommandHandler : {}", message.getText());
TextCommandCode commandCode = TextCommandCode.find(message.getText());
TextCommandProcessor applicableProcessor = getApplicableProcessor(commandCode);
applicableProcessor.process(message);
}
private TextCommandProcessor getApplicableProcessor(TextCommandCode commandCode) {
return processors.stream()
.filter(e -> e.isApplicable(commandCode))
.findFirst()
.orElseThrow(() -> new IllegalArgumentException("Can not found"));
}
}

View File

@@ -1,30 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.receiver.handler.message.text;
import org.springframework.stereotype.Component;
import org.telegram.telegrambots.meta.api.objects.Message;
import com.myoa.engineering.crawl.ppomppu.receiver.handler.message.TextMessageHandler;
import lombok.extern.slf4j.Slf4j;
/**
* NormalTextHandler
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-08-21
*
*/
@Slf4j
@Component
public class CommonTextHandler implements TextMessageHandler {
@Override
public boolean isApplicable(Message message) {
return TextMessageHandler.super.isApplicable(message) && message.isCommand() == false;
}
@Override
public void handle(Message message) {
log.info("CommonTextHandler : {}", message.getText());
}
}

View File

@@ -1,24 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.receiver.handler.message.text;
import org.springframework.stereotype.Component;
import org.telegram.telegrambots.meta.api.objects.Message;
/**
* EmptyTextCommandProcessor
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-09-05
*
*/
@Component
public class EmptyTextCommandProcessor implements TextCommandProcessor {
@Override
public boolean isApplicable(TextCommandCode commandCode) {
return commandCode == TextCommandCode.EMPTY;
}
@Override
public void process(Message message) {
}
}

View File

@@ -1,27 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.receiver.handler.message.text;
import org.springframework.stereotype.Component;
import org.telegram.telegrambots.meta.api.objects.Message;
import lombok.extern.slf4j.Slf4j;
/**
* StartCommandProcessor
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-09-05
*
*/
@Slf4j
@Component
public class StartTextCommandProcessor implements TextCommandProcessor {
@Override
public boolean isApplicable(TextCommandCode commandCode) {
return TextCommandCode.START == commandCode;
}
@Override
public void process(Message message) {
log.info("[process] user: {}, command: {}", message.getChatId(), message.getText());
}
}

View File

@@ -1,33 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.receiver.handler.message.text;
import java.util.Arrays;
import lombok.Getter;
import lombok.NoArgsConstructor;
/**
* CommandTextCode
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-09-05
*
*/
@Getter
@NoArgsConstructor
public enum TextCommandCode {
EMPTY(null),
START("/start"),
;
private String value;
TextCommandCode(String value) {
this.value = value;
}
public static TextCommandCode find(String value) {
return Arrays.stream(TextCommandCode.values())
.filter(e -> e != EMPTY)
.filter(e -> value.startsWith(e.getValue()))
.findFirst()
.orElse(TextCommandCode.EMPTY);
}
}

View File

@@ -1,17 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.receiver.handler.message.text;
import org.telegram.telegrambots.meta.api.objects.Message;
/**
* TextCommandProcessor
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-09-05
*
*/
public interface TextCommandProcessor {
boolean isApplicable(TextCommandCode commandCode);
void process(Message message);
}

View File

@@ -1,44 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.receiver.infrastructure.client;
import org.springframework.core.ParameterizedTypeReference;
import org.springframework.stereotype.Component;
import org.springframework.web.reactive.function.client.WebClient;
import org.springframework.web.reactive.function.client.WebClientRequestException;
import com.myoa.engineering.crawl.ppomppu.receiver.dto.constant.WebClientPropertiesUnitName;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebClientFilterFactory;
import com.myoa.engineering.crawl.ppomppu.support.webclient.properties.WebClientProperties;
import com.myoa.engineering.crawl.ppomppu.support.webclient.properties.WebClientProperties.WebClientPropertiesUnit;
import lombok.extern.slf4j.Slf4j;
import reactor.core.publisher.Mono;
import reactor.core.scheduler.Schedulers;
@Slf4j
@Component
public class ProcessorAPIWebClient {
private final WebClient webClient;
public ProcessorAPIWebClient(WebClientProperties webClientProperties) {
WebClientPropertiesUnit webClientPropertiesUnit =
webClientProperties.find(WebClientPropertiesUnitName.PPOMPPU_NOTIFIER_PROCESSOR_API.getUnitName());
this.webClient = WebClient.builder()
.baseUrl(webClientPropertiesUnit.getBaseUrl())
.filter(WebClientFilterFactory.logRequest())
.filter(WebClientFilterFactory.logResponse())
.build();
}
public Mono<String> emitParseEvent(PpomppuBoardName boardName) {
return webClient.post()
.uri("/api/v1/crawl/boards/{boardName}", boardName)
.exchangeToMono(e -> e.bodyToMono(new ParameterizedTypeReference<String>() {}))
.publishOn(Schedulers.boundedElastic())
.onErrorResume(WebClientRequestException.class, t -> {
log.info("Exception occured, ignoring. : {}", t.getClass().getSimpleName());
return Mono.empty();
});
}
}

View File

@@ -1,38 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.receiver.scheduler;
import java.util.Arrays;
import org.springframework.scheduling.annotation.EnableScheduling;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import com.myoa.engineering.crawl.ppomppu.receiver.service.ProcessorAPIService;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import lombok.extern.slf4j.Slf4j;
/**
* ParseEventEmitter
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-09-05
*
*/
@Slf4j
@Component
@EnableScheduling
public class ParseEventEmitter {
private final ProcessorAPIService processorAPIService;
public ParseEventEmitter(ProcessorAPIService processorAPIService) {
this.processorAPIService = processorAPIService;
}
@Scheduled(fixedRate = 600 * 1000L)
public void emitBoards() {
log.info("[emitDomesticBoard] trigger fired!");
Arrays.stream(PpomppuBoardName.values())
.filter(PpomppuBoardName::isCrawlWithDefaultTimer)
.forEach(boardName -> processorAPIService.emitParseEvent(boardName).block());
}
}

View File

@@ -1,30 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.receiver.service;
import org.springframework.stereotype.Service;
import com.myoa.engineering.crawl.ppomppu.receiver.infrastructure.client.ProcessorAPIWebClient;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import lombok.extern.slf4j.Slf4j;
import reactor.core.publisher.Mono;
/**
* ProcessorAPIService
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-09-05
*
*/
@Slf4j
@Service
public class ProcessorAPIService {
private final ProcessorAPIWebClient processorAPIWebClient;
public ProcessorAPIService(ProcessorAPIWebClient processorAPIWebClient) {
this.processorAPIWebClient = processorAPIWebClient;
}
public Mono<String> emitParseEvent(PpomppuBoardName boardName) {
return processorAPIWebClient.emitParseEvent(boardName);
}
}

View File

@@ -1,6 +0,0 @@
spring:
config:
activate:
on-profile: development
import:
- "configserver:http://192.168.0.100:11080"

View File

@@ -1,6 +0,0 @@
spring:
config:
activate:
on-profile: local
import:
- "configserver:http://localhost:20085"

View File

@@ -1,6 +0,0 @@
spring:
config:
activate:
on-profile: production
import:
- "configserver:http://ppn-config-server:20080"

View File

@@ -1,25 +0,0 @@
spring:
application:
name: ppn-receiver
main:
allow-bean-definition-overriding: true
profiles:
active: ${SPRING_ACTIVE_PROFILE:local}
group:
local: "local,webclient-local"
development: "development,webclient-development"
production: "production,webclient-production"
freemarker:
enabled: false
server:
port: 20080
error:
whitelabel:
enabled: false
management:
endpoints:
web:
exposure:
include: refresh,health

View File

@@ -1,15 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<configuration>
<springProperty name="DEFAULT_LEVEL_CONFIG" source="log.defaultLevel" />
<springProfile name="local">
<include resource="logback/logback-development.xml" />
<logger name="org.apache.kafka" level="INFO" />
</springProfile>
<springProfile name="development">
<include resource="logback/logback-development.xml" />
<logger name="org.apache.kafka" level="INFO" />
</springProfile>
<springProfile name="production">
<include resource="logback/logback-production.xml" />
</springProfile>
</configuration>

View File

@@ -1,23 +0,0 @@
<included>
<property name="FILE_LOG_PATTERN"
value="%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{45}:%L - %msg%n" />
<property name="LOG_FILE_BASE" value="lcp-benefit-benefit-api" />
<appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${DIRECTORY}/${LOG_FILE_BASE}_log</file>
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<fileNamePattern>${DIRECTORY}/${LOG_FILE_BASE}_log.%d{yyyyMMdd}.%i</fileNamePattern>
<timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
<maxFileSize>1000MB</maxFileSize>
</timeBasedFileNamingAndTriggeringPolicy>
<maxHistory>60</maxHistory>
</rollingPolicy>
<encoder>
<pattern>${FILE_LOG_PATTERN}</pattern>
<immediateFlush>${IMMEDIATE_FLUSH}</immediateFlush>
</encoder>
</appender>
<appender name="ASYNC_FILE" class="ch.qos.logback.classic.AsyncAppender">
<queueSize>1024</queueSize>
<appender-ref ref="FILE" />
</appender>
</included>

View File

@@ -1,19 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<included>
<!-- =========== property BETA ========= -->
<property name="DEFAULT_LEVEL" value="${DEFAULT_LEVEL_CONFIG:-INFO}"/>
<!--file-->
<property name="DIRECTORY" value="/home1/www/logs/supervisor"/>
<property name="IMMEDIATE_FLUSH" value="true"/>
<!--nelo2-->
<property name="NELO2_LEVEL" value="WARN"/>
<!-- =========== include appender =========== -->
<include resource="org/springframework/boot/logging/logback/defaults.xml"/>
<include resource="org/springframework/boot/logging/logback/console-appender.xml"/>
<include resource="logback/component/logback-nelo2.xml"/>
<include resource="logback/component/logback-datachain.xml"/>
<!-- =========== root logger ============== -->
<root level="${DEFAULT_LEVEL}">
<appender-ref ref="CONSOLE"/>
</root>
</included>

View File

@@ -1,19 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<included>
<!-- =========== property RELEASE ========= -->
<property name="DEFAULT_LEVEL" value="${DEFAULT_LEVEL_CONFIG:-INFO}"/>
<!--file-->
<property name="DIRECTORY" value="/home1/www/logs/supervisor"/>
<property name="IMMEDIATE_FLUSH" value="true"/>
<!--nelo2-->
<property name="NELO2_LEVEL" value="WARN"/>
<!-- =========== include appender =========== -->
<include resource="org/springframework/boot/logging/logback/defaults.xml"/>
<include resource="org/springframework/boot/logging/logback/console-appender.xml"/>
<include resource="logback/component/logback-nelo2.xml"/>
<include resource="logback/component/logback-datachain.xml"/>
<!-- =========== root logger ============== -->
<root level="${DEFAULT_LEVEL}">
<appender-ref ref="CONSOLE"/>
</root>
</included>

View File

@@ -1,23 +0,0 @@
dependencies {
developmentOnly 'org.springframework.boot:spring-boot-devtools'
runtimeOnly 'mysql:mysql-connector-java'
compileOnly 'org.projectlombok:lombok'
implementation project(':support')
// https://projectreactor.io/docs/core/release/reference/#debug-activate
implementation 'org.springframework.boot:spring-boot-starter-webflux'
implementation 'org.springframework.boot:spring-boot-starter-actuator'
implementation 'org.springframework.cloud:spring-cloud-starter-config'
annotationProcessor 'org.springframework.boot:spring-boot-configuration-processor'
annotationProcessor 'org.projectlombok:lombok'
testImplementation 'org.springframework.boot:spring-boot-starter-test'
}
test {
useJUnitPlatform()
testLogging {
events "passed", "skipped", "failed"
}
}

View File

@@ -1,22 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.sender;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.context.annotation.Import;
import com.myoa.engineering.crawl.ppomppu.support.webclient.PpomppuNotifierWebClientConfiguration;
/**
* SenderApplication
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-08-20
*
*/
@Import({ PpomppuNotifierWebClientConfiguration.class })
@SpringBootApplication
public class SenderApplication {
public static void main(String[] args) {
SpringApplication.run(SenderApplication.class, args);
}
}

View File

@@ -1,62 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.sender.controller;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import com.myoa.engineering.crawl.ppomppu.sender.dto.SlackBaseMessageBlock;
import com.myoa.engineering.crawl.ppomppu.sender.dto.SlackMessageDTO;
import com.myoa.engineering.crawl.ppomppu.sender.infrastructure.client.MongeShoppingBotSlackMessageSender;
import com.myoa.engineering.crawl.ppomppu.support.dto.APIResponse;
import com.myoa.engineering.crawl.ppomppu.support.dto.BlockMessageDTO;
import com.myoa.engineering.crawl.ppomppu.support.dto.SimpleMessageDTO;
import com.myoa.engineering.crawl.ppomppu.support.util.ObjectMapperFactory;
import lombok.extern.slf4j.Slf4j;
import reactor.core.publisher.Mono;
/**
* MessageSenderAPIController
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-11-21
*
*/
@Slf4j
@RestController
@RequestMapping("/api/v1")
public class MessageSenderAPIController {
private final MongeShoppingBotSlackMessageSender sender;
public MessageSenderAPIController(MongeShoppingBotSlackMessageSender sender) {
this.sender = sender;
}
@PostMapping("/messages/sendSimpleMessage/messengers/slack")
public Mono<APIResponse<SimpleMessageDTO>> sendSimpleMessageToSlack(@RequestBody SimpleMessageDTO dto) {
return sender.sendMessage(sender.ofMessage(dto.getBody()))
.then(Mono.just(APIResponse.success(dto)));
}
@PostMapping("/messages/sendBlockMessage/messengers/slack")
public Mono<APIResponse<BlockMessageDTO>> sendBlockMessageToSlack(@RequestBody BlockMessageDTO dto) {
if (dto.getBlocks().isEmpty()) {
return Mono.just(APIResponse.fail(dto, "empty blocks"));
}
return sender.sendMessage(buildSlackMessageDTO(dto))
// .doOnNext(e -> log.info("[sendBlockMessageToSlack] slackMessageDTO: {}",
// ObjectMapperFactory.writeAsString(buildSlackMessageDTO(dto))))
.then(Mono.just(APIResponse.success(dto)));
}
private SlackMessageDTO buildSlackMessageDTO(BlockMessageDTO dto) {
SlackMessageDTO slackMessageDTO = sender.ofBlockMessageBased();
slackMessageDTO.addSectionBlock(dto.getTitle());
dto.getBlocks().forEach(slackMessageDTO::addSectionBlock);
slackMessageDTO.addBlock(SlackBaseMessageBlock.ofDivider());
return slackMessageDTO;
}
}

View File

@@ -1,33 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.sender.controller;
import com.myoa.engineering.crawl.ppomppu.sender.infrastructure.client.MongeShoppingBotSlackMessageSender;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import lombok.extern.slf4j.Slf4j;
import reactor.core.publisher.Mono;
/**
* TestAPIController
*
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-11-15
*/
@Slf4j
@RestController
@RequestMapping("/api/v1")
public class TestAPIController {
private final MongeShoppingBotSlackMessageSender sender;
public TestAPIController(MongeShoppingBotSlackMessageSender sender) {
this.sender = sender;
}
@GetMapping("/test")
public Mono<String> test() {
log.info("received!!!");
return sender.sendMessage(sender.ofMessage("testtesttest!!!"));
}
}

View File

@@ -1,47 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.sender.dto;
import com.fasterxml.jackson.annotation.JsonInclude;
import lombok.Builder;
import lombok.Getter;
import lombok.NoArgsConstructor;
/**
* MessageBlock
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-11-30
*
*/
@Getter
@NoArgsConstructor
@JsonInclude(JsonInclude.Include.NON_NULL)
public class SlackBaseMessageBlock implements SlackMessageBlock {
private static final long serialVersionUID = 1597984001727808419L;
private SlackMessageBlockType type;
private String text;
@Builder
private SlackBaseMessageBlock(SlackMessageBlockType type, String text) {
this.type = type;
this.text = text;
}
public static SlackBaseMessageBlock ofMarkDown(String message) {
return SlackBaseMessageBlock.builder()
.type(SlackMessageBlockType.MARKDOWN)
.text(message)
.build();
}
public static SlackBaseMessageBlock ofDivider() {
return SlackBaseMessageBlock.builder()
.type(SlackMessageBlockType.DIVIDER)
.build();
}
@Override
public String getType() {
return type.getType();
}
}

View File

@@ -1,49 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.sender.dto;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.Builder;
import lombok.Getter;
import lombok.NoArgsConstructor;
/**
* SlackImageMessageBlock
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-11-30
*
*/
@Getter
@NoArgsConstructor
@JsonInclude(JsonInclude.Include.NON_NULL)
public class SlackImageMessageBlock implements SlackMessageBlock {
private static final long serialVersionUID = 1597984001727808419L;
private SlackMessageBlockType type;
@JsonProperty(value = "image_url", required = true)
private String imageUrl;
@JsonProperty(value = "alt_text", required = true)
private String altText;
@Builder
private SlackImageMessageBlock(SlackMessageBlockType type, String imageUrl, String altText) {
this.type = type;
this.imageUrl = imageUrl;
this.altText = altText;
}
public static SlackImageMessageBlock of(String imageUrl, String altText) {
return SlackImageMessageBlock.builder()
.type(SlackMessageBlockType.IMAGE)
.imageUrl(imageUrl)
.altText(altText)
.build();
}
@Override
public String getType() {
return type.getType();
}
}

View File

@@ -1,15 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.sender.dto;
import java.io.Serializable;
/**
* SlackMessageBlock
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-12-01
*
*/
public interface SlackMessageBlock extends Serializable {
String getType();
}

View File

@@ -1,22 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.sender.dto;
import lombok.AllArgsConstructor;
import lombok.Getter;
/**
* BlockType
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-11-30
*
*/
@Getter
@AllArgsConstructor
public enum SlackMessageBlockType {
SECTION("section"),
MARKDOWN("mrkdwn"),
DIVIDER("divider"),
IMAGE("image"),
;
private String type;
}

View File

@@ -1,62 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.sender.dto;
import java.util.List;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.myoa.engineering.crawl.ppomppu.support.dto.BlockMessageDTO;
import com.myoa.engineering.crawl.ppomppu.support.dto.BlockMessageDTO.Block;
import lombok.Builder;
import lombok.Getter;
import lombok.NoArgsConstructor;
/**
* SlackMessageDTO
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-11-14
*
*/
@Getter
@NoArgsConstructor
@JsonInclude(JsonInclude.Include.NON_NULL)
public class SlackMessageDTO implements MessageDTO {
private final static long serialVersionUID = 4737608709660494713L;
private String text;
private String channel;
private String username;
private List<SlackMessageBlock> blocks;
@JsonProperty("icon_emoji")
private String iconEmoji;
@Builder
public SlackMessageDTO(String text, String channel, String username,
List<SlackMessageBlock> blocks, String iconEmoji) {
this.text = text;
this.channel = channel;
this.username = username;
this.blocks = blocks;
this.iconEmoji = iconEmoji;
}
public void applyText(String text) {
this.text = text;
}
public void addSectionBlock(BlockMessageDTO.Block block) {
SlackSectionMessageBlock slackSectionMessageBlock = SlackSectionMessageBlock.ofMarkDown(block.getText());
slackSectionMessageBlock.applyImageaccessory(block.getImageUrl(), block.getAltText());
addBlock(slackSectionMessageBlock);
}
public void addSectionBlock(String rawBlockMessage) {
addBlock(SlackSectionMessageBlock.ofMarkDown(rawBlockMessage));
}
public void addBlock(SlackMessageBlock block) {
blocks.add(block);
}
}

View File

@@ -1,49 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.sender.dto;
import com.fasterxml.jackson.annotation.JsonInclude;
import lombok.Builder;
import lombok.Getter;
import lombok.NoArgsConstructor;
/**
* SectionBlock
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-11-30
*
*/
@Getter
@NoArgsConstructor
@JsonInclude(JsonInclude.Include.NON_NULL)
public class SlackSectionMessageBlock implements SlackMessageBlock {
private static final long serialVersionUID = -7600944576753160168L;
private SlackMessageBlockType type;
private SlackBaseMessageBlock text;
private SlackImageMessageBlock accessory;
@Builder
private SlackSectionMessageBlock(SlackMessageBlockType type, SlackBaseMessageBlock text,
SlackImageMessageBlock accessory) {
this.type = type;
this.text = text;
this.accessory = accessory;
}
public static SlackSectionMessageBlock ofMarkDown(String message) {
return SlackSectionMessageBlock.builder()
.type(SlackMessageBlockType.SECTION)
.text(SlackBaseMessageBlock.ofMarkDown(message))
.build();
}
public SlackSectionMessageBlock applyImageaccessory(String imageUrl, String altText) {
this.accessory = SlackImageMessageBlock.of(imageUrl, altText);
return this;
}
@Override
public String getType() {
return type.getType();
}
}

View File

@@ -1,17 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.sender.infrastructure.client;
import com.myoa.engineering.crawl.ppomppu.sender.dto.MessageDTO;
import reactor.core.publisher.Mono;
/**
* MessageSender
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-11-14
*
*/
public interface MessageSender<T extends MessageDTO> {
Mono<String> sendMessage(T message);
}

View File

@@ -1,52 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.sender.infrastructure.client;
import java.util.ArrayList;
import org.springframework.stereotype.Component;
import com.myoa.engineering.crawl.ppomppu.sender.configuration.properties.SlackSecretProperties;
import com.myoa.engineering.crawl.ppomppu.sender.configuration.properties.SlackSecretProperties.SlackSecretPropertiesUnit;
import com.myoa.engineering.crawl.ppomppu.sender.dto.SlackMessageDTO;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Component
public class MongeShoppingBotSlackMessageSender extends SlackMessageSender {
private static final String SLACK_SECRET_UNIT_NAME = "monge_shopping_bot";
private final SlackSecretPropertiesUnit slackProperties;
public MongeShoppingBotSlackMessageSender(SlackSecretProperties slackSecretProperties) {
super(slackSecretProperties.find(SLACK_SECRET_UNIT_NAME).getToken());
this.slackProperties = slackSecretProperties.find(SLACK_SECRET_UNIT_NAME);
}
public SlackMessageDTO ofMessageTemplate() {
return SlackMessageDTO.builder()
.channel(slackProperties.getChannel())
.iconEmoji(slackProperties.getIconEmoji())
.username(slackProperties.getUsername())
.build();
}
public SlackMessageDTO ofMessage(String text) {
return SlackMessageDTO.builder()
.channel(slackProperties.getChannel())
.iconEmoji(slackProperties.getIconEmoji())
.username(slackProperties.getUsername())
.text(text)
.build();
}
public SlackMessageDTO ofBlockMessageBased() {
return SlackMessageDTO.builder()
.channel(slackProperties.getChannel())
.iconEmoji(slackProperties.getIconEmoji())
.username(slackProperties.getUsername())
.blocks(new ArrayList<>())
.build();
}
}

View File

@@ -1,10 +0,0 @@
spring:
config:
activate:
on-profile: development
import:
- "configserver:http://192.168.0.100:11080"
# import: optional:configserver:http://localhost:11080 # can be start up even config server was not found.
server:
port: 20082

View File

@@ -1,6 +0,0 @@
spring:
config:
activate:
on-profile: production
import:
- "configserver:http://ppn-config-server:20080"

View File

@@ -1,25 +0,0 @@
spring:
application:
name: ppn-sender
main:
allow-bean-definition-overriding: true
profiles:
active: ${SPRING_ACTIVE_PROFILE:local}
group:
local: "local,slackapi-local,webclient-local"
development: "development,slackapi-development,webclient-development"
production: "production,slackapi-production,webclient-production"
freemarker:
enabled: false
server:
port: 20080
error:
whitelabel:
enabled: false
management:
endpoints:
web:
exposure:
include: refresh,health

View File

@@ -1,15 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<configuration>
<springProfile name="local">
<include resource="logback/logback-development.xml" />
<logger name="org.apache.kafka" level="INFO" />
</springProfile>
<springProperty name="DEFAULT_LEVEL_CONFIG" source="log.defaultLevel" />
<springProfile name="development">
<include resource="logback/logback-development.xml" />
<logger name="org.apache.kafka" level="INFO" />
</springProfile>
<springProfile name="production">
<include resource="logback/logback-production.xml" />
</springProfile>
</configuration>

View File

@@ -1,23 +0,0 @@
<included>
<property name="FILE_LOG_PATTERN"
value="%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{45}:%L - %msg%n" />
<property name="LOG_FILE_BASE" value="lcp-benefit-benefit-api" />
<appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${DIRECTORY}/${LOG_FILE_BASE}_log</file>
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<fileNamePattern>${DIRECTORY}/${LOG_FILE_BASE}_log.%d{yyyyMMdd}.%i</fileNamePattern>
<timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
<maxFileSize>1000MB</maxFileSize>
</timeBasedFileNamingAndTriggeringPolicy>
<maxHistory>60</maxHistory>
</rollingPolicy>
<encoder>
<pattern>${FILE_LOG_PATTERN}</pattern>
<immediateFlush>${IMMEDIATE_FLUSH}</immediateFlush>
</encoder>
</appender>
<appender name="ASYNC_FILE" class="ch.qos.logback.classic.AsyncAppender">
<queueSize>1024</queueSize>
<appender-ref ref="FILE" />
</appender>
</included>

View File

@@ -1,19 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<included>
<!-- =========== property BETA ========= -->
<property name="DEFAULT_LEVEL" value="${DEFAULT_LEVEL_CONFIG:-INFO}"/>
<!--file-->
<property name="DIRECTORY" value="/home1/www/logs/supervisor"/>
<property name="IMMEDIATE_FLUSH" value="true"/>
<!--nelo2-->
<property name="NELO2_LEVEL" value="WARN"/>
<!-- =========== include appender =========== -->
<include resource="org/springframework/boot/logging/logback/defaults.xml"/>
<include resource="org/springframework/boot/logging/logback/console-appender.xml"/>
<include resource="logback/component/logback-nelo2.xml"/>
<include resource="logback/component/logback-datachain.xml"/>
<!-- =========== root logger ============== -->
<root level="${DEFAULT_LEVEL}">
<appender-ref ref="CONSOLE"/>
</root>
</included>

View File

@@ -1,19 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<included>
<!-- =========== property RELEASE ========= -->
<property name="DEFAULT_LEVEL" value="${DEFAULT_LEVEL_CONFIG:-INFO}"/>
<!--file-->
<property name="DIRECTORY" value="/home1/www/logs/supervisor"/>
<property name="IMMEDIATE_FLUSH" value="true"/>
<!--nelo2-->
<property name="NELO2_LEVEL" value="WARN"/>
<!-- =========== include appender =========== -->
<include resource="org/springframework/boot/logging/logback/defaults.xml"/>
<include resource="org/springframework/boot/logging/logback/console-appender.xml"/>
<include resource="logback/component/logback-nelo2.xml"/>
<include resource="logback/component/logback-datachain.xml"/>
<!-- =========== root logger ============== -->
<root level="${DEFAULT_LEVEL}">
<appender-ref ref="CONSOLE"/>
</root>
</included>

View File

@@ -1,14 +1,14 @@
rootProject.name = 'PpomppuNotifier' rootProject.name = 'PpomppuNotifier'
include 'receiver' //include 'reader'
project(':receiver').projectDir = "$rootDir/receiver" as File //project(':reader').projectDir = "$rootDir/reader" as File
//include 'processor'
include 'processor' //project(':processor').projectDir = "$rootDir/processor" as File
project(':processor').projectDir = "$rootDir/processor" as File //include 'writer'
//project(':writer').projectDir = "$rootDir/writer" as File
include 'sender'
project(':sender').projectDir = "$rootDir/sender" as File
include 'shopping-crawler'
project(':shopping-crawler').projectDir = "$rootDir/shopping-crawler" as File
include 'support' include 'support'
project(':support').projectDir = "$rootDir/support" as File project(':support').projectDir = "$rootDir/support" as File

Binary file not shown.

View File

@@ -0,0 +1,42 @@
dependencies {
developmentOnly 'org.springframework.boot:spring-boot-devtools'
runtimeOnly 'com.h2database:h2'
runtimeOnly 'com.mysql:mysql-connector-j'
compileOnly 'org.projectlombok:lombok'
implementation project(':support')
// https://projectreactor.io/docs/core/release/reference/#debug-activate
implementation("org.springframework.boot:spring-boot-starter-web") {
exclude group: "org.springframework.boot", module: "spring-boot-starter-tomcat"
}
implementation("org.springframework.boot:spring-boot-starter-undertow") {
exclude group: "io.undertow", module: "undertow-websockets-jsr"
}
implementation 'org.springframework.boot:spring-boot-starter-data-jpa'
implementation 'org.springframework.boot:spring-boot-configuration-processor'
implementation 'org.springframework.cloud:spring-cloud-starter-config'
implementation 'org.springframework.boot:spring-boot-starter-actuator'
implementation 'com.rometools:rome:2.1.0'
implementation 'org.jsoup:jsoup:1.17.2'
implementation 'com.h2database:h2:2.2.224'
implementation "org.springframework.cloud:spring-cloud-starter-openfeign"
implementation "io.github.openfeign:feign-hc5"
implementation 'org.ahocorasick:ahocorasick:0.6.3'
annotationProcessor 'org.springframework.boot:spring-boot-configuration-processor'
annotationProcessor 'org.projectlombok:lombok'
testImplementation 'org.springframework.boot:spring-boot-starter-test'
testImplementation('org.assertj:assertj-core:3.25.3')
testImplementation("org.jeasy:easy-random-core:5.0.0")
testCompileOnly 'org.projectlombok:lombok'
testAnnotationProcessor('org.projectlombok:lombok')
}
test {
useJUnitPlatform()
testLogging {
events "passed", "skipped", "failed"
}
}

View File

@@ -0,0 +1,19 @@
package com.myoa.engineering.crawl.shopping;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.cloud.openfeign.EnableFeignClients;
import org.springframework.scheduling.annotation.EnableScheduling;
@EnableScheduling
@EnableFeignClients
@EnableConfigurationProperties
@SpringBootApplication
public class ShoppingCrawlerApplication {
public static void main(String[] args) {
SpringApplication.run(ShoppingCrawlerApplication.class, args);
}
}

View File

@@ -0,0 +1,30 @@
package com.myoa.engineering.crawl.shopping.configuration;
import feign.Logger;
import feign.RequestInterceptor;
import feign.codec.ErrorDecoder;
import lombok.extern.slf4j.Slf4j;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.http.HttpHeaders;
import org.springframework.http.MediaType;
@Slf4j
@Configuration
public class FeignDefaultConfig {
@Bean
Logger.Level feignLoggerLevel() {
return Logger.Level.FULL;
}
public static final String MIME_TYPE =
MediaType.APPLICATION_JSON_VALUE + ";charset=utf-8";
@Bean
public RequestInterceptor requestInterceptor() {
return requestTemplate -> requestTemplate.header(HttpHeaders.CONTENT_TYPE, MIME_TYPE);
}
}

View File

@@ -1,6 +1,5 @@
package com.myoa.engineering.crawl.ppomppu.processor.configuration; package com.myoa.engineering.crawl.shopping.configuration.datasource;
import java.sql.SQLException;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.h2.tools.Server; import org.h2.tools.Server;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;
@@ -10,6 +9,8 @@ import org.springframework.context.event.ContextClosedEvent;
import org.springframework.context.event.ContextRefreshedEvent; import org.springframework.context.event.ContextRefreshedEvent;
import org.springframework.context.event.EventListener; import org.springframework.context.event.EventListener;
import java.sql.SQLException;
@Slf4j @Slf4j
@Profile({"datasource-local", "datasource-development"}) @Profile({"datasource-local", "datasource-development"})
@Configuration @Configuration

View File

@@ -1,54 +1,58 @@
package com.myoa.engineering.crawl.ppomppu.processor.configuration; package com.myoa.engineering.crawl.shopping.configuration.datasource;
import com.myoa.engineering.crawl.ppomppu.processor.configuration.properties.DatasourceProperties; import com.myoa.engineering.crawl.shopping.configuration.datasource.properties.DatasourceProperties;
import com.myoa.engineering.crawl.ppomppu.processor.configuration.properties.DatasourceProperties.DataSourcePropertiesUnit; import com.myoa.engineering.crawl.shopping.configuration.datasource.properties.HibernateProperties;
import com.myoa.engineering.crawl.ppomppu.processor.configuration.properties.HibernateProperties; import com.myoa.engineering.crawl.shopping.configuration.datasource.properties.HikariProperties;
import com.myoa.engineering.crawl.ppomppu.processor.configuration.properties.HikariProperties; import com.myoa.engineering.crawl.shopping.domain.entity.BaseScanDomain;
import com.myoa.engineering.crawl.ppomppu.processor.domain.BaseScanDomain; import com.myoa.engineering.crawl.shopping.infra.repository.BaseScanRepository;
import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository.BaseScanRepository;
import com.zaxxer.hikari.HikariConfig; import com.zaxxer.hikari.HikariConfig;
import com.zaxxer.hikari.HikariDataSource; import com.zaxxer.hikari.HikariDataSource;
import java.util.Enumeration; import jakarta.persistence.EntityManagerFactory;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
import javax.persistence.EntityManagerFactory;
import javax.sql.DataSource;
import lombok.NonNull; import lombok.NonNull;
import org.hibernate.boot.model.naming.CamelCaseToUnderscoresNamingStrategy;
import org.hibernate.boot.model.naming.ImplicitNamingStrategyJpaCompliantImpl;
import org.hibernate.cfg.AvailableSettings; import org.hibernate.cfg.AvailableSettings;
import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.boot.orm.jpa.EntityManagerFactoryBuilder; import org.springframework.boot.orm.jpa.EntityManagerFactoryBuilder;
import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Configuration;
import org.springframework.data.jpa.repository.config.EnableJpaAuditing;
import org.springframework.data.jpa.repository.config.EnableJpaRepositories; import org.springframework.data.jpa.repository.config.EnableJpaRepositories;
import org.springframework.orm.jpa.JpaTransactionManager; import org.springframework.orm.jpa.JpaTransactionManager;
import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean; import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean;
import org.springframework.transaction.PlatformTransactionManager; import org.springframework.transaction.PlatformTransactionManager;
@Configuration import javax.sql.DataSource;
@EnableJpaRepositories(basePackageClasses = BaseScanRepository.class, import java.util.Enumeration;
entityManagerFactoryRef = "ppomppuNotifierProcessorEntityManagerFactory", import java.util.HashMap;
transactionManagerRef = "ppomppuNotifierProcessorTransactionManager" import java.util.Map;
) import java.util.Properties;
public class PpomppuDatasourceConfiguration {
private static final String DATA_SOURCE_UNIT_NAME = "ppn_mysql"; @Configuration
@EnableJpaAuditing
@EnableJpaRepositories(basePackageClasses = BaseScanRepository.class,
entityManagerFactoryRef = "shoppingCrawlerEntityManagerFactory",
transactionManagerRef = "shoppingCrawlerTransactionManager"
)
public class ShoppingCrawlerDatasourceConfiguration {
private static final String DATA_SOURCE_UNIT_NAME = "crawler-shopping";
private final DatasourceProperties dataSourceProeprties; private final DatasourceProperties dataSourceProeprties;
private final HikariProperties hikariProperties; private final HikariProperties hikariProperties;
private final HibernateProperties hibernateProperties; private final HibernateProperties hibernateProperties;
public PpomppuDatasourceConfiguration(DatasourceProperties dataSourceProeprties, public ShoppingCrawlerDatasourceConfiguration(DatasourceProperties dataSourceProeprties,
HikariProperties hikariProperties, HikariProperties hikariProperties,
HibernateProperties hibernateProperties) { HibernateProperties hibernateProperties) {
this.dataSourceProeprties = dataSourceProeprties; this.dataSourceProeprties = dataSourceProeprties;
this.hikariProperties = hikariProperties; this.hikariProperties = hikariProperties;
this.hibernateProperties = hibernateProperties; this.hibernateProperties = hibernateProperties;
} }
@Bean(name = "ppomppuNotifierProcessorDataSource") @Bean(name = "shoppingCrawlerDataSource")
public DataSource dataSource() { public DataSource dataSource() {
DataSourcePropertiesUnit dataSourcePropertiesUnit = dataSourceProeprties.find(DATA_SOURCE_UNIT_NAME); DatasourceProperties.DataSourcePropertiesUnit dataSourcePropertiesUnit = dataSourceProeprties.find(DATA_SOURCE_UNIT_NAME);
final HikariConfig hikariConfig = new HikariConfig(); final HikariConfig hikariConfig = new HikariConfig();
hikariConfig.setJdbcUrl(dataSourcePropertiesUnit.toCompletedJdbcUrl()); hikariConfig.setJdbcUrl(dataSourcePropertiesUnit.toCompletedJdbcUrl());
@@ -69,19 +73,19 @@ public class PpomppuDatasourceConfiguration {
return dataSource; return dataSource;
} }
@Bean("ppomppuNotifierProcessorEntityManagerFactory") @Bean("shoppingCrawlerEntityManagerFactory")
public LocalContainerEntityManagerFactoryBean entityManagerFactory( public LocalContainerEntityManagerFactoryBean entityManagerFactory(
EntityManagerFactoryBuilder builder, EntityManagerFactoryBuilder builder,
@Qualifier("ppomppuNotifierProcessorDataSource") DataSource dataSource) { @Qualifier("shoppingCrawlerDataSource") DataSource dataSource) {
return builder.dataSource(dataSource) return builder.dataSource(dataSource)
.packages(BaseScanDomain.class) .packages(BaseScanDomain.class)
.properties(getPropsMap(DATA_SOURCE_UNIT_NAME)) .properties(getPropsMap(DATA_SOURCE_UNIT_NAME))
.build(); .build();
} }
@Bean("ppomppuNotifierProcessorTransactionManager") @Bean("shoppingCrawlerTransactionManager")
public PlatformTransactionManager transactionManager( public PlatformTransactionManager transactionManager(
@Qualifier("ppomppuNotifierProcessorEntityManagerFactory") EntityManagerFactory entityManagerFactory) { @Qualifier("shoppingCrawlerEntityManagerFactory") EntityManagerFactory entityManagerFactory) {
return new JpaTransactionManager(entityManagerFactory); return new JpaTransactionManager(entityManagerFactory);
} }
@@ -91,20 +95,17 @@ public class PpomppuDatasourceConfiguration {
properties.put(AvailableSettings.FORMAT_SQL, hibernateProperties.getFormatSql()); properties.put(AvailableSettings.FORMAT_SQL, hibernateProperties.getFormatSql());
properties.put(AvailableSettings.SHOW_SQL, hibernateProperties.getShowSql()); properties.put(AvailableSettings.SHOW_SQL, hibernateProperties.getShowSql());
properties.put(AvailableSettings.HBM2DDL_AUTO, hibernateProperties.getHbm2ddlAuto()); properties.put(AvailableSettings.HBM2DDL_AUTO, hibernateProperties.getHbm2ddlAuto());
properties.put(AvailableSettings.CONNECTION_PROVIDER_DISABLES_AUTOCOMMIT, properties.put(AvailableSettings.CONNECTION_PROVIDER_DISABLES_AUTOCOMMIT, hibernateProperties.getDisableAutoCommit());
hibernateProperties.getDisableAutoCommit()); properties.put(AvailableSettings.IMPLICIT_NAMING_STRATEGY, ImplicitNamingStrategyJpaCompliantImpl.class.getName());
properties.put(AvailableSettings.IMPLICIT_NAMING_STRATEGY, properties.put(AvailableSettings.PHYSICAL_NAMING_STRATEGY, CamelCaseToUnderscoresNamingStrategy.class.getName());
"org.springframework.boot.orm.jpa.hibernate.SpringImplicitNamingStrategy");
properties.put(AvailableSettings.PHYSICAL_NAMING_STRATEGY,
"org.springframework.boot.orm.jpa.hibernate.SpringPhysicalNamingStrategy");
properties.put(AvailableSettings.GENERATE_STATISTICS, "false"); properties.put(AvailableSettings.GENERATE_STATISTICS, "false");
properties.put(AvailableSettings.GLOBALLY_QUOTED_IDENTIFIERS, "true"); // properties.put(AvailableSettings.GLOBALLY_QUOTED_IDENTIFIERS, "true");
properties.put(AvailableSettings.GLOBALLY_QUOTED_IDENTIFIERS_SKIP_COLUMN_DEFINITIONS, "true"); // properties.put(AvailableSettings.GLOBALLY_QUOTED_IDENTIFIERS_SKIP_COLUMN_DEFINITIONS, "true");
properties.put(AvailableSettings.STATEMENT_BATCH_SIZE, "20"); properties.put(AvailableSettings.STATEMENT_BATCH_SIZE, "20");
properties.put(AvailableSettings.ORDER_INSERTS, "true"); properties.put(AvailableSettings.ORDER_INSERTS, "true");
properties.put(AvailableSettings.ORDER_UPDATES, "true"); properties.put(AvailableSettings.ORDER_UPDATES, "true");
properties.put(AvailableSettings.BATCH_VERSIONED_DATA, "true"); properties.put(AvailableSettings.BATCH_VERSIONED_DATA, "true");
properties.put(AvailableSettings.USE_NEW_ID_GENERATOR_MAPPINGS, "false"); // properties.put(AvailableSettings.JPA_ID_GENERATOR_GLOBAL_SCOPE_COMPLIANCE, "false");
return properties; return properties;
} }

View File

@@ -1,29 +1,26 @@
package com.myoa.engineering.crawl.ppomppu.processor.configuration.properties; package com.myoa.engineering.crawl.shopping.configuration.datasource.properties;
import com.myoa.engineering.crawl.ppomppu.support.util.ObjectUtil; import com.myoa.engineering.crawl.shopping.support.util.ObjectUtil;
import java.util.List; import lombok.Data;
import lombok.Getter;
import lombok.Setter;
import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import java.util.List;
@Component @Component
@Setter @Data
@Getter @ConfigurationProperties(prefix = "datasource")
@ConfigurationProperties(prefix = "infra.database")
public class DatasourceProperties { public class DatasourceProperties {
private List<DataSourcePropertiesUnit> units; private List<DataSourcePropertiesUnit> units;
@Getter @Data
@Setter
public static class DataSourcePropertiesUnit { public static class DataSourcePropertiesUnit {
private String unitName; private String unitName;
private String schemaName; private String schemaName;
private String connectionParameters; private String connectionParameters;
private String datasourceUrl; private String dbConnectionUrl;
private Boolean isSimpleConnectionUrl; private Boolean isSimpleConnectionUrl;
private String username; private String username;
private String password; private String password;
@@ -31,9 +28,9 @@ public class DatasourceProperties {
public String toCompletedJdbcUrl() { public String toCompletedJdbcUrl() {
if (ObjectUtil.isEmpty(isSimpleConnectionUrl) || isSimpleConnectionUrl == false) { if (ObjectUtil.isEmpty(isSimpleConnectionUrl) || isSimpleConnectionUrl == false) {
return String.format("%s/%s?%s", datasourceUrl, schemaName, connectionParameters); return String.format("%s/%s?%s", dbConnectionUrl, schemaName, connectionParameters);
} }
return datasourceUrl; return dbConnectionUrl;
} }
} }
@@ -42,7 +39,7 @@ public class DatasourceProperties {
.filter(e -> e.getUnitName().equals(unitName)) .filter(e -> e.getUnitName().equals(unitName))
.findFirst() .findFirst()
.orElseThrow( .orElseThrow(
() -> new IllegalArgumentException(this.getClass().getName() + ": unitName Not found. " + unitName)); () -> new IllegalArgumentException(this.getClass().getName() + ": unitName Not found. " + unitName));
} }
} }

View File

@@ -1,14 +1,12 @@
package com.myoa.engineering.crawl.ppomppu.processor.configuration.properties; package com.myoa.engineering.crawl.shopping.configuration.datasource.properties;
import java.util.List;
import lombok.Getter; import lombok.Getter;
import lombok.Setter; import lombok.Setter;
import org.hibernate.dialect.MySQL8Dialect;
import org.hibernate.dialect.MySQLDialect;
import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import java.util.List;
@Component @Component
@Setter @Setter
@Getter @Getter

View File

@@ -1,4 +1,4 @@
package com.myoa.engineering.crawl.ppomppu.processor.configuration.properties; package com.myoa.engineering.crawl.shopping.configuration.datasource.properties;
import lombok.Getter; import lombok.Getter;
import lombok.Setter; import lombok.Setter;

View File

@@ -1,4 +1,4 @@
package com.myoa.engineering.crawl.ppomppu.sender.configuration.properties; package com.myoa.engineering.crawl.shopping.configuration.slack.properties;
import java.util.List; import java.util.List;
import lombok.Data; import lombok.Data;
@@ -7,18 +7,16 @@ import lombok.Setter;
import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
@Getter @Data
@Setter
@Component @Component
@ConfigurationProperties("infra.slack.bot") @ConfigurationProperties(prefix = "slack.bot")
public class SlackSecretProperties { public class SlackSecretProperties {
private List<SlackSecretPropertiesUnit> units; private List<SlackSecretPropertiesUnit> units;
@Data @Data
public static class SlackSecretPropertiesUnit { public static class SlackSecretPropertiesUnit {
private String botUnitName;
private String botName;
private String username; private String username;
private String iconEmoji; private String iconEmoji;
private String channel; private String channel;
@@ -27,7 +25,7 @@ public class SlackSecretProperties {
public SlackSecretPropertiesUnit find(String botUnitName) { public SlackSecretPropertiesUnit find(String botUnitName) {
return units.stream() return units.stream()
.filter(e -> e.getBotName().equals(botUnitName)) .filter(e -> e.getBotUnitName().equals(botUnitName))
.findFirst() .findFirst()
.orElseThrow(() -> new IllegalArgumentException("not found bot unit name : " + botUnitName)); .orElseThrow(() -> new IllegalArgumentException("not found bot unit name : " + botUnitName));
} }

View File

@@ -0,0 +1,22 @@
package com.myoa.engineering.crawl.shopping.controller;
import com.myoa.engineering.crawl.shopping.crawlhandler.PpomppuCrawlDomesticHandler;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
@RestController
@RequestMapping("/api/v1/exploit")
public class TestAPIController {
private final PpomppuCrawlDomesticHandler ppomppuCrawlDomesticHandler;
public TestAPIController(PpomppuCrawlDomesticHandler ppomppuCrawlDomesticHandler) {
this.ppomppuCrawlDomesticHandler = ppomppuCrawlDomesticHandler;
}
@GetMapping("/triggers")
public void triggerExploit() {
ppomppuCrawlDomesticHandler.handle();
}
}

View File

@@ -0,0 +1,10 @@
package com.myoa.engineering.crawl.shopping.crawlhandler;
import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget;
public interface CrawlHandler {
CrawlTarget getCrawlTarget();
void handle();
}

View File

@@ -0,0 +1,18 @@
package com.myoa.engineering.crawl.shopping.crawlhandler;
import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
@Slf4j
@Component
public class FmkoreaCrawlHandler implements CrawlHandler {
@Override
public CrawlTarget getCrawlTarget() {
return CrawlTarget.FMKOREA;
}
@Override
public void handle() {
}
}

View File

@@ -0,0 +1,58 @@
package com.myoa.engineering.crawl.shopping.crawlhandler;
import com.myoa.engineering.crawl.shopping.crawlhandler.parser.PpomppuArticleParserV2;
import com.myoa.engineering.crawl.shopping.domain.entity.v2.Article;
import com.myoa.engineering.crawl.shopping.infra.client.ppomppu.PpomppuBoardClientV2;
import com.myoa.engineering.crawl.shopping.service.ArticleCommandService;
import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget;
import org.springframework.stereotype.Component;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Stream;
@Component
public class PpomppuCrawlDomesticHandler implements CrawlHandler {
private final PpomppuBoardClientV2 ppomppuBoardClient;
private final PpomppuArticleParserV2 ppomppuArticleParserV2;
private final ArticleCommandService articleCommandService;
public PpomppuCrawlDomesticHandler(PpomppuBoardClientV2 ppomppuBoardClient,
PpomppuArticleParserV2 ppomppuArticleParserV2,
ArticleCommandService articleCommandService) {
this.ppomppuBoardClient = ppomppuBoardClient;
this.ppomppuArticleParserV2 = ppomppuArticleParserV2;
this.articleCommandService = articleCommandService;
}
@Override
public CrawlTarget getCrawlTarget() {
return CrawlTarget.PPOMPPU_DOMESTIC;
}
@Override
public void handle() {
String boardHtmlPage1 = ppomppuBoardClient.getBoardHtml("/zboard/zboard.php", generateRequestParams(1));
List<Article> parsedPage1 = ppomppuArticleParserV2.parse(boardHtmlPage1);
String boardHtmlPage2 = ppomppuBoardClient.getBoardHtml("/zboard/zboard.php", generateRequestParams(2));
List<Article> parsedPage2 = ppomppuArticleParserV2.parse(boardHtmlPage2);
List<Article> merged = Stream.of(parsedPage1, parsedPage2)
.flatMap(List::stream)
.map(e -> e.updateCrawlTarget(getCrawlTarget()))
.toList();
articleCommandService.upsert(merged);
}
private Map<String, String> generateRequestParams(int pageId) {
Map<String, String> params = new HashMap<>();
params.put("id", "ppomppu");
params.put("page", String.valueOf(pageId));
return params;
}
}

View File

@@ -0,0 +1,115 @@
package com.myoa.engineering.crawl.shopping.crawlhandler.parser;
import com.myoa.engineering.crawl.shopping.domain.entity.v2.Article;
import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName;
import com.myoa.engineering.crawl.shopping.util.DateTimeUtils;
import com.myoa.engineering.crawl.shopping.util.NumberUtils;
import io.micrometer.core.instrument.util.StringUtils;
import lombok.extern.slf4j.Slf4j;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.stereotype.Component;
import java.time.ZoneId;
import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@Slf4j
@Component
public final class PpomppuArticleParserV2 {
private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yy.MM.dd HH:mm:ss")
.withZone(ZoneId.of("Asia/Seoul"));
private PpomppuArticleParserV2() {
}
public List<Article> parse(String html) {
Elements trElements = converHtmlToTrElements(html);
return trElements.stream()
.filter(this::isRealArticle)
.map(this::parse)
.toList();
}
private Elements converHtmlToTrElements(String data) {
Document document = Jsoup.parse(data);
Elements trList = document.getElementById("revolution_main_table").getElementsByTag("tr");
return trList;
}
private boolean isRealArticle(Element tr) {
Elements tdList = tr.getElementsByTag("td");
if (tdList.size() != 6) {
return false;
}
if (!hasOnlyNumeric(tdList.get(0))) {
return false;
}
return true;
}
private Pattern pattern_numeric = Pattern.compile("\\d+");
private boolean hasOnlyNumeric(Element td) {
return pattern_numeric.matcher(td.text()).matches();
}
private Article parse(Element tr) {
Elements tdList = tr.getElementsByTag("td");
Long articleId = Long.parseLong(tdList.get(0).text());
String title = tdList.get(2).text();
String articleUrl = parseArticleUrl(tdList.get(2).getElementsByTag("a").attr("href"));
String boardName = parseBoardName(title);
Integer recommended = parseRecommended(tdList.get(4));
Integer hit = NumberUtils.parseInt(tdList.get(5).text(), 0);
ZonedDateTime registeredAt = DateTimeUtils.parse(tdList.get(3).text());
return Article.builder()
.articleId(articleId)
.title(title)
.boardName(boardName)
.articleUrl(articleUrl)
.recommended(recommended)
.hit(hit)
.registeredAt(registeredAt)
.build();
}
public Integer parseRecommended(Element td) {
final String voteString = td.text();
if (StringUtils.isEmpty(voteString)) {
return null;
}
final int voteUp = Integer.parseInt(td.text().split(" - ")[0]);
final int voteDown = Integer.parseInt(td.text().split(" - ")[1]);
int recommended = voteUp - voteDown;
return recommended;
}
public static String parseArticleUrl(String data) {
return PpomppuBoardName.ofViewPageUrl(data);
}
Pattern patternBoardName = Pattern.compile("\\[(.+?)\\]");
public String parseBoardName(String fullTitle) {
Matcher matcher = patternBoardName.matcher(fullTitle);
String lastMatched = null;
while (matcher.find()) {
lastMatched = matcher.group(1);
}
return lastMatched;
}
}

View File

@@ -1,17 +1,18 @@
package com.myoa.engineering.crawl.ppomppu.processor.domain; package com.myoa.engineering.crawl.shopping.domain.entity;
import java.io.Serializable; import lombok.Getter;
import java.time.Instant;
import javax.persistence.Column;
import org.springframework.data.annotation.CreatedDate; import org.springframework.data.annotation.CreatedDate;
import org.springframework.data.annotation.LastModifiedDate; import org.springframework.data.annotation.LastModifiedDate;
/** import jakarta.persistence.*;
* Auditable import org.springframework.data.jpa.domain.support.AuditingEntityListener;
*
* @author Shin Woo-jin (woozu.shin@kakaoent.com) import java.io.Serializable;
* @since 2021-09-08 import java.time.Instant;
*/
@Getter
@MappedSuperclass
@EntityListeners(AuditingEntityListener.class)
public abstract class Auditable implements Serializable { public abstract class Auditable implements Serializable {
private static final long serialVersionUID = -7105030870015828551L; private static final long serialVersionUID = -7105030870015828551L;

View File

@@ -1,4 +1,4 @@
package com.myoa.engineering.crawl.ppomppu.processor.domain; package com.myoa.engineering.crawl.shopping.domain.entity;
/** /**
* BaseScanDomain * BaseScanDomain

View File

@@ -1,19 +1,16 @@
package com.myoa.engineering.crawl.ppomppu.processor.domain; package com.myoa.engineering.crawl.shopping.domain.entity.v1;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName; import com.myoa.engineering.crawl.shopping.domain.entity.Auditable;
import java.time.Instant; import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName;
import javax.persistence.Column; import jakarta.persistence.*;
import javax.persistence.Entity;
import javax.persistence.EnumType;
import javax.persistence.Enumerated;
import javax.persistence.GeneratedValue;
import javax.persistence.GenerationType;
import javax.persistence.Id;
import javax.persistence.Table;
import lombok.Builder; import lombok.Builder;
import lombok.Getter; import lombok.Getter;
import lombok.NoArgsConstructor; import lombok.NoArgsConstructor;
import lombok.ToString;
import java.time.ZonedDateTime;
@ToString
@Getter @Getter
@NoArgsConstructor @NoArgsConstructor
@Entity @Entity
@@ -34,9 +31,6 @@ public class PpomppuArticle extends Auditable {
@Column @Column
private String articleUrl; private String articleUrl;
@Column
private String thumbnailUrl;
@Column @Column
private String title; private String title;
@@ -47,25 +41,19 @@ public class PpomppuArticle extends Auditable {
private Integer recommended; private Integer recommended;
@Column @Column
private Instant registeredAt; private ZonedDateTime registeredAt;
@Builder @Builder
public PpomppuArticle(Long id, Long articleId, PpomppuBoardName boardName, String articleUrl, public PpomppuArticle(Long id, Long articleId, PpomppuBoardName boardName, String articleUrl,
String thumbnailUrl, String title, Integer recommended, Integer hit, String title, Integer recommended, Integer hit, ZonedDateTime registeredAt) {
Instant registeredAt) {
this.id = id; this.id = id;
this.articleId = articleId; this.articleId = articleId;
this.boardName = boardName; this.boardName = boardName;
this.articleUrl = articleUrl; this.articleUrl = articleUrl;
this.thumbnailUrl = thumbnailUrl;
this.title = title; this.title = title;
this.recommended = recommended; this.recommended = recommended;
this.hit = hit; this.hit = hit;
this.registeredAt = registeredAt; this.registeredAt = registeredAt;
} }
public PpomppuArticle updateBoardName(PpomppuBoardName boardName) {
this.boardName = boardName;
return this;
}
} }

View File

@@ -1,19 +1,14 @@
package com.myoa.engineering.crawl.ppomppu.processor.domain; package com.myoa.engineering.crawl.shopping.domain.entity.v1;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName; import com.myoa.engineering.crawl.shopping.domain.entity.Auditable;
import java.time.Instant; import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName;
import javax.persistence.Column; import jakarta.persistence.*;
import javax.persistence.Entity;
import javax.persistence.EnumType;
import javax.persistence.Enumerated;
import javax.persistence.GeneratedValue;
import javax.persistence.GenerationType;
import javax.persistence.Id;
import javax.persistence.Table;
import lombok.Builder; import lombok.Builder;
import lombok.Getter; import lombok.Getter;
import lombok.NoArgsConstructor; import lombok.NoArgsConstructor;
import java.time.Instant;
@Getter @Getter
@NoArgsConstructor @NoArgsConstructor
@Entity @Entity

View File

@@ -1,15 +1,12 @@
package com.myoa.engineering.crawl.ppomppu.processor.domain; package com.myoa.engineering.crawl.shopping.domain.entity.v1;
import java.time.Instant; import com.myoa.engineering.crawl.shopping.domain.entity.Auditable;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.GeneratedValue;
import javax.persistence.GenerationType;
import javax.persistence.Id;
import javax.persistence.Table;
import lombok.Getter; import lombok.Getter;
import lombok.NoArgsConstructor; import lombok.NoArgsConstructor;
import jakarta.persistence.*;
import java.time.Instant;
@Getter @Getter
@NoArgsConstructor @NoArgsConstructor
@Entity @Entity

View File

@@ -0,0 +1,27 @@
package com.myoa.engineering.crawl.shopping.domain.entity.v1;
import com.myoa.engineering.crawl.shopping.domain.entity.Auditable;
import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName;
import lombok.Getter;
import lombok.NoArgsConstructor;
import jakarta.persistence.*;
@Getter
@NoArgsConstructor
@Entity
@Table(name = "subscribed_board")
public class SubscribedBoard extends Auditable {
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
private Long id;
@Column
private Long userId;
@Column
@Enumerated(EnumType.STRING)
private PpomppuBoardName boardName;
}

View File

@@ -1,20 +1,17 @@
package com.myoa.engineering.crawl.ppomppu.processor.domain; package com.myoa.engineering.crawl.shopping.domain.entity.v1;
import java.time.Instant; import com.myoa.engineering.crawl.shopping.domain.entity.Auditable;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.GeneratedValue;
import javax.persistence.GenerationType;
import javax.persistence.Id;
import javax.persistence.Table;
import lombok.Getter; import lombok.Getter;
import lombok.NoArgsConstructor; import lombok.NoArgsConstructor;
import jakarta.persistence.*;
import java.time.Instant;
@Getter @Getter
@NoArgsConstructor @NoArgsConstructor
@Entity @Entity
@Table(name = "subscribed_user") @Table(name = "subscribed_user")
public class SubscribedUser extends Auditable{ public class SubscribedUser extends Auditable {
@Id @Id
@GeneratedValue(strategy = GenerationType.IDENTITY) @GeneratedValue(strategy = GenerationType.IDENTITY)

View File

@@ -0,0 +1,29 @@
package com.myoa.engineering.crawl.shopping.domain.entity.v2;
import com.myoa.engineering.crawl.shopping.domain.entity.Auditable;
import jakarta.persistence.*;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Getter;
import lombok.NoArgsConstructor;
@Getter
@Builder
@NoArgsConstructor
@AllArgsConstructor
@Entity
@Table
public class AppUser extends Auditable {
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
private Long id;
@Column
private String name;
@Column
private String slackId;
@Column
private Boolean enabled;
}

View File

@@ -0,0 +1,63 @@
package com.myoa.engineering.crawl.shopping.domain.entity.v2;
import com.myoa.engineering.crawl.shopping.domain.entity.Auditable;
import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget;
import jakarta.persistence.*;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Getter;
import lombok.NoArgsConstructor;
import java.time.ZonedDateTime;
@Getter
@Builder
@NoArgsConstructor
@AllArgsConstructor
@Entity
@Table
public class Article extends Auditable {
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
private Long id;
@Column
private Long articleId;
@Column
@Enumerated(EnumType.STRING)
private CrawlTarget crawlTarget;
@Column
private String boardName;
@Column
private String articleUrl;
@Column
private String title;
@Column
private Integer hit;
@Column
private Integer recommended;
@Column
private ZonedDateTime registeredAt;
public Article update(Article article) {
this.boardName = article.getBoardName();
this.articleUrl = article.getArticleUrl();
this.title = article.getTitle();
this.hit = article.getHit();
this.recommended = article.getRecommended();
return this;
}
public Article updateCrawlTarget(CrawlTarget crawlTarget) {
this.crawlTarget = crawlTarget;
return this;
}
}

View File

@@ -0,0 +1,33 @@
package com.myoa.engineering.crawl.shopping.domain.entity.v2;
import com.myoa.engineering.crawl.shopping.domain.entity.Auditable;
import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget;
import jakarta.persistence.*;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Getter;
import lombok.NoArgsConstructor;
@Getter
@Builder
@NoArgsConstructor
@AllArgsConstructor
@Entity
@Table
public class SubscribedKeyword extends Auditable {
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
private Long id;
@Column
private String keyword;
@Column
@Enumerated(EnumType.STRING)
private CrawlTarget crawlTarget;
@Column
private String userId;
}

View File

@@ -0,0 +1,24 @@
package com.myoa.engineering.crawl.shopping.domain.model;
import com.myoa.engineering.crawl.shopping.domain.model.v2.ArticleModel;
import lombok.*;
import java.util.List;
@ToString
@Getter
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class UserNotifyModel {
private String userId;
private List<ArticleModel> articles;
public static UserNotifyModel of(String userId, List<ArticleModel> articles) {
return UserNotifyModel.builder()
.userId(userId)
.articles(articles)
.build();
}
}

View File

@@ -0,0 +1,26 @@
package com.myoa.engineering.crawl.shopping.domain.model.v2;
import com.myoa.engineering.crawl.shopping.domain.entity.v2.AppUser;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Getter;
@Getter
@Builder
@AllArgsConstructor
public class AppUserModel {
private Long id;
private String name;
private String slackId;
private Boolean enabled;
public static AppUserModel from(AppUser entity) {
return AppUserModel.builder()
.id(entity.getId())
.name(entity.getName())
.slackId(entity.getSlackId())
.enabled(entity.getEnabled())
.build();
}
}

View File

@@ -0,0 +1,25 @@
package com.myoa.engineering.crawl.shopping.domain.model.v2;
import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget;
import lombok.*;
import java.time.ZonedDateTime;
@ToString
@Getter
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class ArticleModel {
private Long id;
private Long articleId;
private CrawlTarget crawlTarget;
private String boardName;
private String articleUrl;
private String title;
private Integer hit;
private Integer recommended;
private ZonedDateTime registeredAt;
}

View File

@@ -0,0 +1,27 @@
package com.myoa.engineering.crawl.shopping.domain.model.v2;
import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget;
import com.myoa.engineering.crawl.shopping.util.AhoCorasickUtils;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Getter;
import org.ahocorasick.trie.Trie;
import java.util.List;
@Getter
@Builder
@AllArgsConstructor
public class SubscribedKeywordAggregatedModel {
private final Trie ahoCorasickTrie;
private final String userId;
private final CrawlTarget crawlTarget;
public static SubscribedKeywordAggregatedModel of(String userId, CrawlTarget crawlTarget, List<String> keywords) {
return SubscribedKeywordAggregatedModel.builder()
.userId(userId)
.crawlTarget(crawlTarget)
.ahoCorasickTrie(AhoCorasickUtils.generateTrie(keywords))
.build();
}
}

View File

@@ -1,12 +1,13 @@
package com.myoa.engineering.crawl.ppomppu.processor.dto; package com.myoa.engineering.crawl.shopping.dto;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName; import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName;
import java.io.Serializable;
import java.time.Instant;
import lombok.Builder; import lombok.Builder;
import lombok.Getter; import lombok.Getter;
import lombok.NoArgsConstructor; import lombok.NoArgsConstructor;
import java.io.Serializable;
import java.time.Instant;
/** /**
* FeedParsedResult * FeedParsedResult
* *

View File

@@ -0,0 +1,10 @@
package com.myoa.engineering.crawl.shopping.dto;
/**
* PpomppuArticle
*
* @author Shin Woo-jin (woozu.shin@kakaoent.com)
* @since 2021-09-08
*/
public class PpomppuArticleDTO {
}

View File

@@ -0,0 +1,53 @@
package com.myoa.engineering.crawl.shopping.dto;
import com.myoa.engineering.crawl.shopping.domain.entity.v1.PpomppuArticle;
import com.myoa.engineering.crawl.shopping.support.dto.SimpleMessageDTO;
import java.time.Instant;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.util.List;
import java.util.stream.Collectors;
/**
* PpomppuArticleTransformer
*
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-11-21
*/
public final class PpomppuArticleTransformer {
private PpomppuArticleTransformer() {
}
private static final String MESSAGE_FORMAT_V1 = "%s)) `%s` <%s:LINK>";
private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")
.withZone(ZoneId.of("Asia/Seoul"));
/*
public static final Function<PpomppuArticle, SimpleMessageDTO> TRANSFORM_TO_MESSAGE_DTO = entity ->
SimpleMessageDTO.builder()
.requestedAt(Instant.now())
.publishedAt(entity.getRegisteredAt())
.title(String.format(MESSAGE_FORMAT_V1, entity.getBoardName().getMenuName(), entity.getTitle()))
.body(entity.getArticleUrl())
.build();
*/
// https://stackoverflow.com/questions/24882927/using-streams-to-convert-a-list-of-objects-into-a-string-obtained-from-the-tostr
public static SimpleMessageDTO transform(List<PpomppuArticle> articles) {
Instant requestedAt = Instant.now();
String body = articles.stream()
.map(PpomppuArticleTransformer::convertToInlineMessage)
.collect(Collectors.joining("\n\n"));
return SimpleMessageDTO.builder()
.requestedAt(requestedAt)
.title(DATE_TIME_FORMATTER.format(requestedAt))
.body(body)
.build();
}
public static String convertToInlineMessage(PpomppuArticle article) {
return String.format(MESSAGE_FORMAT_V1,
article.getBoardName().getMenuName(), article.getTitle(), article.getArticleUrl());
}
}

View File

@@ -1,4 +1,4 @@
package com.myoa.engineering.crawl.ppomppu.processor.dto.constant; package com.myoa.engineering.crawl.shopping.dto.constant;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;
import lombok.Getter; import lombok.Getter;

View File

@@ -1,4 +1,4 @@
package com.myoa.engineering.crawl.ppomppu.sender.dto; package com.myoa.engineering.crawl.shopping.dto.slack;
import java.io.Serializable; import java.io.Serializable;

View File

@@ -0,0 +1,39 @@
package com.myoa.engineering.crawl.shopping.dto.slack;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.Builder;
import lombok.Getter;
import lombok.NoArgsConstructor;
/**
* SlackMessageDTO
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-11-14
*
*/
@Getter
@NoArgsConstructor
public class SlackMessageDTO implements MessageDTO {
private final static long serialVersionUID = 4737608709660494713L;
private String text;
private String channel;
private String username;
@JsonProperty("icon_emoji")
private String iconEmoji;
@Builder
public SlackMessageDTO(String text, String channel, String username, String iconEmoji) {
this.text = text;
this.channel = channel;
this.username = username;
this.iconEmoji = iconEmoji;
}
public void applyText(String text) {
this.text = text;
}
}

Some files were not shown because too many files have changed in this diff Show More