MyoaEngineering
/
ShoppingCrawler
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Compare commits

base: MyoaEngineering:4738b930dfce4db904ae09c2a13bfb067443a480
Branches Tags
MyoaEngineering:develop
MyoaEngineering:master
MyoaEngineering:feature/NO-ISSUE-v2
MyoaEngineering:feature/PPN-13
...
compare: MyoaEngineering:08737664f462dd8f51390659bf4237b92157ebf7
Branches Tags
MyoaEngineering:master
MyoaEngineering:develop
MyoaEngineering:feature/NO-ISSUE-v2
MyoaEngineering:feature/PPN-13

4 Commits
4738b930df ... 08737664f4

Author SHA1 Message Date
woozu-shin 08737664f4 Merge branch 'feature/NO-ISSUE-v2' into develop 2024-05-09 08:50:05 +09:00
woozu-shin 7b015e8093 [NO-ISSUE] Remove reader, processor, writer 2024-05-09 08:49:11 +09:00
woozu-shin b4accbc2c0 [NO-ISSUE] Initialize v2 2024-05-09 08:48:36 +09:00
woozu-shin 0c4be3cc05 [NO-ISSUE] Implement v2 2024-04-30 22:34:49 +09:00
150 changed files with 2775 additions and 1230 deletions
Show Stats Expand all files Collapse all files

3
.gitignore vendored
View File

@ -37,3 +37,6 @@ out/
.vscode/
temppassword.yml
data.sql
**/src/main/resources/slack
**/src/main/resources/datasource

3
gradle/wrapper/gradle-wrapper.properties vendored
View File

@ -1,5 +1,6 @@
#Sun Apr 28 23:47:38 KST 2024
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-7.1.1-bin.zip
distributionUrl=https\://services.gradle.org/distributions/gradle-8.5-bin.zip
zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists

0
gradlew vendored Normal file → Executable file
View File

30
processor/build.gradle
View File

@ -1,30 +0,0 @@
dependencies {
developmentOnly 'org.springframework.boot:spring-boot-devtools'
runtimeOnly 'com.h2database:h2'
runtimeOnly 'mysql:mysql-connector-java'
compileOnly 'org.projectlombok:lombok'
implementation project(':support')
// https://projectreactor.io/docs/core/release/reference/#debug-activate
implementation 'org.springframework.boot:spring-boot-starter-webflux'
implementation 'org.springframework.boot:spring-boot-starter-data-jpa'
implementation 'org.springframework.boot:spring-boot-starter-jdbc'
implementation 'org.springframework.cloud:spring-cloud-starter-config'
implementation 'org.springframework.boot:spring-boot-starter-actuator'
implementation 'com.rometools:rome:1.16.0'
implementation 'org.jsoup:jsoup:1.14.2'
implementation 'com.h2database:h2:1.4.200'
annotationProcessor 'org.springframework.boot:spring-boot-configuration-processor'
annotationProcessor 'org.projectlombok:lombok'
testImplementation 'org.springframework.boot:spring-boot-starter-test'
}
test {
useJUnitPlatform()
testLogging {
events "passed", "skipped", "failed"
}
}

23
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/ProcessorApplication.java
View File

@ -1,23 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.processor;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.context.annotation.Import;
import com.myoa.engineering.crawl.ppomppu.support.webclient.PpomppuNotifierWebClientConfiguration;
/**
* ProcessorApplication
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-08-20
*
*/
@Import({ PpomppuNotifierWebClientConfiguration.class })
@SpringBootApplication
public class ProcessorApplication {
public static void main(String[] args) {
SpringApplication.run(ProcessorApplication.class, args);
}
}

32
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/SubscribedBoard.java
View File

@ -1,32 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.processor.domain;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.EnumType;
import javax.persistence.Enumerated;
import javax.persistence.GeneratedValue;
import javax.persistence.GenerationType;
import javax.persistence.Id;
import javax.persistence.Table;
import lombok.Getter;
import lombok.NoArgsConstructor;
@Getter
@NoArgsConstructor
@Entity
@Table(name = "subscribed_board")
public class SubscribedBoard extends Auditable {
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
private Long id;
@Column
private Long userId;
@Column
@Enumerated(EnumType.STRING)
private PpomppuBoardName boardName;
}

4
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/repository/BaseScanRepository.java
View File

@ -1,4 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository;
public interface BaseScanRepository {
}

11
processor/src/main/resources/application-local.yml
View File

@ -1,11 +0,0 @@
spring:
config:
activate:
on-profile: local
import:
- "configserver:http://localhost:20085"
server:
port: 20081
# import: optional:configserver:http://localhost:11080 # can be start up even config server was not found.

26
receiver/build.gradle
View File

@ -1,26 +0,0 @@
dependencies {
developmentOnly 'org.springframework.boot:spring-boot-devtools'
runtimeOnly 'com.h2database:h2'
runtimeOnly 'mysql:mysql-connector-java'
compileOnly 'org.projectlombok:lombok'
implementation project(':support')
// https://projectreactor.io/docs/core/release/reference/#debug-activate
implementation 'org.springframework.boot:spring-boot-starter-webflux'
implementation 'org.springframework.boot:spring-boot-configuration-processor'
implementation 'org.springframework.cloud:spring-cloud-starter-config'
implementation 'org.springframework.boot:spring-boot-starter-actuator'
implementation 'org.telegram:telegrambots:5.3.0'
annotationProcessor 'org.springframework.boot:spring-boot-configuration-processor'
annotationProcessor 'org.projectlombok:lombok'
testImplementation 'org.springframework.boot:spring-boot-starter-test'
}
test {
useJUnitPlatform()
testLogging {
events "passed", "skipped", "failed"
}
}

25
receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/ReceiverApplication.java
View File

@ -1,25 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.receiver;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.annotation.Import;
import com.myoa.engineering.crawl.ppomppu.receiver.configuration.properties.TelegramBotProperties;
import com.myoa.engineering.crawl.ppomppu.support.webclient.PpomppuNotifierWebClientConfiguration;
/**
* ReceiverApplication
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-08-20
*
*/
@Import({ PpomppuNotifierWebClientConfiguration.class})
@SpringBootApplication
// @EnableConfigurationProperties({ TelegramBotProperties.class })
public class ReceiverApplication {
public static void main(String[] args) {
SpringApplication.run(ReceiverApplication.class, args);
}
}

39
receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/configuration/TelegramBotConfiguration.java
View File

@ -1,39 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.receiver.configuration;
import java.util.List;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.telegram.telegrambots.meta.TelegramBotsApi;
import org.telegram.telegrambots.meta.exceptions.TelegramApiException;
import org.telegram.telegrambots.updatesreceivers.DefaultBotSession;
import com.myoa.engineering.crawl.ppomppu.receiver.configuration.properties.TelegramBotProperties;
import com.myoa.engineering.crawl.ppomppu.receiver.configuration.properties.TelegramBotProperties.TelegramBotPropertiesUnit;
import com.myoa.engineering.crawl.ppomppu.receiver.dispatch.MessageDispatcher;
import com.myoa.engineering.crawl.ppomppu.receiver.handler.message.MessageHandler;
/**
* TelegramBotConfiguration
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-08-21
*
*/
@Configuration
public class TelegramBotConfiguration {
private static final String BOT_PROPERTIES_UNIT_NAME = "ppomppu_notify_bot";
@Bean
public TelegramBotsApi telegramBotsApi(MessageDispatcher messageDispatcher) throws TelegramApiException {
TelegramBotsApi api = new TelegramBotsApi(DefaultBotSession.class);
api.registerBot(messageDispatcher);
return api;
}
@Bean
public MessageDispatcher messageDispatcher(List<MessageHandler> messageHandlers,
TelegramBotProperties telegramBotProperties) {
TelegramBotPropertiesUnit propertiesUnit = telegramBotProperties.find(BOT_PROPERTIES_UNIT_NAME);
return new MessageDispatcher(messageHandlers, propertiesUnit.getName(), propertiesUnit.getToken());
}
}

44
receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/configuration/properties/TelegramBotProperties.java
View File

@ -1,44 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.receiver.configuration.properties;
import java.util.ArrayList;
import java.util.List;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.boot.context.properties.ConstructorBinding;
import org.springframework.stereotype.Component;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
/**
* TelegramBotProperties
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-09-05
*
*/
@Component
@Setter
@Getter
@ConfigurationProperties(prefix = "infra.telegram.bot")
public class TelegramBotProperties {
private List<TelegramBotPropertiesUnit> units = new ArrayList<>();
@Data
public static class TelegramBotPropertiesUnit {
private String unitName;
private String name;
private String token;
}
public TelegramBotPropertiesUnit find(String unitName) {
return units.stream()
.filter(e -> e.getUnitName().equals(unitName))
.findFirst()
.orElseThrow(
() -> new IllegalArgumentException(this.getClass().getName() + ": unitName Not found. " + unitName));
}
}

33
receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/controller/v1/EventAPIController.java
View File

@ -1,33 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.receiver.controller.v1;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import com.myoa.engineering.crawl.ppomppu.receiver.service.ProcessorAPIService;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import reactor.core.publisher.Mono;
/**
* EventAPIController
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-09-05
*
*/
@RestController
@RequestMapping("/api/v1/")
public class EventAPIController {
private final ProcessorAPIService processorAPIService;
public EventAPIController(ProcessorAPIService processorAPIService) {
this.processorAPIService = processorAPIService;
}
@PostMapping("/exploit/parsePpomppuRSS/{boardName}")
public Mono<String> parsePpomppuRSS(@PathVariable("boardName") PpomppuBoardName boardName) {
return processorAPIService.emitParseEvent(boardName);
}
}

50
receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/dispatch/MessageDispatcher.java
View File

@ -1,50 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.receiver.dispatch;
import java.util.List;
import org.telegram.telegrambots.bots.TelegramLongPollingBot;
import org.telegram.telegrambots.meta.api.objects.Message;
import org.telegram.telegrambots.meta.api.objects.Update;
import com.myoa.engineering.crawl.ppomppu.receiver.handler.message.MessageHandler;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class MessageDispatcher extends TelegramLongPollingBot {
private final List<MessageHandler> messageHandlers;
private final String botName;
private final String botToken;
public MessageDispatcher(List<MessageHandler> messageHandlers, String botName, String botToken) {
this.messageHandlers = messageHandlers;
this.botName = botName;
this.botToken = botToken;
}
@Override
public String getBotToken() {
return botToken;
}
@Override
public void onUpdateReceived(Update update) {
Message message = update.getMessage();
MessageHandler handler = getMessageHandler(message);
handler.handle(message);
}
private MessageHandler getMessageHandler(Message message) {
return messageHandlers.stream()
.filter(e -> e.isApplicable(message))
.findFirst()
.orElseThrow(() -> new IllegalArgumentException("Can not found applicable handler"));
}
@Override
public String getBotUsername() {
return botName;
}
}

25
receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/dto/FeedParsedResult.java
View File

@ -1,25 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.receiver.dto;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import java.io.Serializable;
import java.time.Instant;
import lombok.Getter;
import lombok.NoArgsConstructor;
/**
* FeedParsedResult
*
* @author Shin Woo-jin (woozu.shin@kakaoent.com)
* @since 2021-09-08
*/
@Getter
@NoArgsConstructor
public class FeedParsedResult implements Serializable {
private static final long serialVersionUID = -3771310078623481348L;
private PpomppuBoardName boardName;
private Instant requestedAt;
private Instant processedAt;
}

19
receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/dto/constant/WebClientPropertiesUnitName.java
View File

@ -1,19 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.receiver.dto.constant;
import lombok.AllArgsConstructor;
import lombok.Getter;
/**
* WebClientPropertiesUnitName
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-11-18
*
*/
@Getter
@AllArgsConstructor
public enum WebClientPropertiesUnitName {
PPOMPPU_NOTIFIER_PROCESSOR_API("ppn-processor-api"),
;
private String unitName;
}

21
receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/HelloWorldMessageHandler.java
View File

@ -1,21 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.receiver.handler.message;
import com.myoa.engineering.crawl.ppomppu.support.util.ObjectUtil;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import org.telegram.telegrambots.meta.api.objects.Message;
@Slf4j
@Component
public class HelloWorldMessageHandler implements MessageHandler {
@Override
public boolean isApplicable(Message message) {
return ObjectUtil.isEmpty(message);
}
@Override
public void handle(Message message) {
// skip empty event message.
}
}

12
receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/ImageMessageHandler.java
View File

@ -1,12 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.receiver.handler.message;
/**
* ImageHandler
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-08-21
*
*/
public interface ImageMessageHandler extends MessageHandler {
}

17
receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/MessageHandler.java
View File

@ -1,17 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.receiver.handler.message;
import org.telegram.telegrambots.meta.api.objects.Message;
/**
* MessageHandler
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-08-21
*
*/
public interface MessageHandler {
boolean isApplicable(Message message);
void handle(Message message);
}

18
receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/TextMessageHandler.java
View File

@ -1,18 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.receiver.handler.message;
import com.myoa.engineering.crawl.ppomppu.support.util.ObjectUtil;
import org.telegram.telegrambots.meta.api.objects.Message;
/**
* TextMessageHandler
*
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-08-21
*/
public interface TextMessageHandler extends MessageHandler {
@Override
default boolean isApplicable(Message message) {
return ObjectUtil.isNotEmpty(message) && message.isUserMessage() && message.hasText();
}
}

48
receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/text/CommandHandler.java
View File

@ -1,48 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.receiver.handler.message.text;
import java.util.List;
import org.springframework.stereotype.Component;
import org.telegram.telegrambots.meta.api.objects.Message;
import com.myoa.engineering.crawl.ppomppu.receiver.handler.message.TextMessageHandler;
import lombok.extern.slf4j.Slf4j;
/**
* CommandHandler
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-08-21
*
*/
@Slf4j
@Component
public class CommandHandler implements TextMessageHandler {
private final List<TextCommandProcessor> processors;
public CommandHandler(List<TextCommandProcessor> processors) {
this.processors = processors;
}
@Override
public boolean isApplicable(Message message) {
return TextMessageHandler.super.isApplicable(message)
&& message.isCommand(); // && message.getText().startsWith("/");
}
@Override
public void handle(Message message) {
log.info("CommandHandler : {}", message.getText());
TextCommandCode commandCode = TextCommandCode.find(message.getText());
TextCommandProcessor applicableProcessor = getApplicableProcessor(commandCode);
applicableProcessor.process(message);
}
private TextCommandProcessor getApplicableProcessor(TextCommandCode commandCode) {
return processors.stream()
.filter(e -> e.isApplicable(commandCode))
.findFirst()
.orElseThrow(() -> new IllegalArgumentException("Can not found"));
}
}

30
receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/text/CommonTextHandler.java
View File

@ -1,30 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.receiver.handler.message.text;
import org.springframework.stereotype.Component;
import org.telegram.telegrambots.meta.api.objects.Message;
import com.myoa.engineering.crawl.ppomppu.receiver.handler.message.TextMessageHandler;
import lombok.extern.slf4j.Slf4j;
/**
* NormalTextHandler
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-08-21
*
*/
@Slf4j
@Component
public class CommonTextHandler implements TextMessageHandler {
@Override
public boolean isApplicable(Message message) {
return TextMessageHandler.super.isApplicable(message) && message.isCommand() == false;
}
@Override
public void handle(Message message) {
log.info("CommonTextHandler : {}", message.getText());
}
}

24
receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/text/EmptyTextCommandProcessor.java
View File

@ -1,24 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.receiver.handler.message.text;
import org.springframework.stereotype.Component;
import org.telegram.telegrambots.meta.api.objects.Message;
/**
* EmptyTextCommandProcessor
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-09-05
*
*/
@Component
public class EmptyTextCommandProcessor implements TextCommandProcessor {
@Override
public boolean isApplicable(TextCommandCode commandCode) {
return commandCode == TextCommandCode.EMPTY;
}
@Override
public void process(Message message) {
}
}

27
receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/text/StartTextCommandProcessor.java
View File

@ -1,27 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.receiver.handler.message.text;
import org.springframework.stereotype.Component;
import org.telegram.telegrambots.meta.api.objects.Message;
import lombok.extern.slf4j.Slf4j;
/**
* StartCommandProcessor
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-09-05
*
*/
@Slf4j
@Component
public class StartTextCommandProcessor implements TextCommandProcessor {
@Override
public boolean isApplicable(TextCommandCode commandCode) {
return TextCommandCode.START == commandCode;
}
@Override
public void process(Message message) {
log.info("[process] user: {}, command: {}", message.getChatId(), message.getText());
}
}

33
receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/text/TextCommandCode.java
View File

@ -1,33 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.receiver.handler.message.text;
import java.util.Arrays;
import lombok.Getter;
import lombok.NoArgsConstructor;
/**
* CommandTextCode
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-09-05
*
*/
@Getter
@NoArgsConstructor
public enum TextCommandCode {
EMPTY(null),
START("/start"),
;
private String value;
TextCommandCode(String value) {
this.value = value;
}
public static TextCommandCode find(String value) {
return Arrays.stream(TextCommandCode.values())
.filter(e -> e != EMPTY)
.filter(e -> value.startsWith(e.getValue()))
.findFirst()
.orElse(TextCommandCode.EMPTY);
}
}

17
receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/text/TextCommandProcessor.java
View File

@ -1,17 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.receiver.handler.message.text;
import org.telegram.telegrambots.meta.api.objects.Message;
/**
* TextCommandProcessor
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-09-05
*
*/
public interface TextCommandProcessor {
boolean isApplicable(TextCommandCode commandCode);
void process(Message message);
}

44
receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/infrastructure/client/ProcessorAPIWebClient.java
View File

@ -1,44 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.receiver.infrastructure.client;
import org.springframework.core.ParameterizedTypeReference;
import org.springframework.stereotype.Component;
import org.springframework.web.reactive.function.client.WebClient;
import org.springframework.web.reactive.function.client.WebClientRequestException;
import com.myoa.engineering.crawl.ppomppu.receiver.dto.constant.WebClientPropertiesUnitName;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebClientFilterFactory;
import com.myoa.engineering.crawl.ppomppu.support.webclient.properties.WebClientProperties;
import com.myoa.engineering.crawl.ppomppu.support.webclient.properties.WebClientProperties.WebClientPropertiesUnit;
import lombok.extern.slf4j.Slf4j;
import reactor.core.publisher.Mono;
import reactor.core.scheduler.Schedulers;
@Slf4j
@Component
public class ProcessorAPIWebClient {
private final WebClient webClient;
public ProcessorAPIWebClient(WebClientProperties webClientProperties) {
WebClientPropertiesUnit webClientPropertiesUnit =
webClientProperties.find(WebClientPropertiesUnitName.PPOMPPU_NOTIFIER_PROCESSOR_API.getUnitName());
this.webClient = WebClient.builder()
.baseUrl(webClientPropertiesUnit.getBaseUrl())
.filter(WebClientFilterFactory.logRequest())
.filter(WebClientFilterFactory.logResponse())
.build();
}
public Mono<String> emitParseEvent(PpomppuBoardName boardName) {
return webClient.post()
.uri("/api/v1/crawl/boards/{boardName}", boardName)
.exchangeToMono(e -> e.bodyToMono(new ParameterizedTypeReference<String>() {}))
.publishOn(Schedulers.boundedElastic())
.onErrorResume(WebClientRequestException.class, t -> {
log.info("Exception occured, ignoring. : {}", t.getClass().getSimpleName());
return Mono.empty();
});
}
}

30
receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/service/ProcessorAPIService.java
View File

@ -1,30 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.receiver.service;
import org.springframework.stereotype.Service;
import com.myoa.engineering.crawl.ppomppu.receiver.infrastructure.client.ProcessorAPIWebClient;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import lombok.extern.slf4j.Slf4j;
import reactor.core.publisher.Mono;
/**
* ProcessorAPIService
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-09-05
*
*/
@Slf4j
@Service
public class ProcessorAPIService {
private final ProcessorAPIWebClient processorAPIWebClient;
public ProcessorAPIService(ProcessorAPIWebClient processorAPIWebClient) {
this.processorAPIWebClient = processorAPIWebClient;
}
public Mono<String> emitParseEvent(PpomppuBoardName boardName) {
return processorAPIWebClient.emitParseEvent(boardName);
}
}

6
receiver/src/main/resources/application-development.yml
View File

@ -1,6 +0,0 @@
spring:
config:
activate:
on-profile: development
import:
- "configserver:http://192.168.0.100:11080"

6
receiver/src/main/resources/application-local.yml
View File

@ -1,6 +0,0 @@
spring:
config:
activate:
on-profile: local
import:
- "configserver:http://localhost:20085"

15
receiver/src/main/resources/logback-spring.xml
View File

@ -1,15 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<configuration>
<springProperty name="DEFAULT_LEVEL_CONFIG" source="log.defaultLevel" />
<springProfile name="local">
<include resource="logback/logback-development.xml" />
<logger name="org.apache.kafka" level="INFO" />
</springProfile>
<springProfile name="development">
<include resource="logback/logback-development.xml" />
<logger name="org.apache.kafka" level="INFO" />
</springProfile>
<springProfile name="production">
<include resource="logback/logback-production.xml" />
</springProfile>
</configuration>

23
receiver/src/main/resources/logback/component/logback-file.xml
View File

@ -1,23 +0,0 @@
<included>
<property name="FILE_LOG_PATTERN"
value="%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{45}:%L - %msg%n" />
<property name="LOG_FILE_BASE" value="lcp-benefit-benefit-api" />
<appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${DIRECTORY}/${LOG_FILE_BASE}_log</file>
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<fileNamePattern>${DIRECTORY}/${LOG_FILE_BASE}_log.%d{yyyyMMdd}.%i</fileNamePattern>
<timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
<maxFileSize>1000MB</maxFileSize>
</timeBasedFileNamingAndTriggeringPolicy>
<maxHistory>60</maxHistory>
</rollingPolicy>
<encoder>
<pattern>${FILE_LOG_PATTERN}</pattern>
<immediateFlush>${IMMEDIATE_FLUSH}</immediateFlush>
</encoder>
</appender>
<appender name="ASYNC_FILE" class="ch.qos.logback.classic.AsyncAppender">
<queueSize>1024</queueSize>
<appender-ref ref="FILE" />
</appender>
</included>

19
receiver/src/main/resources/logback/logback-development.xml
View File

@ -1,19 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<included>
<!-- =========== property BETA ========= -->
<property name="DEFAULT_LEVEL" value="${DEFAULT_LEVEL_CONFIG:-INFO}"/>
<!--file-->
<property name="DIRECTORY" value="/home1/www/logs/supervisor"/>
<property name="IMMEDIATE_FLUSH" value="true"/>
<!--nelo2-->
<property name="NELO2_LEVEL" value="WARN"/>
<!-- =========== include appender =========== -->
<include resource="org/springframework/boot/logging/logback/defaults.xml"/>
<include resource="org/springframework/boot/logging/logback/console-appender.xml"/>
<include resource="logback/component/logback-nelo2.xml"/>
<include resource="logback/component/logback-datachain.xml"/>
<!-- =========== root logger ============== -->
<root level="${DEFAULT_LEVEL}">
<appender-ref ref="CONSOLE"/>
</root>
</included>

19
receiver/src/main/resources/logback/logback-production.xml
View File

@ -1,19 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<included>
<!-- =========== property RELEASE ========= -->
<property name="DEFAULT_LEVEL" value="${DEFAULT_LEVEL_CONFIG:-INFO}"/>
<!--file-->
<property name="DIRECTORY" value="/home1/www/logs/supervisor"/>
<property name="IMMEDIATE_FLUSH" value="true"/>
<!--nelo2-->
<property name="NELO2_LEVEL" value="WARN"/>
<!-- =========== include appender =========== -->
<include resource="org/springframework/boot/logging/logback/defaults.xml"/>
<include resource="org/springframework/boot/logging/logback/console-appender.xml"/>
<include resource="logback/component/logback-nelo2.xml"/>
<include resource="logback/component/logback-datachain.xml"/>
<!-- =========== root logger ============== -->
<root level="${DEFAULT_LEVEL}">
<appender-ref ref="CONSOLE"/>
</root>
</included>

23
sender/build.gradle
View File

@ -1,23 +0,0 @@
dependencies {
developmentOnly 'org.springframework.boot:spring-boot-devtools'
runtimeOnly 'mysql:mysql-connector-java'
compileOnly 'org.projectlombok:lombok'
implementation project(':support')
// https://projectreactor.io/docs/core/release/reference/#debug-activate
implementation 'org.springframework.boot:spring-boot-starter-webflux'
implementation 'org.springframework.boot:spring-boot-starter-actuator'
implementation 'org.springframework.cloud:spring-cloud-starter-config'
annotationProcessor 'org.springframework.boot:spring-boot-configuration-processor'
annotationProcessor 'org.projectlombok:lombok'
testImplementation 'org.springframework.boot:spring-boot-starter-test'
}
test {
useJUnitPlatform()
testLogging {
events "passed", "skipped", "failed"
}
}

22
sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/SenderApplication.java
View File

@ -1,22 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.sender;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.context.annotation.Import;
import com.myoa.engineering.crawl.ppomppu.support.webclient.PpomppuNotifierWebClientConfiguration;
/**
* SenderApplication
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-08-20
*
*/
@Import({ PpomppuNotifierWebClientConfiguration.class })
@SpringBootApplication
public class SenderApplication {
public static void main(String[] args) {
SpringApplication.run(SenderApplication.class, args);
}
}

33
sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/controller/TestAPIController.java
View File

@ -1,33 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.sender.controller;
import com.myoa.engineering.crawl.ppomppu.sender.infrastructure.client.MongeShoppingBotSlackMessageSender;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import lombok.extern.slf4j.Slf4j;
import reactor.core.publisher.Mono;
/**
* TestAPIController
*
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-11-15
*/
@Slf4j
@RestController
@RequestMapping("/api/v1")
public class TestAPIController {
private final MongeShoppingBotSlackMessageSender sender;
public TestAPIController(MongeShoppingBotSlackMessageSender sender) {
this.sender = sender;
}
@GetMapping("/test")
public Mono<String> test() {
log.info("received!!!");
return sender.sendMessage(sender.ofMessage("testtesttest!!!"));
}
}

17
sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/infrastructure/client/MessageSender.java
View File

@ -1,17 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.sender.infrastructure.client;
import com.myoa.engineering.crawl.ppomppu.sender.dto.MessageDTO;
import reactor.core.publisher.Mono;
/**
* MessageSender
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-11-14
*
*/
public interface MessageSender<T extends MessageDTO> {
Mono<String> sendMessage(T message);
}

10
sender/src/main/resources/application-development.yml
View File

@ -1,10 +0,0 @@
spring:
config:
activate:
on-profile: development
import:
- "configserver:http://192.168.0.100:11080"
# import: optional:configserver:http://localhost:11080 # can be start up even config server was not found.
server:
port: 20082

6
sender/src/main/resources/application-production.yml
View File

@ -1,6 +0,0 @@
spring:
config:
activate:
on-profile: production
import:
- "configserver:http://ppn-config-server:20080"

15
sender/src/main/resources/logback-spring.xml
View File

@ -1,15 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<configuration>
<springProfile name="local">
<include resource="logback/logback-development.xml" />
<logger name="org.apache.kafka" level="INFO" />
</springProfile>
<springProperty name="DEFAULT_LEVEL_CONFIG" source="log.defaultLevel" />
<springProfile name="development">
<include resource="logback/logback-development.xml" />
<logger name="org.apache.kafka" level="INFO" />
</springProfile>
<springProfile name="production">
<include resource="logback/logback-production.xml" />
</springProfile>
</configuration>

23
sender/src/main/resources/logback/component/logback-file.xml
View File

@ -1,23 +0,0 @@
<included>
<property name="FILE_LOG_PATTERN"
value="%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{45}:%L - %msg%n" />
<property name="LOG_FILE_BASE" value="lcp-benefit-benefit-api" />
<appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${DIRECTORY}/${LOG_FILE_BASE}_log</file>
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<fileNamePattern>${DIRECTORY}/${LOG_FILE_BASE}_log.%d{yyyyMMdd}.%i</fileNamePattern>
<timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
<maxFileSize>1000MB</maxFileSize>
</timeBasedFileNamingAndTriggeringPolicy>
<maxHistory>60</maxHistory>
</rollingPolicy>
<encoder>
<pattern>${FILE_LOG_PATTERN}</pattern>
<immediateFlush>${IMMEDIATE_FLUSH}</immediateFlush>
</encoder>
</appender>
<appender name="ASYNC_FILE" class="ch.qos.logback.classic.AsyncAppender">
<queueSize>1024</queueSize>
<appender-ref ref="FILE" />
</appender>
</included>

19
sender/src/main/resources/logback/logback-development.xml
View File

@ -1,19 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<included>
<!-- =========== property BETA ========= -->
<property name="DEFAULT_LEVEL" value="${DEFAULT_LEVEL_CONFIG:-INFO}"/>
<!--file-->
<property name="DIRECTORY" value="/home1/www/logs/supervisor"/>
<property name="IMMEDIATE_FLUSH" value="true"/>
<!--nelo2-->
<property name="NELO2_LEVEL" value="WARN"/>
<!-- =========== include appender =========== -->
<include resource="org/springframework/boot/logging/logback/defaults.xml"/>
<include resource="org/springframework/boot/logging/logback/console-appender.xml"/>
<include resource="logback/component/logback-nelo2.xml"/>
<include resource="logback/component/logback-datachain.xml"/>
<!-- =========== root logger ============== -->
<root level="${DEFAULT_LEVEL}">
<appender-ref ref="CONSOLE"/>
</root>
</included>

19
sender/src/main/resources/logback/logback-production.xml
View File

@ -1,19 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<included>
<!-- =========== property RELEASE ========= -->
<property name="DEFAULT_LEVEL" value="${DEFAULT_LEVEL_CONFIG:-INFO}"/>
<!--file-->
<property name="DIRECTORY" value="/home1/www/logs/supervisor"/>
<property name="IMMEDIATE_FLUSH" value="true"/>
<!--nelo2-->
<property name="NELO2_LEVEL" value="WARN"/>
<!-- =========== include appender =========== -->
<include resource="org/springframework/boot/logging/logback/defaults.xml"/>
<include resource="org/springframework/boot/logging/logback/console-appender.xml"/>
<include resource="logback/component/logback-nelo2.xml"/>
<include resource="logback/component/logback-datachain.xml"/>
<!-- =========== root logger ============== -->
<root level="${DEFAULT_LEVEL}">
<appender-ref ref="CONSOLE"/>
</root>
</included>

16
settings.gradle
View File

@ -1,14 +1,14 @@
rootProject.name = 'PpomppuNotifier'
include 'receiver'
project(':receiver').projectDir = "$rootDir/receiver" as File
include 'processor'
project(':processor').projectDir = "$rootDir/processor" as File
include 'sender'
project(':sender').projectDir = "$rootDir/sender" as File
//include 'reader'
//project(':reader').projectDir = "$rootDir/reader" as File
//include 'processor'
//project(':processor').projectDir = "$rootDir/processor" as File
//include 'writer'
//project(':writer').projectDir = "$rootDir/writer" as File
include 'shopping-crawler'
project(':shopping-crawler').projectDir = "$rootDir/shopping-crawler" as File
include 'support'
project(':support').projectDir = "$rootDir/support" as File

BIN
shopping-crawler/.jqwik-database Normal file
View File

Binary file not shown.

42
shopping-crawler/build.gradle Normal file
View File

@ -0,0 +1,42 @@
dependencies {
developmentOnly 'org.springframework.boot:spring-boot-devtools'
runtimeOnly 'com.h2database:h2'
runtimeOnly 'com.mysql:mysql-connector-j'
compileOnly 'org.projectlombok:lombok'
implementation project(':support')
// https://projectreactor.io/docs/core/release/reference/#debug-activate
implementation("org.springframework.boot:spring-boot-starter-web") {
exclude group: "org.springframework.boot", module: "spring-boot-starter-tomcat"
}
implementation("org.springframework.boot:spring-boot-starter-undertow") {
exclude group: "io.undertow", module: "undertow-websockets-jsr"
}
implementation 'org.springframework.boot:spring-boot-starter-data-jpa'
implementation 'org.springframework.boot:spring-boot-configuration-processor'
implementation 'org.springframework.cloud:spring-cloud-starter-config'
implementation 'org.springframework.boot:spring-boot-starter-actuator'
implementation 'com.rometools:rome:2.1.0'
implementation 'org.jsoup:jsoup:1.17.2'
implementation 'com.h2database:h2:2.2.224'
implementation "org.springframework.cloud:spring-cloud-starter-openfeign"
implementation "io.github.openfeign:feign-hc5"
implementation 'org.ahocorasick:ahocorasick:0.6.3'
annotationProcessor 'org.springframework.boot:spring-boot-configuration-processor'
annotationProcessor 'org.projectlombok:lombok'
testImplementation 'org.springframework.boot:spring-boot-starter-test'
testImplementation('org.assertj:assertj-core:3.25.3')
testImplementation("org.jeasy:easy-random-core:5.0.0")
testCompileOnly 'org.projectlombok:lombok'
testAnnotationProcessor('org.projectlombok:lombok')
}
test {
useJUnitPlatform()
testLogging {
events "passed", "skipped", "failed"
}
}

19
shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/ShoppingCrawlerApplication.java Normal file
View File

@ -0,0 +1,19 @@
package com.myoa.engineering.crawl.shopping;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.cloud.openfeign.EnableFeignClients;
import org.springframework.scheduling.annotation.EnableScheduling;
@EnableScheduling
@EnableFeignClients
@EnableConfigurationProperties
@SpringBootApplication
public class ShoppingCrawlerApplication {
public static void main(String[] args) {
SpringApplication.run(ShoppingCrawlerApplication.class, args);
}
}

30
shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/FeignDefaultConfig.java Normal file
View File

@ -0,0 +1,30 @@
package com.myoa.engineering.crawl.shopping.configuration;
import feign.Logger;
import feign.RequestInterceptor;
import feign.codec.ErrorDecoder;
import lombok.extern.slf4j.Slf4j;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.http.HttpHeaders;
import org.springframework.http.MediaType;
@Slf4j
@Configuration
public class FeignDefaultConfig {
@Bean
Logger.Level feignLoggerLevel() {
return Logger.Level.FULL;
}
public static final String MIME_TYPE =
MediaType.APPLICATION_JSON_VALUE + ";charset=utf-8";
@Bean
public RequestInterceptor requestInterceptor() {
return requestTemplate -> requestTemplate.header(HttpHeaders.CONTENT_TYPE, MIME_TYPE);
}
}

5
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/H2ConsoleConfiguration.java → shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/datasource/H2ConsoleConfiguration.java
View File

@ -1,6 +1,5 @@
package com.myoa.engineering.crawl.ppomppu.processor.configuration;
package com.myoa.engineering.crawl.shopping.configuration.datasource;
import java.sql.SQLException;
import lombok.extern.slf4j.Slf4j;
import org.h2.tools.Server;
import org.springframework.beans.factory.annotation.Value;
@ -10,6 +9,8 @@ import org.springframework.context.event.ContextClosedEvent;
import org.springframework.context.event.ContextRefreshedEvent;
import org.springframework.context.event.EventListener;
import java.sql.SQLException;
@Slf4j
@Profile({"datasource-local", "datasource-development"})
@Configuration

73
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/PpomppuDatasourceConfiguration.java → shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/datasource/ShoppingCrawlerDatasourceConfiguration.java
View File

@ -1,44 +1,48 @@
package com.myoa.engineering.crawl.ppomppu.processor.configuration;
package com.myoa.engineering.crawl.shopping.configuration.datasource;
import com.myoa.engineering.crawl.ppomppu.processor.configuration.properties.DatasourceProperties;
import com.myoa.engineering.crawl.ppomppu.processor.configuration.properties.DatasourceProperties.DataSourcePropertiesUnit;
import com.myoa.engineering.crawl.ppomppu.processor.configuration.properties.HibernateProperties;
import com.myoa.engineering.crawl.ppomppu.processor.configuration.properties.HikariProperties;
import com.myoa.engineering.crawl.ppomppu.processor.domain.BaseScanDomain;
import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository.BaseScanRepository;
import com.myoa.engineering.crawl.shopping.configuration.datasource.properties.DatasourceProperties;
import com.myoa.engineering.crawl.shopping.configuration.datasource.properties.HibernateProperties;
import com.myoa.engineering.crawl.shopping.configuration.datasource.properties.HikariProperties;
import com.myoa.engineering.crawl.shopping.domain.entity.BaseScanDomain;
import com.myoa.engineering.crawl.shopping.infra.repository.BaseScanRepository;
import com.zaxxer.hikari.HikariConfig;
import com.zaxxer.hikari.HikariDataSource;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
import javax.persistence.EntityManagerFactory;
import javax.sql.DataSource;
import jakarta.persistence.EntityManagerFactory;
import lombok.NonNull;
import org.hibernate.boot.model.naming.CamelCaseToUnderscoresNamingStrategy;
import org.hibernate.boot.model.naming.ImplicitNamingStrategyJpaCompliantImpl;
import org.hibernate.cfg.AvailableSettings;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.boot.orm.jpa.EntityManagerFactoryBuilder;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.data.jpa.repository.config.EnableJpaAuditing;
import org.springframework.data.jpa.repository.config.EnableJpaRepositories;
import org.springframework.orm.jpa.JpaTransactionManager;
import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean;
import org.springframework.transaction.PlatformTransactionManager;
@Configuration
@EnableJpaRepositories(basePackageClasses = BaseScanRepository.class,
entityManagerFactoryRef = "ppomppuNotifierProcessorEntityManagerFactory",
transactionManagerRef = "ppomppuNotifierProcessorTransactionManager"
)
public class PpomppuDatasourceConfiguration {
import javax.sql.DataSource;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
private static final String DATA_SOURCE_UNIT_NAME = "ppn_mysql";
@Configuration
@EnableJpaAuditing
@EnableJpaRepositories(basePackageClasses = BaseScanRepository.class,
entityManagerFactoryRef = "shoppingCrawlerEntityManagerFactory",
transactionManagerRef = "shoppingCrawlerTransactionManager"
)
public class ShoppingCrawlerDatasourceConfiguration {
private static final String DATA_SOURCE_UNIT_NAME = "crawler-shopping";
private final DatasourceProperties dataSourceProeprties;
private final HikariProperties hikariProperties;
private final HibernateProperties hibernateProperties;
public PpomppuDatasourceConfiguration(DatasourceProperties dataSourceProeprties,
public ShoppingCrawlerDatasourceConfiguration(DatasourceProperties dataSourceProeprties,
HikariProperties hikariProperties,
HibernateProperties hibernateProperties) {
this.dataSourceProeprties = dataSourceProeprties;
@ -46,9 +50,9 @@ public class PpomppuDatasourceConfiguration {
this.hibernateProperties = hibernateProperties;
}
@Bean(name = "ppomppuNotifierProcessorDataSource")
@Bean(name = "shoppingCrawlerDataSource")
public DataSource dataSource() {
DataSourcePropertiesUnit dataSourcePropertiesUnit = dataSourceProeprties.find(DATA_SOURCE_UNIT_NAME);
DatasourceProperties.DataSourcePropertiesUnit dataSourcePropertiesUnit = dataSourceProeprties.find(DATA_SOURCE_UNIT_NAME);
final HikariConfig hikariConfig = new HikariConfig();
hikariConfig.setJdbcUrl(dataSourcePropertiesUnit.toCompletedJdbcUrl());
@ -69,19 +73,19 @@ public class PpomppuDatasourceConfiguration {
return dataSource;
}
@Bean("ppomppuNotifierProcessorEntityManagerFactory")
@Bean("shoppingCrawlerEntityManagerFactory")
public LocalContainerEntityManagerFactoryBean entityManagerFactory(
EntityManagerFactoryBuilder builder,
@Qualifier("ppomppuNotifierProcessorDataSource") DataSource dataSource) {
@Qualifier("shoppingCrawlerDataSource") DataSource dataSource) {
return builder.dataSource(dataSource)
.packages(BaseScanDomain.class)
.properties(getPropsMap(DATA_SOURCE_UNIT_NAME))
.build();
}
@Bean("ppomppuNotifierProcessorTransactionManager")
@Bean("shoppingCrawlerTransactionManager")
public PlatformTransactionManager transactionManager(
@Qualifier("ppomppuNotifierProcessorEntityManagerFactory") EntityManagerFactory entityManagerFactory) {
@Qualifier("shoppingCrawlerEntityManagerFactory") EntityManagerFactory entityManagerFactory) {
return new JpaTransactionManager(entityManagerFactory);
}
@ -91,20 +95,17 @@ public class PpomppuDatasourceConfiguration {
properties.put(AvailableSettings.FORMAT_SQL, hibernateProperties.getFormatSql());
properties.put(AvailableSettings.SHOW_SQL, hibernateProperties.getShowSql());
properties.put(AvailableSettings.HBM2DDL_AUTO, hibernateProperties.getHbm2ddlAuto());
properties.put(AvailableSettings.CONNECTION_PROVIDER_DISABLES_AUTOCOMMIT,
hibernateProperties.getDisableAutoCommit());
properties.put(AvailableSettings.IMPLICIT_NAMING_STRATEGY,
"org.springframework.boot.orm.jpa.hibernate.SpringImplicitNamingStrategy");
properties.put(AvailableSettings.PHYSICAL_NAMING_STRATEGY,
"org.springframework.boot.orm.jpa.hibernate.SpringPhysicalNamingStrategy");
properties.put(AvailableSettings.CONNECTION_PROVIDER_DISABLES_AUTOCOMMIT, hibernateProperties.getDisableAutoCommit());
properties.put(AvailableSettings.IMPLICIT_NAMING_STRATEGY, ImplicitNamingStrategyJpaCompliantImpl.class.getName());
properties.put(AvailableSettings.PHYSICAL_NAMING_STRATEGY, CamelCaseToUnderscoresNamingStrategy.class.getName());
properties.put(AvailableSettings.GENERATE_STATISTICS, "false");
properties.put(AvailableSettings.GLOBALLY_QUOTED_IDENTIFIERS, "true");
properties.put(AvailableSettings.GLOBALLY_QUOTED_IDENTIFIERS_SKIP_COLUMN_DEFINITIONS, "true");
// properties.put(AvailableSettings.GLOBALLY_QUOTED_IDENTIFIERS, "true");
// properties.put(AvailableSettings.GLOBALLY_QUOTED_IDENTIFIERS_SKIP_COLUMN_DEFINITIONS, "true");
properties.put(AvailableSettings.STATEMENT_BATCH_SIZE, "20");
properties.put(AvailableSettings.ORDER_INSERTS, "true");
properties.put(AvailableSettings.ORDER_UPDATES, "true");
properties.put(AvailableSettings.BATCH_VERSIONED_DATA, "true");
properties.put(AvailableSettings.USE_NEW_ID_GENERATOR_MAPPINGS, "false");
// properties.put(AvailableSettings.JPA_ID_GENERATOR_GLOBAL_SCOPE_COMPLIANCE, "false");
return properties;
}

25
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/properties/DatasourceProperties.java → shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/datasource/properties/DatasourceProperties.java
View File

@ -1,29 +1,26 @@
package com.myoa.engineering.crawl.ppomppu.processor.configuration.properties;
package com.myoa.engineering.crawl.shopping.configuration.datasource.properties;
import com.myoa.engineering.crawl.ppomppu.support.util.ObjectUtil;
import java.util.List;
import lombok.Getter;
import lombok.Setter;
import com.myoa.engineering.crawl.shopping.support.util.ObjectUtil;
import lombok.Data;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;
import java.util.List;
@Component
@Setter
@Getter
@ConfigurationProperties(prefix = "infra.database")
@Data
@ConfigurationProperties(prefix = "datasource")
public class DatasourceProperties {
private List<DataSourcePropertiesUnit> units;
@Getter
@Setter
@Data
public static class DataSourcePropertiesUnit {
private String unitName;
private String schemaName;
private String connectionParameters;
private String datasourceUrl;
private String dbConnectionUrl;
private Boolean isSimpleConnectionUrl;
private String username;
private String password;
@ -31,9 +28,9 @@ public class DatasourceProperties {
public String toCompletedJdbcUrl() {
if (ObjectUtil.isEmpty(isSimpleConnectionUrl) || isSimpleConnectionUrl == false) {
return String.format("%s/%s?%s", datasourceUrl, schemaName, connectionParameters);
return String.format("%s/%s?%s", dbConnectionUrl, schemaName, connectionParameters);
}
return datasourceUrl;
return dbConnectionUrl;
}
}

8
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/properties/HibernateProperties.java → shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/datasource/properties/HibernateProperties.java
View File

@ -1,14 +1,12 @@
package com.myoa.engineering.crawl.ppomppu.processor.configuration.properties;
package com.myoa.engineering.crawl.shopping.configuration.datasource.properties;
import java.util.List;
import lombok.Getter;
import lombok.Setter;
import org.hibernate.dialect.MySQL8Dialect;
import org.hibernate.dialect.MySQLDialect;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;
import java.util.List;
@Component
@Setter
@Getter

2
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/properties/HikariProperties.java → shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/datasource/properties/HikariProperties.java
View File

@ -1,4 +1,4 @@
package com.myoa.engineering.crawl.ppomppu.processor.configuration.properties;
package com.myoa.engineering.crawl.shopping.configuration.datasource.properties;
import lombok.Getter;
import lombok.Setter;

12
sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/configuration/properties/SlackSecretProperties.java → shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/slack/properties/SlackSecretProperties.java
View File

@ -1,4 +1,4 @@
package com.myoa.engineering.crawl.ppomppu.sender.configuration.properties;
package com.myoa.engineering.crawl.shopping.configuration.slack.properties;
import java.util.List;
import lombok.Data;
@ -7,18 +7,16 @@ import lombok.Setter;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;
@Getter
@Setter
@Data
@Component
@ConfigurationProperties("infra.slack.bot")
@ConfigurationProperties(prefix = "slack.bot")
public class SlackSecretProperties {
private List<SlackSecretPropertiesUnit> units;
@Data
public static class SlackSecretPropertiesUnit {
private String botName;
private String botUnitName;
private String username;
private String iconEmoji;
private String channel;
@ -27,7 +25,7 @@ public class SlackSecretProperties {
public SlackSecretPropertiesUnit find(String botUnitName) {
return units.stream()
.filter(e -> e.getBotName().equals(botUnitName))
.filter(e -> e.getBotUnitName().equals(botUnitName))
.findFirst()
.orElseThrow(() -> new IllegalArgumentException("not found bot unit name : " + botUnitName));
}

22
shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/controller/TestAPIController.java Normal file
View File

@ -0,0 +1,22 @@
package com.myoa.engineering.crawl.shopping.controller;
import com.myoa.engineering.crawl.shopping.crawlhandler.PpomppuCrawlDomesticHandler;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
@RestController
@RequestMapping("/api/v1/exploit")
public class TestAPIController {
private final PpomppuCrawlDomesticHandler ppomppuCrawlDomesticHandler;
public TestAPIController(PpomppuCrawlDomesticHandler ppomppuCrawlDomesticHandler) {
this.ppomppuCrawlDomesticHandler = ppomppuCrawlDomesticHandler;
}
@GetMapping("/triggers")
public void triggerExploit() {
ppomppuCrawlDomesticHandler.handle();
}
}

10
shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/crawlhandler/CrawlHandler.java Normal file
View File

@ -0,0 +1,10 @@
package com.myoa.engineering.crawl.shopping.crawlhandler;
import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget;
public interface CrawlHandler {
CrawlTarget getCrawlTarget();
void handle();
}

18
shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/crawlhandler/FmkoreaCrawlHandler.java Normal file
View File

@ -0,0 +1,18 @@
package com.myoa.engineering.crawl.shopping.crawlhandler;
import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
@Slf4j
@Component
public class FmkoreaCrawlHandler implements CrawlHandler {
@Override
public CrawlTarget getCrawlTarget() {
return CrawlTarget.FMKOREA;
}
@Override
public void handle() {
}
}

58
shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/crawlhandler/PpomppuCrawlDomesticHandler.java Normal file
View File

@ -0,0 +1,58 @@
package com.myoa.engineering.crawl.shopping.crawlhandler;
import com.myoa.engineering.crawl.shopping.crawlhandler.parser.PpomppuArticleParserV2;
import com.myoa.engineering.crawl.shopping.domain.entity.v2.Article;
import com.myoa.engineering.crawl.shopping.infra.client.ppomppu.PpomppuBoardClientV2;
import com.myoa.engineering.crawl.shopping.service.ArticleCommandService;
import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget;
import org.springframework.stereotype.Component;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Stream;
@Component
public class PpomppuCrawlDomesticHandler implements CrawlHandler {
private final PpomppuBoardClientV2 ppomppuBoardClient;
private final PpomppuArticleParserV2 ppomppuArticleParserV2;
private final ArticleCommandService articleCommandService;
public PpomppuCrawlDomesticHandler(PpomppuBoardClientV2 ppomppuBoardClient,
PpomppuArticleParserV2 ppomppuArticleParserV2,
ArticleCommandService articleCommandService) {
this.ppomppuBoardClient = ppomppuBoardClient;
this.ppomppuArticleParserV2 = ppomppuArticleParserV2;
this.articleCommandService = articleCommandService;
}
@Override
public CrawlTarget getCrawlTarget() {
return CrawlTarget.PPOMPPU_DOMESTIC;
}
@Override
public void handle() {
String boardHtmlPage1 = ppomppuBoardClient.getBoardHtml("/zboard/zboard.php", generateRequestParams(1));
List<Article> parsedPage1 = ppomppuArticleParserV2.parse(boardHtmlPage1);
String boardHtmlPage2 = ppomppuBoardClient.getBoardHtml("/zboard/zboard.php", generateRequestParams(2));
List<Article> parsedPage2 = ppomppuArticleParserV2.parse(boardHtmlPage2);
List<Article> merged = Stream.of(parsedPage1, parsedPage2)
.flatMap(List::stream)
.map(e -> e.updateCrawlTarget(getCrawlTarget()))
.toList();
articleCommandService.upsert(merged);
}
private Map<String, String> generateRequestParams(int pageId) {
Map<String, String> params = new HashMap<>();
params.put("id", "ppomppu");
params.put("page", String.valueOf(pageId));
return params;
}
}

115
shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/crawlhandler/parser/PpomppuArticleParserV2.java Normal file
View File

@ -0,0 +1,115 @@
package com.myoa.engineering.crawl.shopping.crawlhandler.parser;
import com.myoa.engineering.crawl.shopping.domain.entity.v2.Article;
import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName;
import com.myoa.engineering.crawl.shopping.util.DateTimeUtils;
import com.myoa.engineering.crawl.shopping.util.NumberUtils;
import io.micrometer.core.instrument.util.StringUtils;
import lombok.extern.slf4j.Slf4j;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.stereotype.Component;
import java.time.ZoneId;
import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@Slf4j
@Component
public final class PpomppuArticleParserV2 {
private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yy.MM.dd HH:mm:ss")
.withZone(ZoneId.of("Asia/Seoul"));
private PpomppuArticleParserV2() {
}
public List<Article> parse(String html) {
Elements trElements = converHtmlToTrElements(html);
return trElements.stream()
.filter(this::isRealArticle)
.map(this::parse)
.toList();
}
private Elements converHtmlToTrElements(String data) {
Document document = Jsoup.parse(data);
Elements trList = document.getElementById("revolution_main_table").getElementsByTag("tr");
return trList;
}
private boolean isRealArticle(Element tr) {
Elements tdList = tr.getElementsByTag("td");
if (tdList.size() != 6) {
return false;
}
if (!hasOnlyNumeric(tdList.get(0))) {
return false;
}
return true;
}
private Pattern pattern_numeric = Pattern.compile("\\d+");
private boolean hasOnlyNumeric(Element td) {
return pattern_numeric.matcher(td.text()).matches();
}
private Article parse(Element tr) {
Elements tdList = tr.getElementsByTag("td");
Long articleId = Long.parseLong(tdList.get(0).text());
String title = tdList.get(2).text();
String articleUrl = parseArticleUrl(tdList.get(2).getElementsByTag("a").attr("href"));
String boardName = parseBoardName(title);
Integer recommended = parseRecommended(tdList.get(4));
Integer hit = NumberUtils.parseInt(tdList.get(5).text(), 0);
ZonedDateTime registeredAt = DateTimeUtils.parse(tdList.get(3).text());
return Article.builder()
.articleId(articleId)
.title(title)
.boardName(boardName)
.articleUrl(articleUrl)
.recommended(recommended)
.hit(hit)
.registeredAt(registeredAt)
.build();
}
public Integer parseRecommended(Element td) {
final String voteString = td.text();
if (StringUtils.isEmpty(voteString)) {
return null;
}
final int voteUp = Integer.parseInt(td.text().split(" - ")[0]);
final int voteDown = Integer.parseInt(td.text().split(" - ")[1]);
int recommended = voteUp - voteDown;
return recommended;
}
public static String parseArticleUrl(String data) {
return PpomppuBoardName.ofViewPageUrl(data);
}
Pattern patternBoardName = Pattern.compile("\\[(.+?)\\]");
public String parseBoardName(String fullTitle) {
Matcher matcher = patternBoardName.matcher(fullTitle);
String lastMatched = null;
while (matcher.find()) {
lastMatched = matcher.group(1);
}
return lastMatched;
}
}

21
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/Auditable.java → shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/Auditable.java
View File

@ -1,17 +1,18 @@
package com.myoa.engineering.crawl.ppomppu.processor.domain;
package com.myoa.engineering.crawl.shopping.domain.entity;
import java.io.Serializable;
import java.time.Instant;
import javax.persistence.Column;
import lombok.Getter;
import org.springframework.data.annotation.CreatedDate;
import org.springframework.data.annotation.LastModifiedDate;
/**
* Auditable
*
* @author Shin Woo-jin (woozu.shin@kakaoent.com)
* @since 2021-09-08
*/
import jakarta.persistence.*;
import org.springframework.data.jpa.domain.support.AuditingEntityListener;
import java.io.Serializable;
import java.time.Instant;
@Getter
@MappedSuperclass
@EntityListeners(AuditingEntityListener.class)
public abstract class Auditable implements Serializable {
private static final long serialVersionUID = -7105030870015828551L;

2
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/BaseScanDomain.java → shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/BaseScanDomain.java
View File

@ -1,4 +1,4 @@
package com.myoa.engineering.crawl.ppomppu.processor.domain;
package com.myoa.engineering.crawl.shopping.domain.entity;
/**
* BaseScanDomain

0
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/PpomppuArticle.java → shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v1/PpomppuArticle.java
View File

17
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/PpomppuBoardFeedStatus.java → shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v1/PpomppuBoardFeedStatus.java
View File

@ -1,19 +1,14 @@
package com.myoa.engineering.crawl.ppomppu.processor.domain;
package com.myoa.engineering.crawl.shopping.domain.entity.v1;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import java.time.Instant;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.EnumType;
import javax.persistence.Enumerated;
import javax.persistence.GeneratedValue;
import javax.persistence.GenerationType;
import javax.persistence.Id;
import javax.persistence.Table;
import com.myoa.engineering.crawl.shopping.domain.entity.Auditable;
import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName;
import jakarta.persistence.*;
import lombok.Builder;
import lombok.Getter;
import lombok.NoArgsConstructor;
import java.time.Instant;
@Getter
@NoArgsConstructor
@Entity

13
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/PublishedHistory.java → shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v1/PublishedHistory.java
View File

@ -1,15 +1,12 @@
package com.myoa.engineering.crawl.ppomppu.processor.domain;
package com.myoa.engineering.crawl.shopping.domain.entity.v1;
import java.time.Instant;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.GeneratedValue;
import javax.persistence.GenerationType;
import javax.persistence.Id;
import javax.persistence.Table;
import com.myoa.engineering.crawl.shopping.domain.entity.Auditable;
import lombok.Getter;
import lombok.NoArgsConstructor;
import jakarta.persistence.*;
import java.time.Instant;
@Getter
@NoArgsConstructor
@Entity

27
shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v1/SubscribedBoard.java Normal file
View File

@ -0,0 +1,27 @@
package com.myoa.engineering.crawl.shopping.domain.entity.v1;
import com.myoa.engineering.crawl.shopping.domain.entity.Auditable;
import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName;
import lombok.Getter;
import lombok.NoArgsConstructor;
import jakarta.persistence.*;
@Getter
@NoArgsConstructor
@Entity
@Table(name = "subscribed_board")
public class SubscribedBoard extends Auditable {
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
private Long id;
@Column
private Long userId;
@Column
@Enumerated(EnumType.STRING)
private PpomppuBoardName boardName;
}

15
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/SubscribedUser.java → shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v1/SubscribedUser.java
View File

@ -1,20 +1,17 @@
package com.myoa.engineering.crawl.ppomppu.processor.domain;
package com.myoa.engineering.crawl.shopping.domain.entity.v1;
import java.time.Instant;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.GeneratedValue;
import javax.persistence.GenerationType;
import javax.persistence.Id;
import javax.persistence.Table;
import com.myoa.engineering.crawl.shopping.domain.entity.Auditable;
import lombok.Getter;
import lombok.NoArgsConstructor;
import jakarta.persistence.*;
import java.time.Instant;
@Getter
@NoArgsConstructor
@Entity
@Table(name = "subscribed_user")
public class SubscribedUser extends Auditable{
public class SubscribedUser extends Auditable {
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)

29
shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v2/AppUser.java Normal file
View File

@ -0,0 +1,29 @@
package com.myoa.engineering.crawl.shopping.domain.entity.v2;
import com.myoa.engineering.crawl.shopping.domain.entity.Auditable;
import jakarta.persistence.*;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Getter;
import lombok.NoArgsConstructor;
@Getter
@Builder
@NoArgsConstructor
@AllArgsConstructor
@Entity
@Table
public class AppUser extends Auditable {
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
private Long id;
@Column
private String name;
@Column
private String slackId;
@Column
private Boolean enabled;
}

63
shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v2/Article.java Normal file
View File

@ -0,0 +1,63 @@
package com.myoa.engineering.crawl.shopping.domain.entity.v2;
import com.myoa.engineering.crawl.shopping.domain.entity.Auditable;
import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget;
import jakarta.persistence.*;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Getter;
import lombok.NoArgsConstructor;
import java.time.ZonedDateTime;
@Getter
@Builder
@NoArgsConstructor
@AllArgsConstructor
@Entity
@Table
public class Article extends Auditable {
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
private Long id;
@Column
private Long articleId;
@Column
@Enumerated(EnumType.STRING)
private CrawlTarget crawlTarget;
@Column
private String boardName;
@Column
private String articleUrl;
@Column
private String title;
@Column
private Integer hit;
@Column
private Integer recommended;
@Column
private ZonedDateTime registeredAt;
public Article update(Article article) {
this.boardName = article.getBoardName();
this.articleUrl = article.getArticleUrl();
this.title = article.getTitle();
this.hit = article.getHit();
this.recommended = article.getRecommended();
return this;
}
public Article updateCrawlTarget(CrawlTarget crawlTarget) {
this.crawlTarget = crawlTarget;
return this;
}
}

33
shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v2/SubscribedKeyword.java Normal file
View File

@ -0,0 +1,33 @@
package com.myoa.engineering.crawl.shopping.domain.entity.v2;
import com.myoa.engineering.crawl.shopping.domain.entity.Auditable;
import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget;
import jakarta.persistence.*;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Getter;
import lombok.NoArgsConstructor;
@Getter
@Builder
@NoArgsConstructor
@AllArgsConstructor
@Entity
@Table
public class SubscribedKeyword extends Auditable {
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
private Long id;
@Column
private String keyword;
@Column
@Enumerated(EnumType.STRING)
private CrawlTarget crawlTarget;
@Column
private String userId;
}

24
shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/model/UserNotifyModel.java Normal file
View File

@ -0,0 +1,24 @@
package com.myoa.engineering.crawl.shopping.domain.model;
import com.myoa.engineering.crawl.shopping.domain.model.v2.ArticleModel;
import lombok.*;
import java.util.List;
@ToString
@Getter
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class UserNotifyModel {
private String userId;
private List<ArticleModel> articles;
public static UserNotifyModel of(String userId, List<ArticleModel> articles) {
return UserNotifyModel.builder()
.userId(userId)
.articles(articles)
.build();
}
}

26
shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/model/v2/AppUserModel.java Normal file
View File

@ -0,0 +1,26 @@
package com.myoa.engineering.crawl.shopping.domain.model.v2;
import com.myoa.engineering.crawl.shopping.domain.entity.v2.AppUser;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Getter;
@Getter
@Builder
@AllArgsConstructor
public class AppUserModel {
private Long id;
private String name;
private String slackId;
private Boolean enabled;
public static AppUserModel from(AppUser entity) {
return AppUserModel.builder()
.id(entity.getId())
.name(entity.getName())
.slackId(entity.getSlackId())
.enabled(entity.getEnabled())
.build();
}
}

25
shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/model/v2/ArticleModel.java Normal file
View File

@ -0,0 +1,25 @@
package com.myoa.engineering.crawl.shopping.domain.model.v2;
import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget;
import lombok.*;
import java.time.ZonedDateTime;
@ToString
@Getter
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class ArticleModel {
private Long id;
private Long articleId;
private CrawlTarget crawlTarget;
private String boardName;
private String articleUrl;
private String title;
private Integer hit;
private Integer recommended;
private ZonedDateTime registeredAt;
}

27
shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/model/v2/SubscribedKeywordAggregatedModel.java Normal file
View File

@ -0,0 +1,27 @@
package com.myoa.engineering.crawl.shopping.domain.model.v2;
import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget;
import com.myoa.engineering.crawl.shopping.util.AhoCorasickUtils;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Getter;
import org.ahocorasick.trie.Trie;
import java.util.List;
@Getter
@Builder
@AllArgsConstructor
public class SubscribedKeywordAggregatedModel {
private final Trie ahoCorasickTrie;
private final String userId;
private final CrawlTarget crawlTarget;
public static SubscribedKeywordAggregatedModel of(String userId, CrawlTarget crawlTarget, List<String> keywords) {
return SubscribedKeywordAggregatedModel.builder()
.userId(userId)
.crawlTarget(crawlTarget)
.ahoCorasickTrie(AhoCorasickUtils.generateTrie(keywords))
.build();
}
}

9
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/FeedParsedResult.java → shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/FeedParsedResult.java
View File

@ -1,12 +1,13 @@
package com.myoa.engineering.crawl.ppomppu.processor.dto;
package com.myoa.engineering.crawl.shopping.dto;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import java.io.Serializable;
import java.time.Instant;
import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName;
import lombok.Builder;
import lombok.Getter;
import lombok.NoArgsConstructor;
import java.io.Serializable;
import java.time.Instant;
/**
* FeedParsedResult
*

0
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/PpomppuArticleParseDTO.java → shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/PpomppuArticleParseDTO.java
View File

0
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/PpomppuArticleTransformer.java → shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/PpomppuArticleTransformer.java
View File

2
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/constant/WebClientPropertiesUnitName.java → shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/constant/WebClientPropertiesUnitName.java
View File

@ -1,4 +1,4 @@
package com.myoa.engineering.crawl.ppomppu.processor.dto.constant;
package com.myoa.engineering.crawl.shopping.dto.constant;
import lombok.AllArgsConstructor;
import lombok.Getter;

2
sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/dto/MessageDTO.java → shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/slack/MessageDTO.java
View File

@ -1,4 +1,4 @@
package com.myoa.engineering.crawl.ppomppu.sender.dto;
package com.myoa.engineering.crawl.shopping.dto.slack;
import java.io.Serializable;

0
sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/dto/SlackBaseMessageBlock.java → shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/slack/SlackBaseMessageBlock.java
View File

0
sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/dto/SlackImageMessageBlock.java → shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/slack/SlackImageMessageBlock.java
View File

0
sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/dto/SlackMessageBlock.java → shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/slack/SlackMessageBlock.java
View File

0
sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/dto/SlackMessageBlockType.java → shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/slack/SlackMessageBlockType.java
View File

2
sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/dto/SlackMessageDTO.java → shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/slack/SlackMessageDTO.java
View File

@ -1,4 +1,4 @@
package com.myoa.engineering.crawl.ppomppu.sender.dto;
package com.myoa.engineering.crawl.shopping.dto.slack;
import java.util.List;

0
sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/dto/SlackSectionMessageBlock.java → shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/slack/SlackSectionMessageBlock.java
View File

13
shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/event/ArticleUpsertEvent.java Normal file
View File

@ -0,0 +1,13 @@
package com.myoa.engineering.crawl.shopping.event;
import com.myoa.engineering.crawl.shopping.domain.model.v2.ArticleModel;
import org.springframework.context.ApplicationEvent;
import java.util.List;
public class ArticleUpsertEvent extends ApplicationEvent {
public ArticleUpsertEvent(List<ArticleModel> source) {
super(source);
}
}

15
shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/event/ArticleUpsertEventPayload.java Normal file
View File

@ -0,0 +1,15 @@
package com.myoa.engineering.crawl.shopping.event;
import com.myoa.engineering.crawl.shopping.domain.model.v2.ArticleModel;
import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget;
import lombok.AllArgsConstructor;
import lombok.Getter;
import java.util.List;
@Getter
@AllArgsConstructor
public class ArticleUpsertEventPayload {
private final List<ArticleModel> articles;
private final CrawlTarget crawlTarget;
}

86
shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/event/handler/ArticleUpsertEventListener.java Normal file
View File

@ -0,0 +1,86 @@
package com.myoa.engineering.crawl.shopping.event.handler;
import com.myoa.engineering.crawl.shopping.domain.model.UserNotifyModel;
import com.myoa.engineering.crawl.shopping.domain.model.v2.AppUserModel;
import com.myoa.engineering.crawl.shopping.domain.model.v2.ArticleModel;
import com.myoa.engineering.crawl.shopping.domain.model.v2.SubscribedKeywordAggregatedModel;
import com.myoa.engineering.crawl.shopping.event.ArticleUpsertEvent;
import com.myoa.engineering.crawl.shopping.service.AppUserQueryService;
import com.myoa.engineering.crawl.shopping.service.SubscribedKeywordCacheService;
import com.myoa.engineering.crawl.shopping.service.UserNotifyService;
import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget;
import org.springframework.context.event.EventListener;
import org.springframework.stereotype.Component;
import java.util.List;
import java.util.Map;
import java.util.function.Function;
import java.util.stream.Collectors;
@Component
public class ArticleUpsertEventListener {
private final SubscribedKeywordCacheService subscribedKeywordCacheService;
private final AppUserQueryService appUserQueryService;
private final UserNotifyService userNotifyService;
public ArticleUpsertEventListener(SubscribedKeywordCacheService subscribedKeywordCacheService,
AppUserQueryService appUserQueryService, UserNotifyService userNotifyService) {
this.subscribedKeywordCacheService = subscribedKeywordCacheService;
this.appUserQueryService = appUserQueryService;
this.userNotifyService = userNotifyService;
}
@EventListener
public void handleArticleUpsertEvent(ArticleUpsertEvent event) {
Map<CrawlTarget, List<ArticleModel>> articleMap =
((List<ArticleModel>) event.getSource()).stream()
.collect(Collectors.groupingBy(ArticleModel::getCrawlTarget));
List<AppUserModel> appUsers = appUserQueryService.findAll();
appUsers.stream()
.filter(AppUserModel::getEnabled)
.map(user -> {
List<ArticleModel> filteredArticles = handleAhoCorasick(articleMap)
.apply(subscribedKeywordCacheService.getSubscribedKeywordsCached(user.getName()));
return UserNotifyModel.of(user.getName(), filteredArticles);
})
.forEach(this::notifyMessage);
}
private Function<Map<CrawlTarget, SubscribedKeywordAggregatedModel>, List<ArticleModel>> handleAhoCorasick(
Map<CrawlTarget, List<ArticleModel>> articleMap) {
return userTrieModel -> {
return userTrieModel
.entrySet()
.stream().filter(e -> articleMap.containsKey(e.getKey()))
.map((entry) -> filterAhocorasick(articleMap.get(entry.getKey()), entry.getValue()))
.flatMap(List::stream)
.toList();
// return UserNotifyModel.of(userTrieModel.values().stream().findFirst().get().getUserId(),
// filteredArticle);
};
}
private List<ArticleModel> filterAhocorasick(List<ArticleModel> articles,
SubscribedKeywordAggregatedModel trieModel) {
return articles.stream()
.filter(article -> !trieModel.getAhoCorasickTrie()
.parseText(article.getTitle())
.isEmpty())
.toList();
//ArticleUpsertEventListener::printArticle
}
private void notifyMessage(UserNotifyModel article) {
System.out.println("article = " + article);
if (article.getArticles().isEmpty()){
return;
}
userNotifyService.notify("안녕 " + article.getUserId() + "\n" + article.getArticles());
}
}

0
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/client/MessageSenderAPIClient.java → shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/MessageSenderAPIClient.java
View File

12
shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/fmkorea/FmkoreaBoardClient.java Normal file
View File

@ -0,0 +1,12 @@
package com.myoa.engineering.crawl.shopping.infra.client.fmkorea;
import org.springframework.cloud.openfeign.FeignClient;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
@FeignClient(value = "fmkorea-board-client", url = "https://fmkorea.com")
public interface FmkoreaBoardClient {
@GetMapping("{boardLink}")
String getBoardHtml(@PathVariable("boardLink") String boardLink);
}

19
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/client/PpomppuBoardFeedRetriever.java → shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/ppomppu/PpomppuBoardClient.java
View File

@ -1,16 +1,7 @@
package com.myoa.engineering.crawl.ppomppu.processor.infrastructure.client;
package com.myoa.engineering.crawl.shopping.infra.client.ppomppu;
import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebClientFilterFactory;
import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebFluxExchangeStragiesFactory;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import lombok.extern.slf4j.Slf4j;
import org.springframework.core.ParameterizedTypeReference;
import org.springframework.stereotype.Component;
import org.springframework.web.reactive.function.client.WebClient;
import org.springframework.web.reactive.function.client.WebClientRequestException;
import reactor.core.publisher.Mono;
import reactor.core.scheduler.Schedulers;
/**
* PpomppuBoardFeedRetriever
@ -20,11 +11,11 @@ import reactor.core.scheduler.Schedulers;
*/
@Slf4j
@Component
public class PpomppuBoardFeedRetriever {
public class PpomppuBoardClient {
/*
private final WebClient webClient;
public PpomppuBoardFeedRetriever(WebClient.Builder webClientBuilder) {
public PpomppuBoardClient(WebClient.Builder webClientBuilder) {
this.webClient = webClientBuilder.baseUrl(PpomppuBoardName.PPOMPPU_URL)
.exchangeStrategies(WebFluxExchangeStragiesFactory.ofTextHtml())
.filter(WebClientFilterFactory.logRequest())
@ -43,5 +34,5 @@ public class PpomppuBoardFeedRetriever {
});
// .doOnNext(e -> log.info("[getHtml] {}", e));
}
*/
}

17
shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/ppomppu/PpomppuBoardClientV2.java Normal file
View File

@ -0,0 +1,17 @@
package com.myoa.engineering.crawl.shopping.infra.client.ppomppu;
import org.springframework.cloud.openfeign.FeignClient;
import org.springframework.cloud.openfeign.SpringQueryMap;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestParam;
import java.util.Map;
@FeignClient(value = "ppomppu-board-client", url = "https://www.ppomppu.co.kr")
public interface PpomppuBoardClientV2 {
@GetMapping("{boardLink}")
String getBoardHtml(@PathVariable("boardLink") String boardLink,
@SpringQueryMap Map<String, String> params);
}

0
sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/infrastructure/client/MongeShoppingBotSlackMessageSender.java → shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/slack/MongeShoppingBotSlackMessageSender.java
View File

17
shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/slack/SlackAPIClient.java Normal file
View File

@ -0,0 +1,17 @@
package com.myoa.engineering.crawl.shopping.infra.client.slack;
import com.myoa.engineering.crawl.shopping.configuration.FeignDefaultConfig;
import com.myoa.engineering.crawl.shopping.dto.slack.SlackMessageDTO;
import org.springframework.cloud.openfeign.FeignClient;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestHeader;
@FeignClient(value = "slack-api-client", url = "https://slack.com/api",
configuration = FeignDefaultConfig.class)
public interface SlackAPIClient {
@PostMapping("/chat.postMessage")
String sendMessage(@RequestBody SlackMessageDTO message,
@RequestHeader("Authorization") String token);
}

18
sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/infrastructure/client/SlackMessageSender.java → shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/slack/SlackMessageSender.java
View File

@ -1,17 +1,6 @@
package com.myoa.engineering.crawl.ppomppu.sender.infrastructure.client;
import org.springframework.http.HttpHeaders;
import org.springframework.http.MediaType;
import org.springframework.web.reactive.function.client.WebClient;
import org.springframework.web.reactive.function.client.WebClientRequestException;
import com.myoa.engineering.crawl.ppomppu.sender.dto.SlackMessageDTO;
import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebClientFilterFactory;
import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebFluxExchangeStragiesFactory;
package com.myoa.engineering.crawl.shopping.infra.client.slack;
import lombok.extern.slf4j.Slf4j;
import reactor.core.publisher.Mono;
import reactor.core.scheduler.Schedulers;
/**
* SlackMessageSender
@ -20,8 +9,9 @@ import reactor.core.scheduler.Schedulers;
* @since 2021-09-08
*/
@Slf4j
public class SlackMessageSender implements MessageSender<SlackMessageDTO> {
public class SlackMessageSender { /* implements MessageSender<SlackMessageDTO> {*/
/*
private static final String SLACK_API_URL = "https://slack.com/api";
private final WebClient webClient;
@ -52,4 +42,6 @@ public class SlackMessageSender implements MessageSender<SlackMessageDTO> {
.doOnNext(e -> log.info("[sendMessage] {}", e));
}
*/
}

4
shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/repository/BaseScanRepository.java Normal file
View File

@ -0,0 +1,4 @@
package com.myoa.engineering.crawl.shopping.infra.repository;
public interface BaseScanRepository {
}

4
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/repository/PpomppuArticleRepository.java → shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/repository/v1/PpomppuArticleRepository.java
View File

@ -1,6 +1,6 @@
package com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository;
package com.myoa.engineering.crawl.shopping.infra.repository.v1;
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
import com.myoa.engineering.crawl.shopping.domain.entity.v1.PpomppuArticle;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.stereotype.Repository;

9
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/repository/PpomppuBoardFeedStatusRepository.java → shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/repository/v1/PpomppuBoardFeedStatusRepository.java
View File

@ -1,11 +1,12 @@
package com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository;
package com.myoa.engineering.crawl.shopping.infra.repository.v1;
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuBoardFeedStatus;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import java.util.Optional;
import com.myoa.engineering.crawl.shopping.domain.entity.v1.PpomppuBoardFeedStatus;
import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.stereotype.Repository;
import java.util.Optional;
@Repository
public interface PpomppuBoardFeedStatusRepository extends JpaRepository<PpomppuBoardFeedStatus, Long> {

9
shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/repository/v2/AppUserRepository.java Normal file
View File

@ -0,0 +1,9 @@
package com.myoa.engineering.crawl.shopping.infra.repository.v2;
import com.myoa.engineering.crawl.shopping.domain.entity.v2.AppUser;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.stereotype.Repository;
@Repository
public interface AppUserRepository extends JpaRepository<AppUser, Long> {
}

16
shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/repository/v2/ArticleRepository.java Normal file
View File

@ -0,0 +1,16 @@
package com.myoa.engineering.crawl.shopping.infra.repository.v2;
import com.myoa.engineering.crawl.shopping.domain.entity.v2.Article;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.stereotype.Repository;
import java.util.Collection;
import java.util.List;
import java.util.Optional;
@Repository
public interface ArticleRepository extends JpaRepository<Article, Long> {
Optional<Article> findByArticleId(Long articleId);
List<Article> findByArticleIdIn(Collection<Long> articleId);
}

Some files were not shown because too many files have changed in this diff Show More