diff --git a/.gitignore b/.gitignore index 670f49a..3f1f94d 100644 --- a/.gitignore +++ b/.gitignore @@ -36,4 +36,7 @@ out/ ### VS Code ### .vscode/ -temppassword.yml \ No newline at end of file +temppassword.yml +data.sql +**/src/main/resources/slack +**/src/main/resources/datasource \ No newline at end of file diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 05679dc..6677206 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,6 @@ +#Sun Apr 28 23:47:38 KST 2024 distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.1.1-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.5-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/gradlew b/gradlew old mode 100644 new mode 100755 diff --git a/processor/build.gradle b/processor/build.gradle deleted file mode 100644 index eb68c4c..0000000 --- a/processor/build.gradle +++ /dev/null @@ -1,30 +0,0 @@ - -dependencies { - developmentOnly 'org.springframework.boot:spring-boot-devtools' - runtimeOnly 'com.h2database:h2' - runtimeOnly 'mysql:mysql-connector-java' - compileOnly 'org.projectlombok:lombok' - - implementation project(':support') - // https://projectreactor.io/docs/core/release/reference/#debug-activate - implementation 'org.springframework.boot:spring-boot-starter-webflux' - implementation 'org.springframework.boot:spring-boot-starter-data-jpa' - implementation 'org.springframework.boot:spring-boot-starter-jdbc' - implementation 'org.springframework.cloud:spring-cloud-starter-config' - implementation 'org.springframework.boot:spring-boot-starter-actuator' - implementation 'com.rometools:rome:1.16.0' - implementation 'org.jsoup:jsoup:1.14.2' - implementation 'com.h2database:h2:1.4.200' - - annotationProcessor 'org.springframework.boot:spring-boot-configuration-processor' - annotationProcessor 'org.projectlombok:lombok' - - testImplementation 'org.springframework.boot:spring-boot-starter-test' -} - -test { - useJUnitPlatform() - testLogging { - events "passed", "skipped", "failed" - } -} \ No newline at end of file diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/ProcessorApplication.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/ProcessorApplication.java deleted file mode 100644 index eb9f40c..0000000 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/ProcessorApplication.java +++ /dev/null @@ -1,23 +0,0 @@ -package com.myoa.engineering.crawl.ppomppu.processor; - -import org.springframework.boot.SpringApplication; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.context.annotation.Import; - -import com.myoa.engineering.crawl.ppomppu.support.webclient.PpomppuNotifierWebClientConfiguration; - -/** - * ProcessorApplication - * @author Shin Woo-jin (woo-jin.shin@linecorp.com) - * @since 2021-08-20 - * - */ -@Import({ PpomppuNotifierWebClientConfiguration.class }) -@SpringBootApplication -public class ProcessorApplication { - - public static void main(String[] args) { - SpringApplication.run(ProcessorApplication.class, args); - } - -} diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/SubscribedBoard.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/SubscribedBoard.java deleted file mode 100644 index 2eb7565..0000000 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/SubscribedBoard.java +++ /dev/null @@ -1,32 +0,0 @@ -package com.myoa.engineering.crawl.ppomppu.processor.domain; - -import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName; -import javax.persistence.Column; -import javax.persistence.Entity; -import javax.persistence.EnumType; -import javax.persistence.Enumerated; -import javax.persistence.GeneratedValue; -import javax.persistence.GenerationType; -import javax.persistence.Id; -import javax.persistence.Table; -import lombok.Getter; -import lombok.NoArgsConstructor; - -@Getter -@NoArgsConstructor -@Entity -@Table(name = "subscribed_board") -public class SubscribedBoard extends Auditable { - - @Id - @GeneratedValue(strategy = GenerationType.IDENTITY) - private Long id; - - @Column - private Long userId; - - @Column - @Enumerated(EnumType.STRING) - private PpomppuBoardName boardName; - -} diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/repository/BaseScanRepository.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/repository/BaseScanRepository.java deleted file mode 100644 index ebbfedc..0000000 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/repository/BaseScanRepository.java +++ /dev/null @@ -1,4 +0,0 @@ -package com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository; - -public interface BaseScanRepository { -} diff --git a/processor/src/main/resources/application-local.yml b/processor/src/main/resources/application-local.yml deleted file mode 100644 index 42ba0a4..0000000 --- a/processor/src/main/resources/application-local.yml +++ /dev/null @@ -1,11 +0,0 @@ -spring: - config: - activate: - on-profile: local - import: - - "configserver:http://localhost:20085" - -server: - port: 20081 - - # import: optional:configserver:http://localhost:11080 # can be start up even config server was not found. \ No newline at end of file diff --git a/receiver/build.gradle b/receiver/build.gradle deleted file mode 100644 index d256b7c..0000000 --- a/receiver/build.gradle +++ /dev/null @@ -1,26 +0,0 @@ -dependencies { - developmentOnly 'org.springframework.boot:spring-boot-devtools' - runtimeOnly 'com.h2database:h2' - runtimeOnly 'mysql:mysql-connector-java' - compileOnly 'org.projectlombok:lombok' - - implementation project(':support') - // https://projectreactor.io/docs/core/release/reference/#debug-activate - implementation 'org.springframework.boot:spring-boot-starter-webflux' - implementation 'org.springframework.boot:spring-boot-configuration-processor' - implementation 'org.springframework.cloud:spring-cloud-starter-config' - implementation 'org.springframework.boot:spring-boot-starter-actuator' - implementation 'org.telegram:telegrambots:5.3.0' - - annotationProcessor 'org.springframework.boot:spring-boot-configuration-processor' - annotationProcessor 'org.projectlombok:lombok' - - testImplementation 'org.springframework.boot:spring-boot-starter-test' -} - -test { - useJUnitPlatform() - testLogging { - events "passed", "skipped", "failed" - } -} \ No newline at end of file diff --git a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/ReceiverApplication.java b/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/ReceiverApplication.java deleted file mode 100644 index 339a01d..0000000 --- a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/ReceiverApplication.java +++ /dev/null @@ -1,25 +0,0 @@ -package com.myoa.engineering.crawl.ppomppu.receiver; - -import org.springframework.boot.SpringApplication; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.context.annotation.Import; - -import com.myoa.engineering.crawl.ppomppu.receiver.configuration.properties.TelegramBotProperties; -import com.myoa.engineering.crawl.ppomppu.support.webclient.PpomppuNotifierWebClientConfiguration; - -/** - * ReceiverApplication - * @author Shin Woo-jin (woo-jin.shin@linecorp.com) - * @since 2021-08-20 - * - */ -@Import({ PpomppuNotifierWebClientConfiguration.class}) -@SpringBootApplication -// @EnableConfigurationProperties({ TelegramBotProperties.class }) -public class ReceiverApplication { - - public static void main(String[] args) { - SpringApplication.run(ReceiverApplication.class, args); - } -} diff --git a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/configuration/TelegramBotConfiguration.java b/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/configuration/TelegramBotConfiguration.java deleted file mode 100644 index dd120dc..0000000 --- a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/configuration/TelegramBotConfiguration.java +++ /dev/null @@ -1,39 +0,0 @@ -package com.myoa.engineering.crawl.ppomppu.receiver.configuration; - -import java.util.List; - -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.telegram.telegrambots.meta.TelegramBotsApi; -import org.telegram.telegrambots.meta.exceptions.TelegramApiException; -import org.telegram.telegrambots.updatesreceivers.DefaultBotSession; - -import com.myoa.engineering.crawl.ppomppu.receiver.configuration.properties.TelegramBotProperties; -import com.myoa.engineering.crawl.ppomppu.receiver.configuration.properties.TelegramBotProperties.TelegramBotPropertiesUnit; -import com.myoa.engineering.crawl.ppomppu.receiver.dispatch.MessageDispatcher; -import com.myoa.engineering.crawl.ppomppu.receiver.handler.message.MessageHandler; - -/** - * TelegramBotConfiguration - * @author Shin Woo-jin (woo-jin.shin@linecorp.com) - * @since 2021-08-21 - * - */ -@Configuration -public class TelegramBotConfiguration { - - private static final String BOT_PROPERTIES_UNIT_NAME = "ppomppu_notify_bot"; - @Bean - public TelegramBotsApi telegramBotsApi(MessageDispatcher messageDispatcher) throws TelegramApiException { - TelegramBotsApi api = new TelegramBotsApi(DefaultBotSession.class); - api.registerBot(messageDispatcher); - return api; - } - - @Bean - public MessageDispatcher messageDispatcher(List messageHandlers, - TelegramBotProperties telegramBotProperties) { - TelegramBotPropertiesUnit propertiesUnit = telegramBotProperties.find(BOT_PROPERTIES_UNIT_NAME); - return new MessageDispatcher(messageHandlers, propertiesUnit.getName(), propertiesUnit.getToken()); - } -} diff --git a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/configuration/properties/TelegramBotProperties.java b/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/configuration/properties/TelegramBotProperties.java deleted file mode 100644 index e438889..0000000 --- a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/configuration/properties/TelegramBotProperties.java +++ /dev/null @@ -1,44 +0,0 @@ -package com.myoa.engineering.crawl.ppomppu.receiver.configuration.properties; - -import java.util.ArrayList; -import java.util.List; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.boot.context.properties.ConstructorBinding; -import org.springframework.stereotype.Component; - -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.Setter; - -/** - * TelegramBotProperties - * @author Shin Woo-jin (woo-jin.shin@linecorp.com) - * @since 2021-09-05 - * - */ -@Component -@Setter -@Getter -@ConfigurationProperties(prefix = "infra.telegram.bot") -public class TelegramBotProperties { - - private List units = new ArrayList<>(); - - @Data - public static class TelegramBotPropertiesUnit { - private String unitName; - private String name; - private String token; - } - - public TelegramBotPropertiesUnit find(String unitName) { - return units.stream() - .filter(e -> e.getUnitName().equals(unitName)) - .findFirst() - .orElseThrow( - () -> new IllegalArgumentException(this.getClass().getName() + ": unitName Not found. " + unitName)); - } -} diff --git a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/controller/v1/EventAPIController.java b/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/controller/v1/EventAPIController.java deleted file mode 100644 index d67089c..0000000 --- a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/controller/v1/EventAPIController.java +++ /dev/null @@ -1,33 +0,0 @@ -package com.myoa.engineering.crawl.ppomppu.receiver.controller.v1; - -import org.springframework.web.bind.annotation.PathVariable; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; - -import com.myoa.engineering.crawl.ppomppu.receiver.service.ProcessorAPIService; -import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName; - -import reactor.core.publisher.Mono; - -/** - * EventAPIController - * @author Shin Woo-jin (woo-jin.shin@linecorp.com) - * @since 2021-09-05 - * - */ -@RestController -@RequestMapping("/api/v1/") -public class EventAPIController { - - private final ProcessorAPIService processorAPIService; - - public EventAPIController(ProcessorAPIService processorAPIService) { - this.processorAPIService = processorAPIService; - } - - @PostMapping("/exploit/parsePpomppuRSS/{boardName}") - public Mono parsePpomppuRSS(@PathVariable("boardName") PpomppuBoardName boardName) { - return processorAPIService.emitParseEvent(boardName); - } -} diff --git a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/dispatch/MessageDispatcher.java b/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/dispatch/MessageDispatcher.java deleted file mode 100644 index 22ea270..0000000 --- a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/dispatch/MessageDispatcher.java +++ /dev/null @@ -1,50 +0,0 @@ -package com.myoa.engineering.crawl.ppomppu.receiver.dispatch; - -import java.util.List; - -import org.telegram.telegrambots.bots.TelegramLongPollingBot; -import org.telegram.telegrambots.meta.api.objects.Message; -import org.telegram.telegrambots.meta.api.objects.Update; - -import com.myoa.engineering.crawl.ppomppu.receiver.handler.message.MessageHandler; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class MessageDispatcher extends TelegramLongPollingBot { - - private final List messageHandlers; - private final String botName; - private final String botToken; - - public MessageDispatcher(List messageHandlers, String botName, String botToken) { - this.messageHandlers = messageHandlers; - this.botName = botName; - this.botToken = botToken; - } - - @Override - public String getBotToken() { - return botToken; - } - - @Override - public void onUpdateReceived(Update update) { - - Message message = update.getMessage(); - MessageHandler handler = getMessageHandler(message); - handler.handle(message); - } - - private MessageHandler getMessageHandler(Message message) { - return messageHandlers.stream() - .filter(e -> e.isApplicable(message)) - .findFirst() - .orElseThrow(() -> new IllegalArgumentException("Can not found applicable handler")); - } - - @Override - public String getBotUsername() { - return botName; - } -} diff --git a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/dto/FeedParsedResult.java b/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/dto/FeedParsedResult.java deleted file mode 100644 index 20d0c5e..0000000 --- a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/dto/FeedParsedResult.java +++ /dev/null @@ -1,25 +0,0 @@ -package com.myoa.engineering.crawl.ppomppu.receiver.dto; - -import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName; -import java.io.Serializable; -import java.time.Instant; -import lombok.Getter; -import lombok.NoArgsConstructor; - -/** - * FeedParsedResult - * - * @author Shin Woo-jin (woozu.shin@kakaoent.com) - * @since 2021-09-08 - */ -@Getter -@NoArgsConstructor -public class FeedParsedResult implements Serializable { - - private static final long serialVersionUID = -3771310078623481348L; - - private PpomppuBoardName boardName; - private Instant requestedAt; - private Instant processedAt; - -} diff --git a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/dto/constant/WebClientPropertiesUnitName.java b/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/dto/constant/WebClientPropertiesUnitName.java deleted file mode 100644 index bdab14d..0000000 --- a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/dto/constant/WebClientPropertiesUnitName.java +++ /dev/null @@ -1,19 +0,0 @@ -package com.myoa.engineering.crawl.ppomppu.receiver.dto.constant; - -import lombok.AllArgsConstructor; -import lombok.Getter; - -/** - * WebClientPropertiesUnitName - * @author Shin Woo-jin (woo-jin.shin@linecorp.com) - * @since 2021-11-18 - * - */ -@Getter -@AllArgsConstructor -public enum WebClientPropertiesUnitName { - PPOMPPU_NOTIFIER_PROCESSOR_API("ppn-processor-api"), - ; - - private String unitName; -} diff --git a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/HelloWorldMessageHandler.java b/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/HelloWorldMessageHandler.java deleted file mode 100644 index ee5f83e..0000000 --- a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/HelloWorldMessageHandler.java +++ /dev/null @@ -1,21 +0,0 @@ -package com.myoa.engineering.crawl.ppomppu.receiver.handler.message; - -import com.myoa.engineering.crawl.ppomppu.support.util.ObjectUtil; -import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Component; -import org.telegram.telegrambots.meta.api.objects.Message; - -@Slf4j -@Component -public class HelloWorldMessageHandler implements MessageHandler { - - @Override - public boolean isApplicable(Message message) { - return ObjectUtil.isEmpty(message); - } - - @Override - public void handle(Message message) { - // skip empty event message. - } -} diff --git a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/ImageMessageHandler.java b/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/ImageMessageHandler.java deleted file mode 100644 index b8cc1bc..0000000 --- a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/ImageMessageHandler.java +++ /dev/null @@ -1,12 +0,0 @@ -package com.myoa.engineering.crawl.ppomppu.receiver.handler.message; - -/** - * ImageHandler - * @author Shin Woo-jin (woo-jin.shin@linecorp.com) - * @since 2021-08-21 - * - */ -public interface ImageMessageHandler extends MessageHandler { - - -} diff --git a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/MessageHandler.java b/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/MessageHandler.java deleted file mode 100644 index 01e44f0..0000000 --- a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/MessageHandler.java +++ /dev/null @@ -1,17 +0,0 @@ -package com.myoa.engineering.crawl.ppomppu.receiver.handler.message; - -import org.telegram.telegrambots.meta.api.objects.Message; - -/** - * MessageHandler - * @author Shin Woo-jin (woo-jin.shin@linecorp.com) - * @since 2021-08-21 - * - */ -public interface MessageHandler { - - boolean isApplicable(Message message); - - void handle(Message message); - -} diff --git a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/TextMessageHandler.java b/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/TextMessageHandler.java deleted file mode 100644 index 07c185e..0000000 --- a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/TextMessageHandler.java +++ /dev/null @@ -1,18 +0,0 @@ -package com.myoa.engineering.crawl.ppomppu.receiver.handler.message; - -import com.myoa.engineering.crawl.ppomppu.support.util.ObjectUtil; -import org.telegram.telegrambots.meta.api.objects.Message; - -/** - * TextMessageHandler - * - * @author Shin Woo-jin (woo-jin.shin@linecorp.com) - * @since 2021-08-21 - */ -public interface TextMessageHandler extends MessageHandler { - - @Override - default boolean isApplicable(Message message) { - return ObjectUtil.isNotEmpty(message) && message.isUserMessage() && message.hasText(); - } -} diff --git a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/text/CommandHandler.java b/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/text/CommandHandler.java deleted file mode 100644 index 3c22204..0000000 --- a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/text/CommandHandler.java +++ /dev/null @@ -1,48 +0,0 @@ -package com.myoa.engineering.crawl.ppomppu.receiver.handler.message.text; - -import java.util.List; - -import org.springframework.stereotype.Component; -import org.telegram.telegrambots.meta.api.objects.Message; - -import com.myoa.engineering.crawl.ppomppu.receiver.handler.message.TextMessageHandler; - -import lombok.extern.slf4j.Slf4j; - -/** - * CommandHandler - * @author Shin Woo-jin (woo-jin.shin@linecorp.com) - * @since 2021-08-21 - * - */ -@Slf4j -@Component -public class CommandHandler implements TextMessageHandler { - - private final List processors; - - public CommandHandler(List processors) { - this.processors = processors; - } - - @Override - public boolean isApplicable(Message message) { - return TextMessageHandler.super.isApplicable(message) - && message.isCommand(); // && message.getText().startsWith("/"); - } - - @Override - public void handle(Message message) { - log.info("CommandHandler : {}", message.getText()); - TextCommandCode commandCode = TextCommandCode.find(message.getText()); - TextCommandProcessor applicableProcessor = getApplicableProcessor(commandCode); - applicableProcessor.process(message); - } - - private TextCommandProcessor getApplicableProcessor(TextCommandCode commandCode) { - return processors.stream() - .filter(e -> e.isApplicable(commandCode)) - .findFirst() - .orElseThrow(() -> new IllegalArgumentException("Can not found")); - } -} diff --git a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/text/CommonTextHandler.java b/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/text/CommonTextHandler.java deleted file mode 100644 index 761e14f..0000000 --- a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/text/CommonTextHandler.java +++ /dev/null @@ -1,30 +0,0 @@ -package com.myoa.engineering.crawl.ppomppu.receiver.handler.message.text; - -import org.springframework.stereotype.Component; -import org.telegram.telegrambots.meta.api.objects.Message; - -import com.myoa.engineering.crawl.ppomppu.receiver.handler.message.TextMessageHandler; - -import lombok.extern.slf4j.Slf4j; - -/** - * NormalTextHandler - * @author Shin Woo-jin (woo-jin.shin@linecorp.com) - * @since 2021-08-21 - * - */ -@Slf4j -@Component -public class CommonTextHandler implements TextMessageHandler { - - @Override - public boolean isApplicable(Message message) { - return TextMessageHandler.super.isApplicable(message) && message.isCommand() == false; - } - - @Override - public void handle(Message message) { - log.info("CommonTextHandler : {}", message.getText()); - } - -} diff --git a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/text/EmptyTextCommandProcessor.java b/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/text/EmptyTextCommandProcessor.java deleted file mode 100644 index e0c4449..0000000 --- a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/text/EmptyTextCommandProcessor.java +++ /dev/null @@ -1,24 +0,0 @@ -package com.myoa.engineering.crawl.ppomppu.receiver.handler.message.text; - -import org.springframework.stereotype.Component; -import org.telegram.telegrambots.meta.api.objects.Message; - -/** - * EmptyTextCommandProcessor - * @author Shin Woo-jin (woo-jin.shin@linecorp.com) - * @since 2021-09-05 - * - */ -@Component -public class EmptyTextCommandProcessor implements TextCommandProcessor { - - @Override - public boolean isApplicable(TextCommandCode commandCode) { - return commandCode == TextCommandCode.EMPTY; - } - - @Override - public void process(Message message) { - - } -} diff --git a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/text/StartTextCommandProcessor.java b/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/text/StartTextCommandProcessor.java deleted file mode 100644 index 08c8315..0000000 --- a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/text/StartTextCommandProcessor.java +++ /dev/null @@ -1,27 +0,0 @@ -package com.myoa.engineering.crawl.ppomppu.receiver.handler.message.text; - -import org.springframework.stereotype.Component; -import org.telegram.telegrambots.meta.api.objects.Message; - -import lombok.extern.slf4j.Slf4j; - -/** - * StartCommandProcessor - * @author Shin Woo-jin (woo-jin.shin@linecorp.com) - * @since 2021-09-05 - * - */ -@Slf4j -@Component -public class StartTextCommandProcessor implements TextCommandProcessor { - - @Override - public boolean isApplicable(TextCommandCode commandCode) { - return TextCommandCode.START == commandCode; - } - - @Override - public void process(Message message) { - log.info("[process] user: {}, command: {}", message.getChatId(), message.getText()); - } -} diff --git a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/text/TextCommandCode.java b/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/text/TextCommandCode.java deleted file mode 100644 index 364d655..0000000 --- a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/text/TextCommandCode.java +++ /dev/null @@ -1,33 +0,0 @@ -package com.myoa.engineering.crawl.ppomppu.receiver.handler.message.text; - -import java.util.Arrays; - -import lombok.Getter; -import lombok.NoArgsConstructor; - -/** - * CommandTextCode - * @author Shin Woo-jin (woo-jin.shin@linecorp.com) - * @since 2021-09-05 - * - */ -@Getter -@NoArgsConstructor -public enum TextCommandCode { - EMPTY(null), - START("/start"), - ; - private String value; - - TextCommandCode(String value) { - this.value = value; - } - - public static TextCommandCode find(String value) { - return Arrays.stream(TextCommandCode.values()) - .filter(e -> e != EMPTY) - .filter(e -> value.startsWith(e.getValue())) - .findFirst() - .orElse(TextCommandCode.EMPTY); - } -} diff --git a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/text/TextCommandProcessor.java b/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/text/TextCommandProcessor.java deleted file mode 100644 index 3f8ff49..0000000 --- a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/text/TextCommandProcessor.java +++ /dev/null @@ -1,17 +0,0 @@ -package com.myoa.engineering.crawl.ppomppu.receiver.handler.message.text; - -import org.telegram.telegrambots.meta.api.objects.Message; - -/** - * TextCommandProcessor - * @author Shin Woo-jin (woo-jin.shin@linecorp.com) - * @since 2021-09-05 - * - */ -public interface TextCommandProcessor { - - boolean isApplicable(TextCommandCode commandCode); - - void process(Message message); - -} diff --git a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/infrastructure/client/ProcessorAPIWebClient.java b/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/infrastructure/client/ProcessorAPIWebClient.java deleted file mode 100644 index 2dbca2a..0000000 --- a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/infrastructure/client/ProcessorAPIWebClient.java +++ /dev/null @@ -1,44 +0,0 @@ -package com.myoa.engineering.crawl.ppomppu.receiver.infrastructure.client; - -import org.springframework.core.ParameterizedTypeReference; -import org.springframework.stereotype.Component; -import org.springframework.web.reactive.function.client.WebClient; -import org.springframework.web.reactive.function.client.WebClientRequestException; - -import com.myoa.engineering.crawl.ppomppu.receiver.dto.constant.WebClientPropertiesUnitName; -import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName; -import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebClientFilterFactory; -import com.myoa.engineering.crawl.ppomppu.support.webclient.properties.WebClientProperties; -import com.myoa.engineering.crawl.ppomppu.support.webclient.properties.WebClientProperties.WebClientPropertiesUnit; - -import lombok.extern.slf4j.Slf4j; -import reactor.core.publisher.Mono; -import reactor.core.scheduler.Schedulers; - -@Slf4j -@Component -public class ProcessorAPIWebClient { - - private final WebClient webClient; - - public ProcessorAPIWebClient(WebClientProperties webClientProperties) { - WebClientPropertiesUnit webClientPropertiesUnit = - webClientProperties.find(WebClientPropertiesUnitName.PPOMPPU_NOTIFIER_PROCESSOR_API.getUnitName()); - this.webClient = WebClient.builder() - .baseUrl(webClientPropertiesUnit.getBaseUrl()) - .filter(WebClientFilterFactory.logRequest()) - .filter(WebClientFilterFactory.logResponse()) - .build(); - } - - public Mono emitParseEvent(PpomppuBoardName boardName) { - return webClient.post() - .uri("/api/v1/crawl/boards/{boardName}", boardName) - .exchangeToMono(e -> e.bodyToMono(new ParameterizedTypeReference() {})) - .publishOn(Schedulers.boundedElastic()) - .onErrorResume(WebClientRequestException.class, t -> { - log.info("Exception occured, ignoring. : {}", t.getClass().getSimpleName()); - return Mono.empty(); - }); - } -} diff --git a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/service/ProcessorAPIService.java b/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/service/ProcessorAPIService.java deleted file mode 100644 index b793dbb..0000000 --- a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/service/ProcessorAPIService.java +++ /dev/null @@ -1,30 +0,0 @@ -package com.myoa.engineering.crawl.ppomppu.receiver.service; - -import org.springframework.stereotype.Service; - -import com.myoa.engineering.crawl.ppomppu.receiver.infrastructure.client.ProcessorAPIWebClient; -import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName; - -import lombok.extern.slf4j.Slf4j; -import reactor.core.publisher.Mono; - -/** - * ProcessorAPIService - * @author Shin Woo-jin (woo-jin.shin@linecorp.com) - * @since 2021-09-05 - * - */ -@Slf4j -@Service -public class ProcessorAPIService { - - private final ProcessorAPIWebClient processorAPIWebClient; - - public ProcessorAPIService(ProcessorAPIWebClient processorAPIWebClient) { - this.processorAPIWebClient = processorAPIWebClient; - } - - public Mono emitParseEvent(PpomppuBoardName boardName) { - return processorAPIWebClient.emitParseEvent(boardName); - } -} diff --git a/receiver/src/main/resources/application-development.yml b/receiver/src/main/resources/application-development.yml deleted file mode 100644 index d9b0001..0000000 --- a/receiver/src/main/resources/application-development.yml +++ /dev/null @@ -1,6 +0,0 @@ -spring: - config: - activate: - on-profile: development - import: - - "configserver:http://192.168.0.100:11080" \ No newline at end of file diff --git a/receiver/src/main/resources/application-local.yml b/receiver/src/main/resources/application-local.yml deleted file mode 100644 index 4de32f8..0000000 --- a/receiver/src/main/resources/application-local.yml +++ /dev/null @@ -1,6 +0,0 @@ -spring: - config: - activate: - on-profile: local - import: - - "configserver:http://localhost:20085" \ No newline at end of file diff --git a/receiver/src/main/resources/logback-spring.xml b/receiver/src/main/resources/logback-spring.xml deleted file mode 100644 index 907a4f5..0000000 --- a/receiver/src/main/resources/logback-spring.xml +++ /dev/null @@ -1,15 +0,0 @@ - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/receiver/src/main/resources/logback/component/logback-file.xml b/receiver/src/main/resources/logback/component/logback-file.xml deleted file mode 100644 index 65e54db..0000000 --- a/receiver/src/main/resources/logback/component/logback-file.xml +++ /dev/null @@ -1,23 +0,0 @@ - - - - - ${DIRECTORY}/${LOG_FILE_BASE}_log - - ${DIRECTORY}/${LOG_FILE_BASE}_log.%d{yyyyMMdd}.%i - - 1000MB - - 60 - - - ${FILE_LOG_PATTERN} - ${IMMEDIATE_FLUSH} - - - - 1024 - - - diff --git a/receiver/src/main/resources/logback/logback-development.xml b/receiver/src/main/resources/logback/logback-development.xml deleted file mode 100644 index 458e3d8..0000000 --- a/receiver/src/main/resources/logback/logback-development.xml +++ /dev/null @@ -1,19 +0,0 @@ - - - - - - - - - - - - - - - - - - - diff --git a/receiver/src/main/resources/logback/logback-production.xml b/receiver/src/main/resources/logback/logback-production.xml deleted file mode 100644 index f824e41..0000000 --- a/receiver/src/main/resources/logback/logback-production.xml +++ /dev/null @@ -1,19 +0,0 @@ - - - - - - - - - - - - - - - - - - - diff --git a/sender/build.gradle b/sender/build.gradle deleted file mode 100644 index 3181ae3..0000000 --- a/sender/build.gradle +++ /dev/null @@ -1,23 +0,0 @@ -dependencies { - developmentOnly 'org.springframework.boot:spring-boot-devtools' - runtimeOnly 'mysql:mysql-connector-java' - compileOnly 'org.projectlombok:lombok' - - implementation project(':support') - // https://projectreactor.io/docs/core/release/reference/#debug-activate - implementation 'org.springframework.boot:spring-boot-starter-webflux' - implementation 'org.springframework.boot:spring-boot-starter-actuator' - implementation 'org.springframework.cloud:spring-cloud-starter-config' - - annotationProcessor 'org.springframework.boot:spring-boot-configuration-processor' - annotationProcessor 'org.projectlombok:lombok' - - testImplementation 'org.springframework.boot:spring-boot-starter-test' -} - -test { - useJUnitPlatform() - testLogging { - events "passed", "skipped", "failed" - } -} \ No newline at end of file diff --git a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/SenderApplication.java b/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/SenderApplication.java deleted file mode 100644 index 20f8126..0000000 --- a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/SenderApplication.java +++ /dev/null @@ -1,22 +0,0 @@ -package com.myoa.engineering.crawl.ppomppu.sender; - -import org.springframework.boot.SpringApplication; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.context.annotation.Import; - -import com.myoa.engineering.crawl.ppomppu.support.webclient.PpomppuNotifierWebClientConfiguration; - -/** - * SenderApplication - * @author Shin Woo-jin (woo-jin.shin@linecorp.com) - * @since 2021-08-20 - * - */ -@Import({ PpomppuNotifierWebClientConfiguration.class }) -@SpringBootApplication -public class SenderApplication { - - public static void main(String[] args) { - SpringApplication.run(SenderApplication.class, args); - } -} diff --git a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/controller/TestAPIController.java b/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/controller/TestAPIController.java deleted file mode 100644 index 28192e1..0000000 --- a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/controller/TestAPIController.java +++ /dev/null @@ -1,33 +0,0 @@ -package com.myoa.engineering.crawl.ppomppu.sender.controller; - -import com.myoa.engineering.crawl.ppomppu.sender.infrastructure.client.MongeShoppingBotSlackMessageSender; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; - -import lombok.extern.slf4j.Slf4j; -import reactor.core.publisher.Mono; - -/** - * TestAPIController - * - * @author Shin Woo-jin (woo-jin.shin@linecorp.com) - * @since 2021-11-15 - */ -@Slf4j -@RestController -@RequestMapping("/api/v1") -public class TestAPIController { - - private final MongeShoppingBotSlackMessageSender sender; - - public TestAPIController(MongeShoppingBotSlackMessageSender sender) { - this.sender = sender; - } - - @GetMapping("/test") - public Mono test() { - log.info("received!!!"); - return sender.sendMessage(sender.ofMessage("testtesttest!!!")); - } -} diff --git a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/infrastructure/client/MessageSender.java b/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/infrastructure/client/MessageSender.java deleted file mode 100644 index 4153bb4..0000000 --- a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/infrastructure/client/MessageSender.java +++ /dev/null @@ -1,17 +0,0 @@ -package com.myoa.engineering.crawl.ppomppu.sender.infrastructure.client; - -import com.myoa.engineering.crawl.ppomppu.sender.dto.MessageDTO; - -import reactor.core.publisher.Mono; - -/** - * MessageSender - * @author Shin Woo-jin (woo-jin.shin@linecorp.com) - * @since 2021-11-14 - * - */ -public interface MessageSender { - - Mono sendMessage(T message); - -} diff --git a/sender/src/main/resources/application-development.yml b/sender/src/main/resources/application-development.yml deleted file mode 100644 index da50d27..0000000 --- a/sender/src/main/resources/application-development.yml +++ /dev/null @@ -1,10 +0,0 @@ -spring: - config: - activate: - on-profile: development - import: - - "configserver:http://192.168.0.100:11080" - # import: optional:configserver:http://localhost:11080 # can be start up even config server was not found. - -server: - port: 20082 diff --git a/sender/src/main/resources/application-production.yml b/sender/src/main/resources/application-production.yml deleted file mode 100644 index 4ff32ed..0000000 --- a/sender/src/main/resources/application-production.yml +++ /dev/null @@ -1,6 +0,0 @@ -spring: - config: - activate: - on-profile: production - import: - - "configserver:http://ppn-config-server:20080" diff --git a/sender/src/main/resources/logback-spring.xml b/sender/src/main/resources/logback-spring.xml deleted file mode 100644 index 32276f3..0000000 --- a/sender/src/main/resources/logback-spring.xml +++ /dev/null @@ -1,15 +0,0 @@ - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/sender/src/main/resources/logback/component/logback-file.xml b/sender/src/main/resources/logback/component/logback-file.xml deleted file mode 100644 index 65e54db..0000000 --- a/sender/src/main/resources/logback/component/logback-file.xml +++ /dev/null @@ -1,23 +0,0 @@ - - - - - ${DIRECTORY}/${LOG_FILE_BASE}_log - - ${DIRECTORY}/${LOG_FILE_BASE}_log.%d{yyyyMMdd}.%i - - 1000MB - - 60 - - - ${FILE_LOG_PATTERN} - ${IMMEDIATE_FLUSH} - - - - 1024 - - - diff --git a/sender/src/main/resources/logback/logback-development.xml b/sender/src/main/resources/logback/logback-development.xml deleted file mode 100644 index 458e3d8..0000000 --- a/sender/src/main/resources/logback/logback-development.xml +++ /dev/null @@ -1,19 +0,0 @@ - - - - - - - - - - - - - - - - - - - diff --git a/sender/src/main/resources/logback/logback-production.xml b/sender/src/main/resources/logback/logback-production.xml deleted file mode 100644 index f824e41..0000000 --- a/sender/src/main/resources/logback/logback-production.xml +++ /dev/null @@ -1,19 +0,0 @@ - - - - - - - - - - - - - - - - - - - diff --git a/settings.gradle b/settings.gradle index bc4de22..ab70af1 100644 --- a/settings.gradle +++ b/settings.gradle @@ -1,14 +1,14 @@ rootProject.name = 'PpomppuNotifier' -include 'receiver' -project(':receiver').projectDir = "$rootDir/receiver" as File - -include 'processor' -project(':processor').projectDir = "$rootDir/processor" as File - -include 'sender' -project(':sender').projectDir = "$rootDir/sender" as File +//include 'reader' +//project(':reader').projectDir = "$rootDir/reader" as File +//include 'processor' +//project(':processor').projectDir = "$rootDir/processor" as File +//include 'writer' +//project(':writer').projectDir = "$rootDir/writer" as File +include 'shopping-crawler' +project(':shopping-crawler').projectDir = "$rootDir/shopping-crawler" as File include 'support' project(':support').projectDir = "$rootDir/support" as File diff --git a/shopping-crawler/.jqwik-database b/shopping-crawler/.jqwik-database new file mode 100644 index 0000000..711006c Binary files /dev/null and b/shopping-crawler/.jqwik-database differ diff --git a/shopping-crawler/build.gradle b/shopping-crawler/build.gradle new file mode 100644 index 0000000..89c7625 --- /dev/null +++ b/shopping-crawler/build.gradle @@ -0,0 +1,42 @@ +dependencies { + developmentOnly 'org.springframework.boot:spring-boot-devtools' + runtimeOnly 'com.h2database:h2' + runtimeOnly 'com.mysql:mysql-connector-j' + compileOnly 'org.projectlombok:lombok' + + implementation project(':support') + // https://projectreactor.io/docs/core/release/reference/#debug-activate + implementation("org.springframework.boot:spring-boot-starter-web") { + exclude group: "org.springframework.boot", module: "spring-boot-starter-tomcat" + } + + implementation("org.springframework.boot:spring-boot-starter-undertow") { + exclude group: "io.undertow", module: "undertow-websockets-jsr" + } + implementation 'org.springframework.boot:spring-boot-starter-data-jpa' + implementation 'org.springframework.boot:spring-boot-configuration-processor' + implementation 'org.springframework.cloud:spring-cloud-starter-config' + implementation 'org.springframework.boot:spring-boot-starter-actuator' + implementation 'com.rometools:rome:2.1.0' + implementation 'org.jsoup:jsoup:1.17.2' + implementation 'com.h2database:h2:2.2.224' + implementation "org.springframework.cloud:spring-cloud-starter-openfeign" + implementation "io.github.openfeign:feign-hc5" + implementation 'org.ahocorasick:ahocorasick:0.6.3' + + annotationProcessor 'org.springframework.boot:spring-boot-configuration-processor' + annotationProcessor 'org.projectlombok:lombok' + + testImplementation 'org.springframework.boot:spring-boot-starter-test' + testImplementation('org.assertj:assertj-core:3.25.3') + testImplementation("org.jeasy:easy-random-core:5.0.0") + testCompileOnly 'org.projectlombok:lombok' + testAnnotationProcessor('org.projectlombok:lombok') +} + +test { + useJUnitPlatform() + testLogging { + events "passed", "skipped", "failed" + } +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/ShoppingCrawlerApplication.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/ShoppingCrawlerApplication.java new file mode 100644 index 0000000..71444d8 --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/ShoppingCrawlerApplication.java @@ -0,0 +1,19 @@ +package com.myoa.engineering.crawl.shopping; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.cloud.openfeign.EnableFeignClients; +import org.springframework.scheduling.annotation.EnableScheduling; + +@EnableScheduling +@EnableFeignClients +@EnableConfigurationProperties +@SpringBootApplication +public class ShoppingCrawlerApplication { + + public static void main(String[] args) { + SpringApplication.run(ShoppingCrawlerApplication.class, args); + } + +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/FeignDefaultConfig.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/FeignDefaultConfig.java new file mode 100644 index 0000000..f01a4d4 --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/FeignDefaultConfig.java @@ -0,0 +1,30 @@ +package com.myoa.engineering.crawl.shopping.configuration; + +import feign.Logger; +import feign.RequestInterceptor; +import feign.codec.ErrorDecoder; +import lombok.extern.slf4j.Slf4j; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; + +@Slf4j +@Configuration +public class FeignDefaultConfig { + + @Bean + Logger.Level feignLoggerLevel() { + return Logger.Level.FULL; + } + + + public static final String MIME_TYPE = + MediaType.APPLICATION_JSON_VALUE + ";charset=utf-8"; + + @Bean + public RequestInterceptor requestInterceptor() { + return requestTemplate -> requestTemplate.header(HttpHeaders.CONTENT_TYPE, MIME_TYPE); + } + +} diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/H2ConsoleConfiguration.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/datasource/H2ConsoleConfiguration.java similarity index 93% rename from processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/H2ConsoleConfiguration.java rename to shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/datasource/H2ConsoleConfiguration.java index 3fd4bd3..9f9ab8b 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/H2ConsoleConfiguration.java +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/datasource/H2ConsoleConfiguration.java @@ -1,6 +1,5 @@ -package com.myoa.engineering.crawl.ppomppu.processor.configuration; +package com.myoa.engineering.crawl.shopping.configuration.datasource; -import java.sql.SQLException; import lombok.extern.slf4j.Slf4j; import org.h2.tools.Server; import org.springframework.beans.factory.annotation.Value; @@ -10,6 +9,8 @@ import org.springframework.context.event.ContextClosedEvent; import org.springframework.context.event.ContextRefreshedEvent; import org.springframework.context.event.EventListener; +import java.sql.SQLException; + @Slf4j @Profile({"datasource-local", "datasource-development"}) @Configuration diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/PpomppuDatasourceConfiguration.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/datasource/ShoppingCrawlerDatasourceConfiguration.java similarity index 65% rename from processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/PpomppuDatasourceConfiguration.java rename to shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/datasource/ShoppingCrawlerDatasourceConfiguration.java index b78c475..bd83f05 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/PpomppuDatasourceConfiguration.java +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/datasource/ShoppingCrawlerDatasourceConfiguration.java @@ -1,54 +1,58 @@ -package com.myoa.engineering.crawl.ppomppu.processor.configuration; +package com.myoa.engineering.crawl.shopping.configuration.datasource; -import com.myoa.engineering.crawl.ppomppu.processor.configuration.properties.DatasourceProperties; -import com.myoa.engineering.crawl.ppomppu.processor.configuration.properties.DatasourceProperties.DataSourcePropertiesUnit; -import com.myoa.engineering.crawl.ppomppu.processor.configuration.properties.HibernateProperties; -import com.myoa.engineering.crawl.ppomppu.processor.configuration.properties.HikariProperties; -import com.myoa.engineering.crawl.ppomppu.processor.domain.BaseScanDomain; -import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository.BaseScanRepository; +import com.myoa.engineering.crawl.shopping.configuration.datasource.properties.DatasourceProperties; +import com.myoa.engineering.crawl.shopping.configuration.datasource.properties.HibernateProperties; +import com.myoa.engineering.crawl.shopping.configuration.datasource.properties.HikariProperties; +import com.myoa.engineering.crawl.shopping.domain.entity.BaseScanDomain; +import com.myoa.engineering.crawl.shopping.infra.repository.BaseScanRepository; import com.zaxxer.hikari.HikariConfig; import com.zaxxer.hikari.HikariDataSource; -import java.util.Enumeration; -import java.util.HashMap; -import java.util.Map; -import java.util.Properties; -import javax.persistence.EntityManagerFactory; -import javax.sql.DataSource; +import jakarta.persistence.EntityManagerFactory; import lombok.NonNull; +import org.hibernate.boot.model.naming.CamelCaseToUnderscoresNamingStrategy; +import org.hibernate.boot.model.naming.ImplicitNamingStrategyJpaCompliantImpl; import org.hibernate.cfg.AvailableSettings; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.orm.jpa.EntityManagerFactoryBuilder; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.data.jpa.repository.config.EnableJpaAuditing; import org.springframework.data.jpa.repository.config.EnableJpaRepositories; import org.springframework.orm.jpa.JpaTransactionManager; import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean; import org.springframework.transaction.PlatformTransactionManager; -@Configuration -@EnableJpaRepositories(basePackageClasses = BaseScanRepository.class, - entityManagerFactoryRef = "ppomppuNotifierProcessorEntityManagerFactory", - transactionManagerRef = "ppomppuNotifierProcessorTransactionManager" -) -public class PpomppuDatasourceConfiguration { +import javax.sql.DataSource; +import java.util.Enumeration; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; - private static final String DATA_SOURCE_UNIT_NAME = "ppn_mysql"; +@Configuration +@EnableJpaAuditing +@EnableJpaRepositories(basePackageClasses = BaseScanRepository.class, + entityManagerFactoryRef = "shoppingCrawlerEntityManagerFactory", + transactionManagerRef = "shoppingCrawlerTransactionManager" +) +public class ShoppingCrawlerDatasourceConfiguration { + + private static final String DATA_SOURCE_UNIT_NAME = "crawler-shopping"; private final DatasourceProperties dataSourceProeprties; private final HikariProperties hikariProperties; private final HibernateProperties hibernateProperties; - public PpomppuDatasourceConfiguration(DatasourceProperties dataSourceProeprties, - HikariProperties hikariProperties, - HibernateProperties hibernateProperties) { + public ShoppingCrawlerDatasourceConfiguration(DatasourceProperties dataSourceProeprties, + HikariProperties hikariProperties, + HibernateProperties hibernateProperties) { this.dataSourceProeprties = dataSourceProeprties; this.hikariProperties = hikariProperties; this.hibernateProperties = hibernateProperties; } - @Bean(name = "ppomppuNotifierProcessorDataSource") + @Bean(name = "shoppingCrawlerDataSource") public DataSource dataSource() { - DataSourcePropertiesUnit dataSourcePropertiesUnit = dataSourceProeprties.find(DATA_SOURCE_UNIT_NAME); + DatasourceProperties.DataSourcePropertiesUnit dataSourcePropertiesUnit = dataSourceProeprties.find(DATA_SOURCE_UNIT_NAME); final HikariConfig hikariConfig = new HikariConfig(); hikariConfig.setJdbcUrl(dataSourcePropertiesUnit.toCompletedJdbcUrl()); @@ -69,19 +73,19 @@ public class PpomppuDatasourceConfiguration { return dataSource; } - @Bean("ppomppuNotifierProcessorEntityManagerFactory") + @Bean("shoppingCrawlerEntityManagerFactory") public LocalContainerEntityManagerFactoryBean entityManagerFactory( - EntityManagerFactoryBuilder builder, - @Qualifier("ppomppuNotifierProcessorDataSource") DataSource dataSource) { + EntityManagerFactoryBuilder builder, + @Qualifier("shoppingCrawlerDataSource") DataSource dataSource) { return builder.dataSource(dataSource) .packages(BaseScanDomain.class) .properties(getPropsMap(DATA_SOURCE_UNIT_NAME)) .build(); } - @Bean("ppomppuNotifierProcessorTransactionManager") + @Bean("shoppingCrawlerTransactionManager") public PlatformTransactionManager transactionManager( - @Qualifier("ppomppuNotifierProcessorEntityManagerFactory") EntityManagerFactory entityManagerFactory) { + @Qualifier("shoppingCrawlerEntityManagerFactory") EntityManagerFactory entityManagerFactory) { return new JpaTransactionManager(entityManagerFactory); } @@ -91,20 +95,17 @@ public class PpomppuDatasourceConfiguration { properties.put(AvailableSettings.FORMAT_SQL, hibernateProperties.getFormatSql()); properties.put(AvailableSettings.SHOW_SQL, hibernateProperties.getShowSql()); properties.put(AvailableSettings.HBM2DDL_AUTO, hibernateProperties.getHbm2ddlAuto()); - properties.put(AvailableSettings.CONNECTION_PROVIDER_DISABLES_AUTOCOMMIT, - hibernateProperties.getDisableAutoCommit()); - properties.put(AvailableSettings.IMPLICIT_NAMING_STRATEGY, - "org.springframework.boot.orm.jpa.hibernate.SpringImplicitNamingStrategy"); - properties.put(AvailableSettings.PHYSICAL_NAMING_STRATEGY, - "org.springframework.boot.orm.jpa.hibernate.SpringPhysicalNamingStrategy"); + properties.put(AvailableSettings.CONNECTION_PROVIDER_DISABLES_AUTOCOMMIT, hibernateProperties.getDisableAutoCommit()); + properties.put(AvailableSettings.IMPLICIT_NAMING_STRATEGY, ImplicitNamingStrategyJpaCompliantImpl.class.getName()); + properties.put(AvailableSettings.PHYSICAL_NAMING_STRATEGY, CamelCaseToUnderscoresNamingStrategy.class.getName()); properties.put(AvailableSettings.GENERATE_STATISTICS, "false"); - properties.put(AvailableSettings.GLOBALLY_QUOTED_IDENTIFIERS, "true"); - properties.put(AvailableSettings.GLOBALLY_QUOTED_IDENTIFIERS_SKIP_COLUMN_DEFINITIONS, "true"); +// properties.put(AvailableSettings.GLOBALLY_QUOTED_IDENTIFIERS, "true"); +// properties.put(AvailableSettings.GLOBALLY_QUOTED_IDENTIFIERS_SKIP_COLUMN_DEFINITIONS, "true"); properties.put(AvailableSettings.STATEMENT_BATCH_SIZE, "20"); properties.put(AvailableSettings.ORDER_INSERTS, "true"); properties.put(AvailableSettings.ORDER_UPDATES, "true"); properties.put(AvailableSettings.BATCH_VERSIONED_DATA, "true"); - properties.put(AvailableSettings.USE_NEW_ID_GENERATOR_MAPPINGS, "false"); +// properties.put(AvailableSettings.JPA_ID_GENERATOR_GLOBAL_SCOPE_COMPLIANCE, "false"); return properties; } diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/properties/DatasourceProperties.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/datasource/properties/DatasourceProperties.java similarity index 62% rename from processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/properties/DatasourceProperties.java rename to shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/datasource/properties/DatasourceProperties.java index 351eb13..3a1e583 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/properties/DatasourceProperties.java +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/datasource/properties/DatasourceProperties.java @@ -1,29 +1,26 @@ -package com.myoa.engineering.crawl.ppomppu.processor.configuration.properties; +package com.myoa.engineering.crawl.shopping.configuration.datasource.properties; -import com.myoa.engineering.crawl.ppomppu.support.util.ObjectUtil; -import java.util.List; - -import lombok.Getter; -import lombok.Setter; +import com.myoa.engineering.crawl.shopping.support.util.ObjectUtil; +import lombok.Data; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.stereotype.Component; +import java.util.List; + @Component -@Setter -@Getter -@ConfigurationProperties(prefix = "infra.database") +@Data +@ConfigurationProperties(prefix = "datasource") public class DatasourceProperties { private List units; - @Getter - @Setter + @Data public static class DataSourcePropertiesUnit { private String unitName; private String schemaName; private String connectionParameters; - private String datasourceUrl; + private String dbConnectionUrl; private Boolean isSimpleConnectionUrl; private String username; private String password; @@ -31,9 +28,9 @@ public class DatasourceProperties { public String toCompletedJdbcUrl() { if (ObjectUtil.isEmpty(isSimpleConnectionUrl) || isSimpleConnectionUrl == false) { - return String.format("%s/%s?%s", datasourceUrl, schemaName, connectionParameters); + return String.format("%s/%s?%s", dbConnectionUrl, schemaName, connectionParameters); } - return datasourceUrl; + return dbConnectionUrl; } } @@ -42,7 +39,7 @@ public class DatasourceProperties { .filter(e -> e.getUnitName().equals(unitName)) .findFirst() .orElseThrow( - () -> new IllegalArgumentException(this.getClass().getName() + ": unitName Not found. " + unitName)); + () -> new IllegalArgumentException(this.getClass().getName() + ": unitName Not found. " + unitName)); } } diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/properties/HibernateProperties.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/datasource/properties/HibernateProperties.java similarity index 85% rename from processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/properties/HibernateProperties.java rename to shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/datasource/properties/HibernateProperties.java index 4543d8a..cbdfa49 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/properties/HibernateProperties.java +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/datasource/properties/HibernateProperties.java @@ -1,14 +1,12 @@ -package com.myoa.engineering.crawl.ppomppu.processor.configuration.properties; +package com.myoa.engineering.crawl.shopping.configuration.datasource.properties; -import java.util.List; import lombok.Getter; import lombok.Setter; - -import org.hibernate.dialect.MySQL8Dialect; -import org.hibernate.dialect.MySQLDialect; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.stereotype.Component; +import java.util.List; + @Component @Setter @Getter diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/properties/HikariProperties.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/datasource/properties/HikariProperties.java similarity index 86% rename from processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/properties/HikariProperties.java rename to shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/datasource/properties/HikariProperties.java index 91a52bf..47516db 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/properties/HikariProperties.java +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/datasource/properties/HikariProperties.java @@ -1,4 +1,4 @@ -package com.myoa.engineering.crawl.ppomppu.processor.configuration.properties; +package com.myoa.engineering.crawl.shopping.configuration.datasource.properties; import lombok.Getter; import lombok.Setter; diff --git a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/configuration/properties/SlackSecretProperties.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/slack/properties/SlackSecretProperties.java similarity index 77% rename from sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/configuration/properties/SlackSecretProperties.java rename to shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/slack/properties/SlackSecretProperties.java index d47983c..691f141 100644 --- a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/configuration/properties/SlackSecretProperties.java +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/slack/properties/SlackSecretProperties.java @@ -1,4 +1,4 @@ -package com.myoa.engineering.crawl.ppomppu.sender.configuration.properties; +package com.myoa.engineering.crawl.shopping.configuration.slack.properties; import java.util.List; import lombok.Data; @@ -7,18 +7,16 @@ import lombok.Setter; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.stereotype.Component; -@Getter -@Setter +@Data @Component -@ConfigurationProperties("infra.slack.bot") +@ConfigurationProperties(prefix = "slack.bot") public class SlackSecretProperties { private List units; @Data public static class SlackSecretPropertiesUnit { - - private String botName; + private String botUnitName; private String username; private String iconEmoji; private String channel; @@ -27,7 +25,7 @@ public class SlackSecretProperties { public SlackSecretPropertiesUnit find(String botUnitName) { return units.stream() - .filter(e -> e.getBotName().equals(botUnitName)) + .filter(e -> e.getBotUnitName().equals(botUnitName)) .findFirst() .orElseThrow(() -> new IllegalArgumentException("not found bot unit name : " + botUnitName)); } diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/controller/TestAPIController.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/controller/TestAPIController.java new file mode 100644 index 0000000..7992a38 --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/controller/TestAPIController.java @@ -0,0 +1,22 @@ +package com.myoa.engineering.crawl.shopping.controller; + +import com.myoa.engineering.crawl.shopping.crawlhandler.PpomppuCrawlDomesticHandler; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +@RestController +@RequestMapping("/api/v1/exploit") +public class TestAPIController { + + private final PpomppuCrawlDomesticHandler ppomppuCrawlDomesticHandler; + + public TestAPIController(PpomppuCrawlDomesticHandler ppomppuCrawlDomesticHandler) { + this.ppomppuCrawlDomesticHandler = ppomppuCrawlDomesticHandler; + } + + @GetMapping("/triggers") + public void triggerExploit() { + ppomppuCrawlDomesticHandler.handle(); + } +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/crawlhandler/CrawlHandler.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/crawlhandler/CrawlHandler.java new file mode 100644 index 0000000..8b60ff0 --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/crawlhandler/CrawlHandler.java @@ -0,0 +1,10 @@ +package com.myoa.engineering.crawl.shopping.crawlhandler; + +import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget; + +public interface CrawlHandler { + + CrawlTarget getCrawlTarget(); + + void handle(); +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/crawlhandler/FmkoreaCrawlHandler.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/crawlhandler/FmkoreaCrawlHandler.java new file mode 100644 index 0000000..bfb6938 --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/crawlhandler/FmkoreaCrawlHandler.java @@ -0,0 +1,18 @@ +package com.myoa.engineering.crawl.shopping.crawlhandler; + +import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; + +@Slf4j +@Component +public class FmkoreaCrawlHandler implements CrawlHandler { + @Override + public CrawlTarget getCrawlTarget() { + return CrawlTarget.FMKOREA; + } + + @Override + public void handle() { + } +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/crawlhandler/PpomppuCrawlDomesticHandler.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/crawlhandler/PpomppuCrawlDomesticHandler.java new file mode 100644 index 0000000..541c969 --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/crawlhandler/PpomppuCrawlDomesticHandler.java @@ -0,0 +1,58 @@ +package com.myoa.engineering.crawl.shopping.crawlhandler; + +import com.myoa.engineering.crawl.shopping.crawlhandler.parser.PpomppuArticleParserV2; +import com.myoa.engineering.crawl.shopping.domain.entity.v2.Article; +import com.myoa.engineering.crawl.shopping.infra.client.ppomppu.PpomppuBoardClientV2; +import com.myoa.engineering.crawl.shopping.service.ArticleCommandService; +import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget; +import org.springframework.stereotype.Component; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Stream; + +@Component +public class PpomppuCrawlDomesticHandler implements CrawlHandler { + + private final PpomppuBoardClientV2 ppomppuBoardClient; + private final PpomppuArticleParserV2 ppomppuArticleParserV2; + private final ArticleCommandService articleCommandService; + + public PpomppuCrawlDomesticHandler(PpomppuBoardClientV2 ppomppuBoardClient, + PpomppuArticleParserV2 ppomppuArticleParserV2, + ArticleCommandService articleCommandService) { + this.ppomppuBoardClient = ppomppuBoardClient; + this.ppomppuArticleParserV2 = ppomppuArticleParserV2; + this.articleCommandService = articleCommandService; + } + + @Override + public CrawlTarget getCrawlTarget() { + return CrawlTarget.PPOMPPU_DOMESTIC; + } + + @Override + public void handle() { + + String boardHtmlPage1 = ppomppuBoardClient.getBoardHtml("/zboard/zboard.php", generateRequestParams(1)); + List
parsedPage1 = ppomppuArticleParserV2.parse(boardHtmlPage1); + + String boardHtmlPage2 = ppomppuBoardClient.getBoardHtml("/zboard/zboard.php", generateRequestParams(2)); + List
parsedPage2 = ppomppuArticleParserV2.parse(boardHtmlPage2); + + List
merged = Stream.of(parsedPage1, parsedPage2) + .flatMap(List::stream) + .map(e -> e.updateCrawlTarget(getCrawlTarget())) + .toList(); + + articleCommandService.upsert(merged); + } + + private Map generateRequestParams(int pageId) { + Map params = new HashMap<>(); + params.put("id", "ppomppu"); + params.put("page", String.valueOf(pageId)); + return params; + } +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/crawlhandler/parser/PpomppuArticleParserV2.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/crawlhandler/parser/PpomppuArticleParserV2.java new file mode 100644 index 0000000..850f8a1 --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/crawlhandler/parser/PpomppuArticleParserV2.java @@ -0,0 +1,115 @@ +package com.myoa.engineering.crawl.shopping.crawlhandler.parser; + +import com.myoa.engineering.crawl.shopping.domain.entity.v2.Article; +import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName; +import com.myoa.engineering.crawl.shopping.util.DateTimeUtils; +import com.myoa.engineering.crawl.shopping.util.NumberUtils; +import io.micrometer.core.instrument.util.StringUtils; +import lombok.extern.slf4j.Slf4j; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; +import org.springframework.stereotype.Component; + +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +@Slf4j +@Component +public final class PpomppuArticleParserV2 { + + private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yy.MM.dd HH:mm:ss") + .withZone(ZoneId.of("Asia/Seoul")); + + private PpomppuArticleParserV2() { + } + + public List
parse(String html) { + Elements trElements = converHtmlToTrElements(html); + return trElements.stream() + .filter(this::isRealArticle) + .map(this::parse) + .toList(); + + } + + private Elements converHtmlToTrElements(String data) { + Document document = Jsoup.parse(data); + Elements trList = document.getElementById("revolution_main_table").getElementsByTag("tr"); + return trList; + } + + private boolean isRealArticle(Element tr) { + Elements tdList = tr.getElementsByTag("td"); + if (tdList.size() != 6) { + return false; + } + + if (!hasOnlyNumeric(tdList.get(0))) { + return false; + } + + return true; + } + + private Pattern pattern_numeric = Pattern.compile("\\d+"); + + private boolean hasOnlyNumeric(Element td) { + return pattern_numeric.matcher(td.text()).matches(); + } + + private Article parse(Element tr) { + Elements tdList = tr.getElementsByTag("td"); + Long articleId = Long.parseLong(tdList.get(0).text()); + + String title = tdList.get(2).text(); + String articleUrl = parseArticleUrl(tdList.get(2).getElementsByTag("a").attr("href")); + String boardName = parseBoardName(title); + Integer recommended = parseRecommended(tdList.get(4)); + Integer hit = NumberUtils.parseInt(tdList.get(5).text(), 0); + ZonedDateTime registeredAt = DateTimeUtils.parse(tdList.get(3).text()); + + return Article.builder() + .articleId(articleId) + .title(title) + .boardName(boardName) + .articleUrl(articleUrl) + .recommended(recommended) + .hit(hit) + .registeredAt(registeredAt) + .build(); + } + + public Integer parseRecommended(Element td) { + final String voteString = td.text(); + if (StringUtils.isEmpty(voteString)) { + return null; + } + + final int voteUp = Integer.parseInt(td.text().split(" - ")[0]); + final int voteDown = Integer.parseInt(td.text().split(" - ")[1]); + int recommended = voteUp - voteDown; + return recommended; + } + + public static String parseArticleUrl(String data) { + return PpomppuBoardName.ofViewPageUrl(data); + } + + Pattern patternBoardName = Pattern.compile("\\[(.+?)\\]"); + + public String parseBoardName(String fullTitle) { + Matcher matcher = patternBoardName.matcher(fullTitle); + String lastMatched = null; + while (matcher.find()) { + lastMatched = matcher.group(1); + } + return lastMatched; + } + +} diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/Auditable.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/Auditable.java similarity index 62% rename from processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/Auditable.java rename to shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/Auditable.java index e16f83b..ce5c073 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/Auditable.java +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/Auditable.java @@ -1,17 +1,18 @@ -package com.myoa.engineering.crawl.ppomppu.processor.domain; +package com.myoa.engineering.crawl.shopping.domain.entity; -import java.io.Serializable; -import java.time.Instant; -import javax.persistence.Column; +import lombok.Getter; import org.springframework.data.annotation.CreatedDate; import org.springframework.data.annotation.LastModifiedDate; -/** - * Auditable - * - * @author Shin Woo-jin (woozu.shin@kakaoent.com) - * @since 2021-09-08 - */ +import jakarta.persistence.*; +import org.springframework.data.jpa.domain.support.AuditingEntityListener; + +import java.io.Serializable; +import java.time.Instant; + +@Getter +@MappedSuperclass +@EntityListeners(AuditingEntityListener.class) public abstract class Auditable implements Serializable { private static final long serialVersionUID = -7105030870015828551L; diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/BaseScanDomain.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/BaseScanDomain.java similarity index 69% rename from processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/BaseScanDomain.java rename to shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/BaseScanDomain.java index 62bcf9f..bddd3dd 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/BaseScanDomain.java +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/BaseScanDomain.java @@ -1,4 +1,4 @@ -package com.myoa.engineering.crawl.ppomppu.processor.domain; +package com.myoa.engineering.crawl.shopping.domain.entity; /** * BaseScanDomain diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/PpomppuArticle.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v1/PpomppuArticle.java similarity index 100% rename from processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/PpomppuArticle.java rename to shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v1/PpomppuArticle.java diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/PpomppuBoardFeedStatus.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v1/PpomppuBoardFeedStatus.java similarity index 76% rename from processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/PpomppuBoardFeedStatus.java rename to shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v1/PpomppuBoardFeedStatus.java index b751c13..1ff7d5e 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/PpomppuBoardFeedStatus.java +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v1/PpomppuBoardFeedStatus.java @@ -1,19 +1,14 @@ -package com.myoa.engineering.crawl.ppomppu.processor.domain; +package com.myoa.engineering.crawl.shopping.domain.entity.v1; -import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName; -import java.time.Instant; -import javax.persistence.Column; -import javax.persistence.Entity; -import javax.persistence.EnumType; -import javax.persistence.Enumerated; -import javax.persistence.GeneratedValue; -import javax.persistence.GenerationType; -import javax.persistence.Id; -import javax.persistence.Table; +import com.myoa.engineering.crawl.shopping.domain.entity.Auditable; +import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName; +import jakarta.persistence.*; import lombok.Builder; import lombok.Getter; import lombok.NoArgsConstructor; +import java.time.Instant; + @Getter @NoArgsConstructor @Entity diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/PublishedHistory.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v1/PublishedHistory.java similarity index 61% rename from processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/PublishedHistory.java rename to shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v1/PublishedHistory.java index e52c069..46d39dd 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/PublishedHistory.java +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v1/PublishedHistory.java @@ -1,15 +1,12 @@ -package com.myoa.engineering.crawl.ppomppu.processor.domain; +package com.myoa.engineering.crawl.shopping.domain.entity.v1; -import java.time.Instant; -import javax.persistence.Column; -import javax.persistence.Entity; -import javax.persistence.GeneratedValue; -import javax.persistence.GenerationType; -import javax.persistence.Id; -import javax.persistence.Table; +import com.myoa.engineering.crawl.shopping.domain.entity.Auditable; import lombok.Getter; import lombok.NoArgsConstructor; +import jakarta.persistence.*; +import java.time.Instant; + @Getter @NoArgsConstructor @Entity diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v1/SubscribedBoard.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v1/SubscribedBoard.java new file mode 100644 index 0000000..ebe0e10 --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v1/SubscribedBoard.java @@ -0,0 +1,27 @@ +package com.myoa.engineering.crawl.shopping.domain.entity.v1; + +import com.myoa.engineering.crawl.shopping.domain.entity.Auditable; +import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName; +import lombok.Getter; +import lombok.NoArgsConstructor; + +import jakarta.persistence.*; + +@Getter +@NoArgsConstructor +@Entity +@Table(name = "subscribed_board") +public class SubscribedBoard extends Auditable { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + @Column + private Long userId; + + @Column + @Enumerated(EnumType.STRING) + private PpomppuBoardName boardName; + +} diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/SubscribedUser.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v1/SubscribedUser.java similarity index 50% rename from processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/SubscribedUser.java rename to shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v1/SubscribedUser.java index fd89bde..a8dc32e 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/SubscribedUser.java +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v1/SubscribedUser.java @@ -1,20 +1,17 @@ -package com.myoa.engineering.crawl.ppomppu.processor.domain; +package com.myoa.engineering.crawl.shopping.domain.entity.v1; -import java.time.Instant; -import javax.persistence.Column; -import javax.persistence.Entity; -import javax.persistence.GeneratedValue; -import javax.persistence.GenerationType; -import javax.persistence.Id; -import javax.persistence.Table; +import com.myoa.engineering.crawl.shopping.domain.entity.Auditable; import lombok.Getter; import lombok.NoArgsConstructor; +import jakarta.persistence.*; +import java.time.Instant; + @Getter @NoArgsConstructor @Entity @Table(name = "subscribed_user") -public class SubscribedUser extends Auditable{ +public class SubscribedUser extends Auditable { @Id @GeneratedValue(strategy = GenerationType.IDENTITY) diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v2/AppUser.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v2/AppUser.java new file mode 100644 index 0000000..fc31469 --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v2/AppUser.java @@ -0,0 +1,29 @@ +package com.myoa.engineering.crawl.shopping.domain.entity.v2; + +import com.myoa.engineering.crawl.shopping.domain.entity.Auditable; +import jakarta.persistence.*; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; +import lombok.NoArgsConstructor; + +@Getter +@Builder +@NoArgsConstructor +@AllArgsConstructor +@Entity +@Table +public class AppUser extends Auditable { + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + @Column + private String name; + + @Column + private String slackId; + + @Column + private Boolean enabled; +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v2/Article.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v2/Article.java new file mode 100644 index 0000000..a6e7a3e --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v2/Article.java @@ -0,0 +1,63 @@ +package com.myoa.engineering.crawl.shopping.domain.entity.v2; + +import com.myoa.engineering.crawl.shopping.domain.entity.Auditable; +import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget; +import jakarta.persistence.*; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; +import lombok.NoArgsConstructor; + +import java.time.ZonedDateTime; + +@Getter +@Builder +@NoArgsConstructor +@AllArgsConstructor +@Entity +@Table +public class Article extends Auditable { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + @Column + private Long articleId; + + @Column + @Enumerated(EnumType.STRING) + private CrawlTarget crawlTarget; + + @Column + private String boardName; + + @Column + private String articleUrl; + + @Column + private String title; + + @Column + private Integer hit; + + @Column + private Integer recommended; + + @Column + private ZonedDateTime registeredAt; + + public Article update(Article article) { + this.boardName = article.getBoardName(); + this.articleUrl = article.getArticleUrl(); + this.title = article.getTitle(); + this.hit = article.getHit(); + this.recommended = article.getRecommended(); + return this; + } + + public Article updateCrawlTarget(CrawlTarget crawlTarget) { + this.crawlTarget = crawlTarget; + return this; + } +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v2/SubscribedKeyword.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v2/SubscribedKeyword.java new file mode 100644 index 0000000..1aa294c --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v2/SubscribedKeyword.java @@ -0,0 +1,33 @@ +package com.myoa.engineering.crawl.shopping.domain.entity.v2; + +import com.myoa.engineering.crawl.shopping.domain.entity.Auditable; +import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget; +import jakarta.persistence.*; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; +import lombok.NoArgsConstructor; + +@Getter +@Builder +@NoArgsConstructor +@AllArgsConstructor +@Entity +@Table +public class SubscribedKeyword extends Auditable { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + @Column + private String keyword; + + @Column + @Enumerated(EnumType.STRING) + private CrawlTarget crawlTarget; + + @Column + private String userId; + +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/model/UserNotifyModel.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/model/UserNotifyModel.java new file mode 100644 index 0000000..36a261a --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/model/UserNotifyModel.java @@ -0,0 +1,24 @@ +package com.myoa.engineering.crawl.shopping.domain.model; + +import com.myoa.engineering.crawl.shopping.domain.model.v2.ArticleModel; +import lombok.*; + +import java.util.List; + + +@ToString +@Getter +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class UserNotifyModel { + private String userId; + private List articles; + + public static UserNotifyModel of(String userId, List articles) { + return UserNotifyModel.builder() + .userId(userId) + .articles(articles) + .build(); + } +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/model/v2/AppUserModel.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/model/v2/AppUserModel.java new file mode 100644 index 0000000..8c22c7d --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/model/v2/AppUserModel.java @@ -0,0 +1,26 @@ +package com.myoa.engineering.crawl.shopping.domain.model.v2; + +import com.myoa.engineering.crawl.shopping.domain.entity.v2.AppUser; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; + +@Getter +@Builder +@AllArgsConstructor +public class AppUserModel { + + private Long id; + private String name; + private String slackId; + private Boolean enabled; + + public static AppUserModel from(AppUser entity) { + return AppUserModel.builder() + .id(entity.getId()) + .name(entity.getName()) + .slackId(entity.getSlackId()) + .enabled(entity.getEnabled()) + .build(); + } +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/model/v2/ArticleModel.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/model/v2/ArticleModel.java new file mode 100644 index 0000000..ec7626b --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/model/v2/ArticleModel.java @@ -0,0 +1,25 @@ +package com.myoa.engineering.crawl.shopping.domain.model.v2; + +import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget; +import lombok.*; + +import java.time.ZonedDateTime; + +@ToString +@Getter +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class ArticleModel { + + private Long id; + private Long articleId; + private CrawlTarget crawlTarget; + private String boardName; + private String articleUrl; + private String title; + private Integer hit; + private Integer recommended; + private ZonedDateTime registeredAt; + +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/model/v2/SubscribedKeywordAggregatedModel.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/model/v2/SubscribedKeywordAggregatedModel.java new file mode 100644 index 0000000..a6fa5dd --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/model/v2/SubscribedKeywordAggregatedModel.java @@ -0,0 +1,27 @@ +package com.myoa.engineering.crawl.shopping.domain.model.v2; + +import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget; +import com.myoa.engineering.crawl.shopping.util.AhoCorasickUtils; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; +import org.ahocorasick.trie.Trie; + +import java.util.List; + +@Getter +@Builder +@AllArgsConstructor +public class SubscribedKeywordAggregatedModel { + private final Trie ahoCorasickTrie; + private final String userId; + private final CrawlTarget crawlTarget; + + public static SubscribedKeywordAggregatedModel of(String userId, CrawlTarget crawlTarget, List keywords) { + return SubscribedKeywordAggregatedModel.builder() + .userId(userId) + .crawlTarget(crawlTarget) + .ahoCorasickTrie(AhoCorasickUtils.generateTrie(keywords)) + .build(); + } +} diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/FeedParsedResult.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/FeedParsedResult.java similarity index 89% rename from processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/FeedParsedResult.java rename to shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/FeedParsedResult.java index 2e2eca6..f169a3e 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/FeedParsedResult.java +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/FeedParsedResult.java @@ -1,12 +1,13 @@ -package com.myoa.engineering.crawl.ppomppu.processor.dto; +package com.myoa.engineering.crawl.shopping.dto; -import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName; -import java.io.Serializable; -import java.time.Instant; +import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName; import lombok.Builder; import lombok.Getter; import lombok.NoArgsConstructor; +import java.io.Serializable; +import java.time.Instant; + /** * FeedParsedResult * diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/PpomppuArticleParseDTO.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/PpomppuArticleParseDTO.java similarity index 100% rename from processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/PpomppuArticleParseDTO.java rename to shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/PpomppuArticleParseDTO.java diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/PpomppuArticleTransformer.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/PpomppuArticleTransformer.java similarity index 100% rename from processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/PpomppuArticleTransformer.java rename to shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/PpomppuArticleTransformer.java diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/constant/WebClientPropertiesUnitName.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/constant/WebClientPropertiesUnitName.java similarity index 83% rename from processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/constant/WebClientPropertiesUnitName.java rename to shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/constant/WebClientPropertiesUnitName.java index 3383138..03615c7 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/constant/WebClientPropertiesUnitName.java +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/constant/WebClientPropertiesUnitName.java @@ -1,4 +1,4 @@ -package com.myoa.engineering.crawl.ppomppu.processor.dto.constant; +package com.myoa.engineering.crawl.shopping.dto.constant; import lombok.AllArgsConstructor; import lombok.Getter; diff --git a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/dto/MessageDTO.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/slack/MessageDTO.java similarity index 76% rename from sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/dto/MessageDTO.java rename to shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/slack/MessageDTO.java index db45e43..018a1fd 100644 --- a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/dto/MessageDTO.java +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/slack/MessageDTO.java @@ -1,4 +1,4 @@ -package com.myoa.engineering.crawl.ppomppu.sender.dto; +package com.myoa.engineering.crawl.shopping.dto.slack; import java.io.Serializable; diff --git a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/dto/SlackBaseMessageBlock.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/slack/SlackBaseMessageBlock.java similarity index 100% rename from sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/dto/SlackBaseMessageBlock.java rename to shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/slack/SlackBaseMessageBlock.java diff --git a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/dto/SlackImageMessageBlock.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/slack/SlackImageMessageBlock.java similarity index 100% rename from sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/dto/SlackImageMessageBlock.java rename to shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/slack/SlackImageMessageBlock.java diff --git a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/dto/SlackMessageBlock.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/slack/SlackMessageBlock.java similarity index 100% rename from sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/dto/SlackMessageBlock.java rename to shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/slack/SlackMessageBlock.java diff --git a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/dto/SlackMessageBlockType.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/slack/SlackMessageBlockType.java similarity index 100% rename from sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/dto/SlackMessageBlockType.java rename to shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/slack/SlackMessageBlockType.java diff --git a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/dto/SlackMessageDTO.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/slack/SlackMessageDTO.java similarity index 97% rename from sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/dto/SlackMessageDTO.java rename to shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/slack/SlackMessageDTO.java index 5c8987d..2f55024 100644 --- a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/dto/SlackMessageDTO.java +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/slack/SlackMessageDTO.java @@ -1,4 +1,4 @@ -package com.myoa.engineering.crawl.ppomppu.sender.dto; +package com.myoa.engineering.crawl.shopping.dto.slack; import java.util.List; diff --git a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/dto/SlackSectionMessageBlock.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/slack/SlackSectionMessageBlock.java similarity index 100% rename from sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/dto/SlackSectionMessageBlock.java rename to shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/slack/SlackSectionMessageBlock.java diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/event/ArticleUpsertEvent.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/event/ArticleUpsertEvent.java new file mode 100644 index 0000000..f7cbe79 --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/event/ArticleUpsertEvent.java @@ -0,0 +1,13 @@ +package com.myoa.engineering.crawl.shopping.event; + +import com.myoa.engineering.crawl.shopping.domain.model.v2.ArticleModel; +import org.springframework.context.ApplicationEvent; + +import java.util.List; + +public class ArticleUpsertEvent extends ApplicationEvent { + + public ArticleUpsertEvent(List source) { + super(source); + } +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/event/ArticleUpsertEventPayload.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/event/ArticleUpsertEventPayload.java new file mode 100644 index 0000000..2fab09b --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/event/ArticleUpsertEventPayload.java @@ -0,0 +1,15 @@ +package com.myoa.engineering.crawl.shopping.event; + +import com.myoa.engineering.crawl.shopping.domain.model.v2.ArticleModel; +import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget; +import lombok.AllArgsConstructor; +import lombok.Getter; + +import java.util.List; + +@Getter +@AllArgsConstructor +public class ArticleUpsertEventPayload { + private final List articles; + private final CrawlTarget crawlTarget; +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/event/handler/ArticleUpsertEventListener.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/event/handler/ArticleUpsertEventListener.java new file mode 100644 index 0000000..f0ac31c --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/event/handler/ArticleUpsertEventListener.java @@ -0,0 +1,86 @@ +package com.myoa.engineering.crawl.shopping.event.handler; + +import com.myoa.engineering.crawl.shopping.domain.model.UserNotifyModel; +import com.myoa.engineering.crawl.shopping.domain.model.v2.AppUserModel; +import com.myoa.engineering.crawl.shopping.domain.model.v2.ArticleModel; +import com.myoa.engineering.crawl.shopping.domain.model.v2.SubscribedKeywordAggregatedModel; +import com.myoa.engineering.crawl.shopping.event.ArticleUpsertEvent; +import com.myoa.engineering.crawl.shopping.service.AppUserQueryService; +import com.myoa.engineering.crawl.shopping.service.SubscribedKeywordCacheService; +import com.myoa.engineering.crawl.shopping.service.UserNotifyService; +import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget; +import org.springframework.context.event.EventListener; +import org.springframework.stereotype.Component; + +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; + +@Component +public class ArticleUpsertEventListener { + + private final SubscribedKeywordCacheService subscribedKeywordCacheService; + private final AppUserQueryService appUserQueryService; + private final UserNotifyService userNotifyService; + + + public ArticleUpsertEventListener(SubscribedKeywordCacheService subscribedKeywordCacheService, + AppUserQueryService appUserQueryService, UserNotifyService userNotifyService) { + this.subscribedKeywordCacheService = subscribedKeywordCacheService; + this.appUserQueryService = appUserQueryService; + this.userNotifyService = userNotifyService; + } + + + @EventListener + public void handleArticleUpsertEvent(ArticleUpsertEvent event) { + Map> articleMap = + ((List) event.getSource()).stream() + .collect(Collectors.groupingBy(ArticleModel::getCrawlTarget)); + List appUsers = appUserQueryService.findAll(); + + appUsers.stream() + .filter(AppUserModel::getEnabled) + .map(user -> { + List filteredArticles = handleAhoCorasick(articleMap) + .apply(subscribedKeywordCacheService.getSubscribedKeywordsCached(user.getName())); + return UserNotifyModel.of(user.getName(), filteredArticles); + }) + .forEach(this::notifyMessage); + + } + + private Function, List> handleAhoCorasick( + Map> articleMap) { + return userTrieModel -> { + return userTrieModel + .entrySet() + .stream().filter(e -> articleMap.containsKey(e.getKey())) + .map((entry) -> filterAhocorasick(articleMap.get(entry.getKey()), entry.getValue())) + .flatMap(List::stream) + .toList(); +// return UserNotifyModel.of(userTrieModel.values().stream().findFirst().get().getUserId(), +// filteredArticle); + }; + } + + private List filterAhocorasick(List articles, + SubscribedKeywordAggregatedModel trieModel) { + return articles.stream() + .filter(article -> !trieModel.getAhoCorasickTrie() + .parseText(article.getTitle()) + .isEmpty()) + .toList(); + //ArticleUpsertEventListener::printArticle + } + + private void notifyMessage(UserNotifyModel article) { + System.out.println("article = " + article); + if (article.getArticles().isEmpty()){ + return; + } + userNotifyService.notify("안녕 " + article.getUserId() + "\n" + article.getArticles()); + } + +} diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/client/MessageSenderAPIClient.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/MessageSenderAPIClient.java similarity index 100% rename from processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/client/MessageSenderAPIClient.java rename to shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/MessageSenderAPIClient.java diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/fmkorea/FmkoreaBoardClient.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/fmkorea/FmkoreaBoardClient.java new file mode 100644 index 0000000..1d0f0e0 --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/fmkorea/FmkoreaBoardClient.java @@ -0,0 +1,12 @@ +package com.myoa.engineering.crawl.shopping.infra.client.fmkorea; + +import org.springframework.cloud.openfeign.FeignClient; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; + +@FeignClient(value = "fmkorea-board-client", url = "https://fmkorea.com") +public interface FmkoreaBoardClient { + + @GetMapping("{boardLink}") + String getBoardHtml(@PathVariable("boardLink") String boardLink); +} diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/client/PpomppuBoardFeedRetriever.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/ppomppu/PpomppuBoardClient.java similarity index 62% rename from processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/client/PpomppuBoardFeedRetriever.java rename to shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/ppomppu/PpomppuBoardClient.java index 9db4dab..1ab1b8e 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/client/PpomppuBoardFeedRetriever.java +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/ppomppu/PpomppuBoardClient.java @@ -1,16 +1,7 @@ -package com.myoa.engineering.crawl.ppomppu.processor.infrastructure.client; +package com.myoa.engineering.crawl.shopping.infra.client.ppomppu; -import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebClientFilterFactory; -import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebFluxExchangeStragiesFactory; -import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName; import lombok.extern.slf4j.Slf4j; - -import org.springframework.core.ParameterizedTypeReference; import org.springframework.stereotype.Component; -import org.springframework.web.reactive.function.client.WebClient; -import org.springframework.web.reactive.function.client.WebClientRequestException; -import reactor.core.publisher.Mono; -import reactor.core.scheduler.Schedulers; /** * PpomppuBoardFeedRetriever @@ -20,11 +11,11 @@ import reactor.core.scheduler.Schedulers; */ @Slf4j @Component -public class PpomppuBoardFeedRetriever { - +public class PpomppuBoardClient { +/* private final WebClient webClient; - public PpomppuBoardFeedRetriever(WebClient.Builder webClientBuilder) { + public PpomppuBoardClient(WebClient.Builder webClientBuilder) { this.webClient = webClientBuilder.baseUrl(PpomppuBoardName.PPOMPPU_URL) .exchangeStrategies(WebFluxExchangeStragiesFactory.ofTextHtml()) .filter(WebClientFilterFactory.logRequest()) @@ -43,5 +34,5 @@ public class PpomppuBoardFeedRetriever { }); // .doOnNext(e -> log.info("[getHtml] {}", e)); } - +*/ } diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/ppomppu/PpomppuBoardClientV2.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/ppomppu/PpomppuBoardClientV2.java new file mode 100644 index 0000000..287dbae --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/ppomppu/PpomppuBoardClientV2.java @@ -0,0 +1,17 @@ +package com.myoa.engineering.crawl.shopping.infra.client.ppomppu; + +import org.springframework.cloud.openfeign.FeignClient; +import org.springframework.cloud.openfeign.SpringQueryMap; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestParam; + +import java.util.Map; + +@FeignClient(value = "ppomppu-board-client", url = "https://www.ppomppu.co.kr") +public interface PpomppuBoardClientV2 { + + @GetMapping("{boardLink}") + String getBoardHtml(@PathVariable("boardLink") String boardLink, + @SpringQueryMap Map params); +} diff --git a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/infrastructure/client/MongeShoppingBotSlackMessageSender.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/slack/MongeShoppingBotSlackMessageSender.java similarity index 100% rename from sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/infrastructure/client/MongeShoppingBotSlackMessageSender.java rename to shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/slack/MongeShoppingBotSlackMessageSender.java diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/slack/SlackAPIClient.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/slack/SlackAPIClient.java new file mode 100644 index 0000000..1d1ed89 --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/slack/SlackAPIClient.java @@ -0,0 +1,17 @@ +package com.myoa.engineering.crawl.shopping.infra.client.slack; + +import com.myoa.engineering.crawl.shopping.configuration.FeignDefaultConfig; +import com.myoa.engineering.crawl.shopping.dto.slack.SlackMessageDTO; +import org.springframework.cloud.openfeign.FeignClient; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestHeader; + +@FeignClient(value = "slack-api-client", url = "https://slack.com/api", + configuration = FeignDefaultConfig.class) +public interface SlackAPIClient { + + @PostMapping("/chat.postMessage") + String sendMessage(@RequestBody SlackMessageDTO message, + @RequestHeader("Authorization") String token); +} diff --git a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/infrastructure/client/SlackMessageSender.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/slack/SlackMessageSender.java similarity index 70% rename from sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/infrastructure/client/SlackMessageSender.java rename to shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/slack/SlackMessageSender.java index cfec3f6..700b3c0 100644 --- a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/infrastructure/client/SlackMessageSender.java +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/slack/SlackMessageSender.java @@ -1,17 +1,6 @@ -package com.myoa.engineering.crawl.ppomppu.sender.infrastructure.client; - -import org.springframework.http.HttpHeaders; -import org.springframework.http.MediaType; -import org.springframework.web.reactive.function.client.WebClient; -import org.springframework.web.reactive.function.client.WebClientRequestException; - -import com.myoa.engineering.crawl.ppomppu.sender.dto.SlackMessageDTO; -import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebClientFilterFactory; -import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebFluxExchangeStragiesFactory; +package com.myoa.engineering.crawl.shopping.infra.client.slack; import lombok.extern.slf4j.Slf4j; -import reactor.core.publisher.Mono; -import reactor.core.scheduler.Schedulers; /** * SlackMessageSender @@ -20,8 +9,9 @@ import reactor.core.scheduler.Schedulers; * @since 2021-09-08 */ @Slf4j -public class SlackMessageSender implements MessageSender { +public class SlackMessageSender { /* implements MessageSender {*/ +/* private static final String SLACK_API_URL = "https://slack.com/api"; private final WebClient webClient; @@ -52,4 +42,6 @@ public class SlackMessageSender implements MessageSender { .doOnNext(e -> log.info("[sendMessage] {}", e)); } + */ + } diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/repository/BaseScanRepository.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/repository/BaseScanRepository.java new file mode 100644 index 0000000..04447d8 --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/repository/BaseScanRepository.java @@ -0,0 +1,4 @@ +package com.myoa.engineering.crawl.shopping.infra.repository; + +public interface BaseScanRepository { +} diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/repository/PpomppuArticleRepository.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/repository/v1/PpomppuArticleRepository.java similarity index 58% rename from processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/repository/PpomppuArticleRepository.java rename to shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/repository/v1/PpomppuArticleRepository.java index 0b727d9..885d1ce 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/repository/PpomppuArticleRepository.java +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/repository/v1/PpomppuArticleRepository.java @@ -1,6 +1,6 @@ -package com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository; +package com.myoa.engineering.crawl.shopping.infra.repository.v1; -import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle; +import com.myoa.engineering.crawl.shopping.domain.entity.v1.PpomppuArticle; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Repository; diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/repository/PpomppuBoardFeedStatusRepository.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/repository/v1/PpomppuBoardFeedStatusRepository.java similarity index 58% rename from processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/repository/PpomppuBoardFeedStatusRepository.java rename to shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/repository/v1/PpomppuBoardFeedStatusRepository.java index e84a077..0e87f92 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/repository/PpomppuBoardFeedStatusRepository.java +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/repository/v1/PpomppuBoardFeedStatusRepository.java @@ -1,11 +1,12 @@ -package com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository; +package com.myoa.engineering.crawl.shopping.infra.repository.v1; -import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuBoardFeedStatus; -import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName; -import java.util.Optional; +import com.myoa.engineering.crawl.shopping.domain.entity.v1.PpomppuBoardFeedStatus; +import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Repository; +import java.util.Optional; + @Repository public interface PpomppuBoardFeedStatusRepository extends JpaRepository { diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/repository/v2/AppUserRepository.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/repository/v2/AppUserRepository.java new file mode 100644 index 0000000..6579b4a --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/repository/v2/AppUserRepository.java @@ -0,0 +1,9 @@ +package com.myoa.engineering.crawl.shopping.infra.repository.v2; + +import com.myoa.engineering.crawl.shopping.domain.entity.v2.AppUser; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.stereotype.Repository; + +@Repository +public interface AppUserRepository extends JpaRepository { +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/repository/v2/ArticleRepository.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/repository/v2/ArticleRepository.java new file mode 100644 index 0000000..1d80b33 --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/repository/v2/ArticleRepository.java @@ -0,0 +1,16 @@ +package com.myoa.engineering.crawl.shopping.infra.repository.v2; + +import com.myoa.engineering.crawl.shopping.domain.entity.v2.Article; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.stereotype.Repository; + +import java.util.Collection; +import java.util.List; +import java.util.Optional; + +@Repository +public interface ArticleRepository extends JpaRepository { + + Optional
findByArticleId(Long articleId); + List
findByArticleIdIn(Collection articleId); +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/repository/v2/SubscribedKeywordRepository.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/repository/v2/SubscribedKeywordRepository.java new file mode 100644 index 0000000..90028c1 --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/repository/v2/SubscribedKeywordRepository.java @@ -0,0 +1,23 @@ +package com.myoa.engineering.crawl.shopping.infra.repository.v2; + +import com.myoa.engineering.crawl.shopping.domain.entity.v2.SubscribedKeyword; +import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.stereotype.Repository; + +import java.util.List; + +@Repository +public interface SubscribedKeywordRepository extends JpaRepository { + + List findByCrawlTarget(CrawlTarget crawlTarget); + + + /* @Query("SELECT new com.myoa.engineering.crawl.shopping.domain.model.v2.SubscribedKeywordUserAggregatedModel(" + + " s.userId, s.keyword, s.crawlTarget) " + + " FROM SubscribedKeyword s GROUP BY s.userId ") + List findGroupByUserId(String userId);*/ + List findByUserIdAndCrawlTarget(String userId, CrawlTarget crawlTarget); + + List findByUserId(String userId); +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/scheduler/ParseEventEmitter.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/scheduler/ParseEventEmitter.java new file mode 100644 index 0000000..ae65851 --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/scheduler/ParseEventEmitter.java @@ -0,0 +1,27 @@ +package com.myoa.engineering.crawl.shopping.scheduler; + +import com.myoa.engineering.crawl.shopping.crawlhandler.CrawlHandler; +import lombok.extern.slf4j.Slf4j; +import org.springframework.scheduling.annotation.EnableScheduling; +import org.springframework.stereotype.Component; + +import java.util.List; + +@Slf4j +@Component +@EnableScheduling +public class ParseEventEmitter { + private final List crawlHandlers; + + + public ParseEventEmitter(List crawlHandlers) { + this.crawlHandlers = crawlHandlers; + } + + // @Scheduled(cron = "0 0/5 * * * ?") + public void emit() { + log.info("[emitDomesticBoard] trigger fired!"); + crawlHandlers.forEach(CrawlHandler::handle); + + } +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/AppUserQueryService.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/AppUserQueryService.java new file mode 100644 index 0000000..c2f1d01 --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/AppUserQueryService.java @@ -0,0 +1,25 @@ +package com.myoa.engineering.crawl.shopping.service; + +import com.myoa.engineering.crawl.shopping.domain.model.v2.AppUserModel; +import com.myoa.engineering.crawl.shopping.infra.repository.v2.AppUserRepository; +import org.springframework.stereotype.Service; + +import java.util.List; + +@Service +public class AppUserQueryService { + + private final AppUserRepository appUserRepository; + + public AppUserQueryService(AppUserRepository appUserRepository) { + this.appUserRepository = appUserRepository; + } + + public List findAll() { + return appUserRepository.findAll() + .stream() + .map(AppUserModel::from) + .toList(); + } + +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/ArticleCommandService.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/ArticleCommandService.java new file mode 100644 index 0000000..20aa894 --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/ArticleCommandService.java @@ -0,0 +1,81 @@ +package com.myoa.engineering.crawl.shopping.service; + +import com.myoa.engineering.crawl.shopping.domain.entity.v2.Article; +import com.myoa.engineering.crawl.shopping.domain.model.v2.ArticleModel; +import com.myoa.engineering.crawl.shopping.event.ArticleUpsertEvent; +import com.myoa.engineering.crawl.shopping.infra.repository.v2.ArticleRepository; +import jakarta.transaction.Transactional; +import org.springframework.context.ApplicationEventPublisher; +import org.springframework.stereotype.Service; + +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; + +@Service +public class ArticleCommandService { + + private final ArticleRepository articleRepository; + private final ApplicationEventPublisher applicationEventPublisher; + + public ArticleCommandService(ArticleRepository articleRepository, + ApplicationEventPublisher applicationEventPublisher) { + this.articleRepository = articleRepository; + this.applicationEventPublisher = applicationEventPublisher; + } + + + @Transactional + public void upsert(List
articles) { + Map articleMap = articles.stream() + .collect(Collectors.toMap(Article::getArticleId, e -> e)); + + List
saved = articleRepository.findByArticleIdIn(articleMap.keySet()); + List
updated = saved.stream() + .filter(e -> articleMap.containsKey(e.getArticleId())) + .map(e -> e.update(articleMap.get(e.getArticleId()))) + .peek(e -> articleMap.remove(e.getArticleId())) + .sorted(Comparator.comparing(Article::getArticleId)) + .toList(); + + List
newArticles = articleMap.values() + .stream() + .sorted(Comparator.comparing(Article::getArticleId)) + .toList(); + + articleRepository.saveAll(updated); + articleRepository.saveAll(newArticles); + + publishEvent(newArticles); + } + + private void publishEvent(List
articles) { + List articleModels = + articles.stream() + .map(transformer) + .toList(); + applicationEventPublisher.publishEvent(new ArticleUpsertEvent(articleModels)); + } + + @Transactional + public void upsert(Article article) { + Article saved = articleRepository.findByArticleId(article.getArticleId()) + .orElse(article); + articleRepository.save(saved); + } + + public static Function transformer = + article -> ArticleModel.builder() + .id(article.getId()) + .articleId(article.getArticleId()) + .crawlTarget(article.getCrawlTarget()) + .boardName(article.getBoardName()) + .articleUrl(article.getArticleUrl()) + .title(article.getTitle()) + .hit(article.getHit()) + .recommended(article.getRecommended()) + .registeredAt(article.getRegisteredAt()) + .build(); +} diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/MessageSenderService.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/MessageSenderService.java similarity index 100% rename from processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/MessageSenderService.java rename to shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/MessageSenderService.java diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/PpomppuArticleService.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/PpomppuArticleService.java similarity index 85% rename from processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/PpomppuArticleService.java rename to shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/PpomppuArticleService.java index 201e730..88b89b3 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/PpomppuArticleService.java +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/PpomppuArticleService.java @@ -1,20 +1,18 @@ -package com.myoa.engineering.crawl.ppomppu.processor.service; +package com.myoa.engineering.crawl.shopping.service; + +import com.myoa.engineering.crawl.shopping.domain.entity.v1.PpomppuArticle; +import com.myoa.engineering.crawl.shopping.domain.entity.v1.PpomppuBoardFeedStatus; +import com.myoa.engineering.crawl.shopping.infra.repository.v1.PpomppuArticleRepository; +import com.myoa.engineering.crawl.shopping.infra.repository.v1.PpomppuBoardFeedStatusRepository; +import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; import java.util.List; import java.util.Optional; import java.util.stream.Collectors; -import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; - -import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle; -import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuBoardFeedStatus; -import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository.PpomppuArticleRepository; -import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository.PpomppuBoardFeedStatusRepository; -import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName; - -import lombok.extern.slf4j.Slf4j; - @Slf4j @Service public class PpomppuArticleService { diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/PpomppuFeedService.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/PpomppuFeedService.java similarity index 100% rename from processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/PpomppuFeedService.java rename to shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/PpomppuFeedService.java diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/SubscribedKeywordCacheService.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/SubscribedKeywordCacheService.java new file mode 100644 index 0000000..a0fd644 --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/SubscribedKeywordCacheService.java @@ -0,0 +1,41 @@ +package com.myoa.engineering.crawl.shopping.service; + +import com.myoa.engineering.crawl.shopping.domain.entity.v2.SubscribedKeyword; +import com.myoa.engineering.crawl.shopping.domain.model.v2.SubscribedKeywordAggregatedModel; +import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget; +import org.springframework.cache.annotation.Cacheable; +import org.springframework.stereotype.Component; + +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +@Component +public class SubscribedKeywordCacheService { + + private final SubscribedKeywordQueryService subscribedKeywordQueryService; + + public SubscribedKeywordCacheService(SubscribedKeywordQueryService subscribedKeywordQueryService) { + this.subscribedKeywordQueryService = subscribedKeywordQueryService; + } + + @Cacheable(cacheNames = "subscribe.keywords", key = "#userId + '_' + #crawlTarget.name()") + public SubscribedKeywordAggregatedModel getSubscribedKeywordsCached(String userId, CrawlTarget crawlTarget) { + System.out.println("getSubscribedKeywordsCached"); + List keywords = subscribedKeywordQueryService.findByUserWithTarget(userId, crawlTarget) + .stream().map(SubscribedKeyword::getKeyword).toList(); + return SubscribedKeywordAggregatedModel.of(userId, crawlTarget, keywords); + } + + @Cacheable(cacheNames = "subscribe.keywords", key = "#userId") + public Map getSubscribedKeywordsCached(String userId) { + System.out.println("getSubscribedKeywordsCached"); + return subscribedKeywordQueryService.findByUser(userId) + .stream() + .collect(Collectors.groupingBy(SubscribedKeyword::getCrawlTarget, + Collectors.mapping(SubscribedKeyword::getKeyword, Collectors.toList()))) + .entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getKey, e -> SubscribedKeywordAggregatedModel.of(userId, e.getKey(), e.getValue()))); + } + +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/SubscribedKeywordQueryService.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/SubscribedKeywordQueryService.java new file mode 100644 index 0000000..dc06a58 --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/SubscribedKeywordQueryService.java @@ -0,0 +1,30 @@ +package com.myoa.engineering.crawl.shopping.service; + +import com.myoa.engineering.crawl.shopping.domain.entity.v2.SubscribedKeyword; +import com.myoa.engineering.crawl.shopping.infra.repository.v2.SubscribedKeywordRepository; +import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget; +import org.springframework.stereotype.Service; + +import java.util.List; + +@Service +public class SubscribedKeywordQueryService { + + private final SubscribedKeywordRepository subscribedKeywordRepository; + + public SubscribedKeywordQueryService(SubscribedKeywordRepository subscribedKeywordRepository) { + this.subscribedKeywordRepository = subscribedKeywordRepository; + } + + public List findAll() { + return subscribedKeywordRepository.findAll(); + } + + public List findByUserWithTarget(String userId, CrawlTarget crawlTarget) { + return subscribedKeywordRepository.findByUserIdAndCrawlTarget(userId, crawlTarget); + } + + public List findByUser(String userId) { + return subscribedKeywordRepository.findByUserId(userId); + } +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/UserNotifyService.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/UserNotifyService.java new file mode 100644 index 0000000..13486ee --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/UserNotifyService.java @@ -0,0 +1,34 @@ +package com.myoa.engineering.crawl.shopping.service; + +import com.myoa.engineering.crawl.shopping.configuration.slack.properties.SlackSecretProperties; +import com.myoa.engineering.crawl.shopping.dto.slack.SlackMessageDTO; +import com.myoa.engineering.crawl.shopping.infra.client.slack.SlackAPIClient; +import org.springframework.stereotype.Service; + +@Service +public class UserNotifyService { + + private static final String SLACK_PROPERTIES_UNIT_NAME = "shopping-crawler"; +// private static final String NOTIFY_CHANNEL_ID = "notify_shopping"; +// private static final String NOTIFY_ICON_EMOJI = ":monge_big:"; +// private static final String NOTIFY_BOT_NAME = "몽이 탈호구봇"; + + private final SlackAPIClient slackAPIClient; + private final SlackSecretProperties.SlackSecretPropertiesUnit slackSecretProperties; + + public UserNotifyService(SlackAPIClient slackAPIClient, + SlackSecretProperties slackSecretProperties) { + this.slackAPIClient = slackAPIClient; + this.slackSecretProperties = slackSecretProperties.find(SLACK_PROPERTIES_UNIT_NAME); + } + + public void notify(String message) { + SlackMessageDTO slackMessageDTO = SlackMessageDTO.builder() + .channel(slackSecretProperties.getChannel()) + .text(message) + .iconEmoji(slackSecretProperties.getIconEmoji()) + .username(slackSecretProperties.getUsername()) + .build(); + slackAPIClient.sendMessage(slackMessageDTO, slackSecretProperties.getToken()); + } +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/util/AhoCorasickUtils.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/util/AhoCorasickUtils.java new file mode 100644 index 0000000..1066198 --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/util/AhoCorasickUtils.java @@ -0,0 +1,18 @@ +package com.myoa.engineering.crawl.shopping.util; + +import org.ahocorasick.trie.Trie; + +import java.util.List; + +public final class AhoCorasickUtils { + + private AhoCorasickUtils() { + } + + public static Trie generateTrie(List keywords) { + return Trie.builder() + .addKeywords(keywords) + .ignoreCase() + .build(); + } +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/util/DateTimeUtils.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/util/DateTimeUtils.java new file mode 100644 index 0000000..3ec9d5a --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/util/DateTimeUtils.java @@ -0,0 +1,27 @@ +package com.myoa.engineering.crawl.shopping.util; + +import java.time.*; +import java.time.format.DateTimeFormatter; + +public final class DateTimeUtils { + + private static final DateTimeFormatter FORMATTER_HHMMss = DateTimeFormatter.ofPattern("HH:mm:ss"); + private static final ZoneId ZONE_ASIA_SEOUL = ZoneId.of("Asia/Seoul"); + + private DateTimeUtils() { + } + + public static ZonedDateTime parse(String HHMMss) { + try { + LocalTime time = LocalTime.parse(HHMMss, FORMATTER_HHMMss); + LocalDateTime dateTime = LocalDateTime.of(LocalDate.now(), time); + if (dateTime.isAfter(LocalDateTime.now())) { + dateTime = dateTime.minusDays(1); + } + return dateTime.atZone(ZONE_ASIA_SEOUL); + } catch (Exception e) { + return null; + } + } + +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/util/NumberUtils.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/util/NumberUtils.java new file mode 100644 index 0000000..6efae60 --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/util/NumberUtils.java @@ -0,0 +1,32 @@ +package com.myoa.engineering.crawl.shopping.util; + +public final class NumberUtils { + + private NumberUtils() { + } + + public static Integer parseInt(String value) { + return parseInt(value, null); + } + + public static Integer parseInt(String value, Integer defaultValue) { + try { + return Integer.parseInt(value); + } catch (NumberFormatException e) { + return defaultValue; + } + } + + public static Long parseLong(String value) { + return parseLong(value, null); + } + + public static Long parseLong(String value, Long defaultValue) { + try { + return Long.parseLong(value); + } catch (NumberFormatException e) { + return defaultValue; + } + } + +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/util/ObjectMapperFactory.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/util/ObjectMapperFactory.java new file mode 100644 index 0000000..0785ccb --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/util/ObjectMapperFactory.java @@ -0,0 +1,62 @@ +package com.myoa.engineering.crawl.shopping.util; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.core.json.JsonReadFeature; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.databind.json.JsonMapper; +import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import com.fasterxml.jackson.module.paramnames.ParameterNamesModule; + +public final class ObjectMapperFactory { + + public static final ObjectMapper DEFAULT_MAPPER; + public static final ObjectMapper REDIS_MAPPER; + @Deprecated + public static final ObjectMapper LOGGING_MAPPER; + + static { + DEFAULT_MAPPER = initDefaultMapper(); + REDIS_MAPPER = initRedisMapper(); + LOGGING_MAPPER = JsonMapper.builder() + .serializationInclusion(JsonInclude.Include.NON_EMPTY) + .build(); + } + + private ObjectMapperFactory() { + } + + private static ObjectMapper initDefaultMapper() { + return JsonMapper.builder() + .configure(JsonReadFeature.ALLOW_UNQUOTED_FIELD_NAMES.mappedFeature(), true) + .configure(JsonReadFeature.ALLOW_UNESCAPED_CONTROL_CHARS.mappedFeature(), true) + .disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES) + .disable(SerializationFeature.FAIL_ON_EMPTY_BEANS) + .disable(SerializationFeature.FAIL_ON_UNWRAPPED_TYPE_IDENTIFIERS) + .enable(DeserializationFeature.READ_UNKNOWN_ENUM_VALUES_AS_NULL) + .disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS) + .serializationInclusion(JsonInclude.Include.NON_NULL) + .build() + .registerModule(new ParameterNamesModule()) + .registerModule(new Jdk8Module()) + .registerModule(new JavaTimeModule()); + } + + private static ObjectMapper initRedisMapper() { + return JsonMapper.builder() + .configure(JsonReadFeature.ALLOW_UNQUOTED_FIELD_NAMES.mappedFeature(), true) + .configure(JsonReadFeature.ALLOW_UNESCAPED_CONTROL_CHARS.mappedFeature(), true) + .disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES) + .disable(SerializationFeature.FAIL_ON_EMPTY_BEANS) + .disable(SerializationFeature.FAIL_ON_UNWRAPPED_TYPE_IDENTIFIERS) + .enable(DeserializationFeature.READ_UNKNOWN_ENUM_VALUES_AS_NULL) + .disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS) + .serializationInclusion(JsonInclude.Include.NON_NULL) + .build() + .registerModule(new ParameterNamesModule()) + .registerModule(new Jdk8Module()) + .registerModule(new JavaTimeModule()); + } +} diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/util/ObjectUtil.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/util/ObjectUtils.java similarity index 91% rename from processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/util/ObjectUtil.java rename to shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/util/ObjectUtils.java index 2f03f12..765100b 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/util/ObjectUtil.java +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/util/ObjectUtils.java @@ -3,7 +3,7 @@ * LINE Corporation PROPRIETARY/CONFIDENTIAL. Use is subject to license terms. */ -package com.myoa.engineering.crawl.ppomppu.processor.util; +package com.myoa.engineering.crawl.shopping.util; import java.util.Arrays; import java.util.Collection; @@ -14,9 +14,9 @@ import java.util.Collection; * @author Shin Woo-jin (lp12254@linecorp.com) * @since 2019-10-28 */ -public final class ObjectUtil { +public final class ObjectUtils { - private ObjectUtil() { + private ObjectUtils() { } /** @@ -58,7 +58,7 @@ public final class ObjectUtil { * @return Is there objects array has null? */ public static boolean hasNullObject(Object... args) { - return Arrays.stream(args).anyMatch(ObjectUtil::isNullObject); + return Arrays.stream(args).anyMatch(ObjectUtils::isNullObject); } /** @@ -72,7 +72,7 @@ public final class ObjectUtil { * @return Is there objects array has null? */ public static boolean hasAllObject(Object... args) { - return Arrays.stream(args).noneMatch(ObjectUtil::isNullObject); + return Arrays.stream(args).noneMatch(ObjectUtils::isNullObject); } /** @@ -86,7 +86,7 @@ public final class ObjectUtil { * @return Is there null all of given objects? */ public static boolean hasAllNullObjects(final Object... args) { - return Arrays.stream(args).allMatch(ObjectUtil::isNullObject); + return Arrays.stream(args).allMatch(ObjectUtils::isNullObject); } /** diff --git a/processor/src/main/resources/application-development.yml b/shopping-crawler/src/main/resources/application-development.yml similarity index 100% rename from processor/src/main/resources/application-development.yml rename to shopping-crawler/src/main/resources/application-development.yml diff --git a/sender/src/main/resources/application-local.yml b/shopping-crawler/src/main/resources/application-local.yml similarity index 54% rename from sender/src/main/resources/application-local.yml rename to shopping-crawler/src/main/resources/application-local.yml index 5c6bfbb..67afb03 100644 --- a/sender/src/main/resources/application-local.yml +++ b/shopping-crawler/src/main/resources/application-local.yml @@ -3,8 +3,16 @@ spring: activate: on-profile: local import: - - "configserver:http://localhost:20085" - # import: optional:configserver:http://localhost:11080 # can be start up even config server was not found. + - classpath:/datasource/local.yml + - classpath:/slack/local.yml server: - port: 20082 + port: 20080 + + # import: optional:configserver:http://localhost:11080 # can be start up even config server was not found. + +feign: + client: + config: + default: + loggerLevel: FULL \ No newline at end of file diff --git a/processor/src/main/resources/application-production.yml b/shopping-crawler/src/main/resources/application-production.yml similarity index 100% rename from processor/src/main/resources/application-production.yml rename to shopping-crawler/src/main/resources/application-production.yml diff --git a/processor/src/main/resources/application.yml b/shopping-crawler/src/main/resources/application.yml similarity index 88% rename from processor/src/main/resources/application.yml rename to shopping-crawler/src/main/resources/application.yml index ba38e53..bfd63e5 100644 --- a/processor/src/main/resources/application.yml +++ b/shopping-crawler/src/main/resources/application.yml @@ -1,6 +1,6 @@ spring: application: - name: ppn-processor + name: crawler-shopping main: allow-bean-definition-overriding: true profiles: @@ -11,6 +11,9 @@ spring: production: "production,datasource-production,webclient-production" freemarker: enabled: false + cloud: + config: + enabled: false server: port: 20080 diff --git a/processor/src/main/resources/logback-spring.xml b/shopping-crawler/src/main/resources/logback-spring.xml similarity index 100% rename from processor/src/main/resources/logback-spring.xml rename to shopping-crawler/src/main/resources/logback-spring.xml diff --git a/processor/src/main/resources/logback/component/logback-file.xml b/shopping-crawler/src/main/resources/logback/component/logback-file.xml similarity index 100% rename from processor/src/main/resources/logback/component/logback-file.xml rename to shopping-crawler/src/main/resources/logback/component/logback-file.xml diff --git a/processor/src/main/resources/logback/logback-development.xml b/shopping-crawler/src/main/resources/logback/logback-development.xml similarity index 100% rename from processor/src/main/resources/logback/logback-development.xml rename to shopping-crawler/src/main/resources/logback/logback-development.xml diff --git a/processor/src/main/resources/logback/logback-production.xml b/shopping-crawler/src/main/resources/logback/logback-production.xml similarity index 100% rename from processor/src/main/resources/logback/logback-production.xml rename to shopping-crawler/src/main/resources/logback/logback-production.xml diff --git a/shopping-crawler/src/test/java/com/myoa/engineering/crawl/shopping/crawlhandler/parser/PpomppuArticleParserV2Test.java b/shopping-crawler/src/test/java/com/myoa/engineering/crawl/shopping/crawlhandler/parser/PpomppuArticleParserV2Test.java new file mode 100644 index 0000000..c39869e --- /dev/null +++ b/shopping-crawler/src/test/java/com/myoa/engineering/crawl/shopping/crawlhandler/parser/PpomppuArticleParserV2Test.java @@ -0,0 +1,118 @@ +package com.myoa.engineering.crawl.shopping.crawlhandler.parser; + +import com.myoa.engineering.crawl.shopping.domain.entity.v1.PpomppuArticle; +import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName; +import com.myoa.engineering.crawl.shopping.util.DateTimeUtils; +import com.myoa.engineering.crawl.shopping.util.NumberUtils; +import com.myoa.engineering.crawl.shopping.util.TestDataUtils; +import io.micrometer.core.instrument.util.StringUtils; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; +import org.junit.jupiter.api.Test; + +import java.time.ZonedDateTime; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +class PpomppuArticleParserV2Test { + + @Test + void test1() { + String data = TestDataUtils.fileToString("testdata/zboard/file1.html"); + Document document = Jsoup.parse(data); + Elements trList = document.getElementById("revolution_main_table").getElementsByTag("tr"); + + List articles = trList.stream() + .filter(this::isRealArticle) + .map(this::parse) + .toList(); + System.out.println(articles.size()); + System.out.println(articles.get(0)); + System.out.println(articles.get(1)); + } + + private boolean isRealArticle(Element tr) { + Elements tdList = tr.getElementsByTag("td"); + if (tdList.size() != 6) { + return false; + } + + if (!hasOnlyNumeric(tdList.get(0))) { + return false; + } + + return true; + } + + + Pattern pattern_numeric = Pattern.compile("\\d+"); + + private boolean hasOnlyNumeric(Element td) { + return pattern_numeric.matcher(td.text()).matches(); + } + + public PpomppuArticle parse(Element tr) { + Elements tdList = tr.getElementsByTag("td"); + Long articleId = Long.parseLong(tdList.get(0).text()); + + String title = tdList.get(2).text(); + String articleUrl = parseArticleUrl(tdList.get(2).getElementsByTag("a").attr("href")); + PpomppuBoardName boardName = parseBoardName(title); + Integer recommended = parseRecommended(tdList.get(4)); + Integer hit = NumberUtils.parseInt(tdList.get(5).text(), 0); + + + ZonedDateTime registeredAt = DateTimeUtils.parse(tdList.get(3).text()); + + return PpomppuArticle.builder() + .articleId(articleId) + .title(title) + .boardName(boardName) + .articleUrl(articleUrl) + .recommended(recommended) + .hit(hit) + .registeredAt(registeredAt) + .build(); + } + + public Long parseArticleId(Element td) { + return Long.parseLong(td.text().trim()); + } + + public static Integer parseRecommended(Element td) { + final String voteString = td.text(); + if (StringUtils.isEmpty(voteString)) { + return null; + } + + final int voteUp = Integer.parseInt(td.text().split(" - ")[0]); + final int voteDown = Integer.parseInt(td.text().split(" - ")[1]); + int recommended = voteUp - voteDown; + return recommended; + } + + public static String parseArticleUrl(String data) { + return PpomppuBoardName.ofViewPageUrl(data); + } + + + Pattern patternBoardName = Pattern.compile("\\[(.+?)\\]"); + + public PpomppuBoardName parseBoardName(String fullTitle) { + Matcher matcher = patternBoardName.matcher(fullTitle); + String lastMatched = null; + while (matcher.find()) { + lastMatched = matcher.group(1); + } + return PpomppuBoardName.ofBoardName(lastMatched, true); + } + + @Test + void test2() { + PpomppuBoardName boardName = parseBoardName("[자사몰]푸마 메쉬 폼스트라이프 [에디션] 5종 [세트] (18,220원/무료)6 [의류/잡화]"); + System.out.println(boardName); + } +} \ No newline at end of file diff --git a/shopping-crawler/src/test/java/com/myoa/engineering/crawl/shopping/event/handler/ArticleUpsertEventListenerTest.java b/shopping-crawler/src/test/java/com/myoa/engineering/crawl/shopping/event/handler/ArticleUpsertEventListenerTest.java new file mode 100644 index 0000000..375fe80 --- /dev/null +++ b/shopping-crawler/src/test/java/com/myoa/engineering/crawl/shopping/event/handler/ArticleUpsertEventListenerTest.java @@ -0,0 +1,81 @@ +package com.myoa.engineering.crawl.shopping.event.handler; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.ahocorasick.trie.Emit; +import org.ahocorasick.trie.Trie; +import org.jeasy.random.EasyRandom; +import org.jeasy.random.EasyRandomParameters; +import org.junit.jupiter.api.Test; + +import java.util.Collection; +import java.util.List; + +class ArticleUpsertEventListenerTest { + + @Test + public void test1() { + + List baseData = List.of( + "[공홈]베베숲 시그니처 위드 블루 20팩 (25,990원/무료)2 [기타]", + "[현대H몰]10주년 스페셜 에디션 봉고데기 40mm 세트 (67,640원/무료)3 [가전/가구]", + "[SSG]필립스 면도기 칫솔 기획전( 437,000원~/무료)1 [기타]", + "(티몬)제스프리 골드키위 중대과 1.8kg내외 (카페,토페 14,823원/무배)3 [식품/건강]", + "[공식몰]연세생활건강 당뇨영양식 24팩 (33,500원/유배)1 [식품/건강]", + "[인터파크]크리스탈라이트 아이스티 에이드 온더고 60개입 멀티팩(17,800원/무료)6 [기타]", + "[인팍쇼핑]샤카웨어 액티브 맥스 헤비웨이트 10종 택 1 (7,350원/무료)7 [의류/잡화]" + ); + + Trie trie = Trie.builder() + .ignoreCase() + .addKeyword("블루") + .addKeyword("봉고") + .build(); + + List> list = baseData.stream() + .map(trie::parseText) + .toList(); + + System.out.println(list); + } + + @Test + void test2() throws InterruptedException { + EasyRandomParameters paramsKeyword = new EasyRandomParameters() + .stringLengthRange(2, 10); + EasyRandom GEN_KEYWORD = new EasyRandom(paramsKeyword); + + EasyRandomParameters paramsSentence = new EasyRandomParameters() + .stringLengthRange(100, 1000); + EasyRandom GEN_SENTENCE = new EasyRandom(paramsSentence); + + Trie.TrieBuilder trieBuilder = Trie.builder().ignoreOverlaps(); + + GEN_KEYWORD.objects(TestA.class, 1000) + .map(TestA::getFieldA) + .forEach(trieBuilder::addKeyword); + Trie trie = trieBuilder.build(); + + List> listOfEmits = GEN_SENTENCE.objects(TestA.class, 2000) + .map(TestA::getFieldA) + .map(trie::parseText) + .toList(); + + for(Collection emits : listOfEmits) { + System.out.println(emits); + System.out.println("--------------------------"); + } + + } + + + @Data + @AllArgsConstructor + @NoArgsConstructor + public class TestA { + private String fieldA; + + } + +} \ No newline at end of file diff --git a/shopping-crawler/src/test/java/com/myoa/engineering/crawl/shopping/util/TestDataUtils.java b/shopping-crawler/src/test/java/com/myoa/engineering/crawl/shopping/util/TestDataUtils.java new file mode 100644 index 0000000..e5ff1fe --- /dev/null +++ b/shopping-crawler/src/test/java/com/myoa/engineering/crawl/shopping/util/TestDataUtils.java @@ -0,0 +1,62 @@ +package com.myoa.engineering.crawl.shopping.util; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.springframework.lang.NonNull; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Scanner; +import java.util.stream.Collectors; + +public final class TestDataUtils { + + private static final ObjectMapper OBJECT_MAPPER; + + static { + OBJECT_MAPPER = ObjectMapperFactory.DEFAULT_MAPPER; + } + + private TestDataUtils() { + } + + public static T inputStreamToObject(@NonNull String resourcePath, + @NonNull Class clazz) throws IOException { + return OBJECT_MAPPER.readValue(fileToString(resourcePath), clazz); + } + + public static List inputStreamsToList(@NonNull Class clazz, + @NonNull String... resourcePath) throws IOException { + return inputStreamsToList(Arrays.asList(resourcePath), clazz); + } + + public static List inputStreamsToList(@NonNull List resourcePath, + @NonNull Class clazz) throws IOException { + return resourcePath.stream() + .map(e -> { + try { + return OBJECT_MAPPER.readValue(fileToString(e), clazz); + } catch (JsonProcessingException jsonProcessingException) { + jsonProcessingException.printStackTrace(); + return null; + } + }) + .collect(Collectors.toList()); + } + + public static List inputStreamToList(@NonNull String resourcePath, + @NonNull Class clazz) throws IOException { + return (List) OBJECT_MAPPER.readValue( + fileToString(resourcePath), + OBJECT_MAPPER.getTypeFactory().constructCollectionType(List.class, clazz)); + } + + public static String fileToString(@NonNull String resourcePath) { + + final ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); + final Scanner s = new Scanner(classLoader.getResourceAsStream(resourcePath)).useDelimiter("\\A"); + return s.hasNext() ? s.next() : ""; + } + +} diff --git a/processor/src/test/resources/logback-development.xml b/shopping-crawler/src/test/resources/logback-development.xml similarity index 100% rename from processor/src/test/resources/logback-development.xml rename to shopping-crawler/src/test/resources/logback-development.xml diff --git a/processor/src/test/resources/logback-spring.xml b/shopping-crawler/src/test/resources/logback-spring.xml similarity index 100% rename from processor/src/test/resources/logback-spring.xml rename to shopping-crawler/src/test/resources/logback-spring.xml diff --git a/shopping-crawler/src/test/resources/testdata/zboard/file1.html b/shopping-crawler/src/test/resources/testdata/zboard/file1.html new file mode 100644 index 0000000..2de3e57 --- /dev/null +++ b/shopping-crawler/src/test/resources/testdata/zboard/file1.html @@ -0,0 +1,1100 @@ + + + + + + + + + + + + + + + + 뽐뿌 - 뽐뿌게시판 + + + + + + + + + + + + + + + + + + + + +
+
+
+
+ +
+
+
+ +
+
+
+
 
+
+
+
+ 뽐뿌 + 뽐뿌 +
+ +
+
+
+
+ + +
+ +
+ + +
+
+ +
+ +
+ + +
+ + + + +
+ 로그인 + 회원가입|아이디비번찾기 +
+
+
+ +
+ +
+

뽐뿌게시판 입니다.

+ +
+
+ 가성비 좋은 상품과 쇼핑정보를 공유하는 공간입니다. [뽐뿌게시판 이용규칙] +
+ + + + + + + + + + + + + + + + + + + +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
   번호     글쓴이  제목 등록일 추천    조회   
공지관리자3[필독] 제목에는 특수 문자 사용을 절대 금지합니다.- 2010년 2월 24일0 4324677
알림관리자아이콘알리익스프레스 상품은 [알리뽐뿌]에 공유해 주세요!
알림관리자아이콘[키워드 알림] 관심 키워드 등록하고, 앱push 알림 받자!
537951매력적인뽐뿌 + +
+
+ [지마켓]천연 향수 비누 사봉파퓨메 135g 3개+사은품 (9,950원/무료)1 +
+ [기타] +
1069
537950123aaa + +
+
+ [지마켓]순면 국내생산 무지티3장 (유클12,410원/무배) +
+ [의류/잡화] +
1508
537949유유자적루 + +
+
+ [네이버-미루몰] 벨루아체 올인원 포맨 4종 (29,900원/무료)5 +
+ [화장품] +
1201
537947illilliiill + +
+
+ [티몬]역대가 핫딜! 슈퍼대디 세이프 77평량 아기물티슈 (17,991원/무배)74  +
+ [기타] +
9 - 0 14917
5379461회제공량 + +
+
+ [CGV씨네샵]타카라토미 버즈라이트이어 49,000 / 3,0006 +
+ [기타] +
9522
537945가을말차라떼 + +
+
+ [위메프]샤오미 스마트 선풍기 7세대 1X 업그레이드/Type-C 충전 포트 (43,160원/무배)16 +
+ [기타] +
1 - 0 17666
537944거북반 + +
+
+ [무신사]FCMM 풋볼 프로 스트레치 윈드브레이커 (32,364원/무료)3 +
+ [의류/잡화] +
13039
537942jimin1120 + +
+
+ 북유럽 스타일 양손 스프볼 도자기 스프그릇 소스접시 짱구그릇 라면그릇 만두접시 특가전(8,400원~/무료)23 +
+ [기타] +
2 - 0 5767
537941너구리사랑 + +
+
+ [롯데온]제스프리 골드키위, 토레타, 가자미밥상, 쿡반 새우볶음밥(다양/무료)20 +
+ [식품/건강] +
16 - 0 10050
537940깡패건달 + +
+
+ [자사몰]푸마 메쉬 폼스트라이프 에디션 5종 세트 (18,220원/무료)6 +
+ [의류/잡화] +
4 - 0 7273
537939심의켄 + +
+
+ [위메프]오리온 비쵸비 초코비스킷5pX8개 외 치즈톡/감자톡/오징어톡,도도한 나쵸(13,220원/무료)10 +
+ [식품/건강] +
8 - 0 7724
537938벌꿀오소리는강해 + +
+
+ [네이버]추억의 오란다 250g x2봉(6,200원/무배)9 +
+ [식품/건강] +
5892
537937매카aa + +
+
+ [G마켓]휴대폰 젤리케이스1+1 갤럭시,아이폰 (3,610원/무배)5 +
+ [기타] +
5186
537936나무007 + +
+
+ [CJ온스타일]동원 지중해 황도 410g 12캔(16,110/무료)44 +
+ [식품/건강] +
15 - 0 13237
537935IilliillIi + +
+
+ [네이버]맥주효모 비오틴 아누카사과 프리미엄 1박스 (1개월분) (34,900원/무료)2  +
+ [식품/건강] +
3597
537934최군어뭉 + +
+
+ [위메프]델리만쥬 붕어빵/핫도그 600g 2봉 (8,700/무료)15 +
+ [식품/건강] +
6 - 0 6264
537933무념무상해 + +
+
+ [인터파크]산리오 미니 말랑스티커/케이크하우스 외 다수 (2,970원부터/3개 이상 주문시 무료) +
+ [육아] +
1 - 0 6198
537931p1ano + +
+
+ [위메프]비오틴 울트라 4개월분 (4,900원/무배)8  +
+ [식품/건강] +
3 - 0 6065
537930킹갓제네럴MZ + +
+
+ [위메프]아스트로글라이드젤 148ml 1+1 (27,470원/무료배송)44 +
+ [기타] +
1 - 0 16634
537929꿉지 + +
+
+ [옥션]다농바이오 가무치소주 43도 375ml(32,900원/무료)3 +
+ [식품/건강] +
1 - 0 5952
쇼핑뽐뿌 아이콘아스라슬 + +
+
+ [카카오톡 스토어] 캘빈클라인 코튼 반팔티 15,900원 외 폴로반팔티, 반바지, PK카라티, 원피스, (무배,무교,무반) + 256 +
+
+ 		56 - 05325
쇼핑뽐뿌 아이콘LG전자쿠팡공식대행... + +
+
+ [쿠팡] LG 그램 주말 특가 찬스! 와우 즉시할인+농협카드 추가 할인+내일 배송 도착 + 246 +
+
+ 		57 - 03128
+
+

+ 쇼핑포럼뽐뿌상담실 아이콘더 많은 쇼핑 정보와 구매 전 자유로운 쇼핑 이야기는 쇼핑포럼에서! +

+
+
핫딜 아이콘 + 쿠팡 + + +
+
+ 나이키 에브리데이 쿠션크루 트레이닝삭스 장목양말 1세트 3켤레입 (9,490원) +
+
+ 		7
쇼핑포럼 아이콘 + ll427ll + + +
+
+ [알리]펩시제로 라임향 355ml 24캔(13,582원/무료) +
+ [식품/건강]
+ 		907
쇼핑포럼 아이콘 + HBAF + + +
+
+  국산 갈치 1마리 39800원 +
+ [식품/건강]
+ 		1 - 010894
쇼핑포럼 아이콘 + 시뻘건맛 + + +
+
+  남자 집에서 입을 반바지 통풍잘되는거 있을까요? +
+ [의류/잡화]
+ 		10935
+
+
+ +
+
+
+
+
+
+ + + + +
+ +
+
+

+ 실시간 질렀어요 +
    +
  1. 1
  2. 2
+

+ +
    +
  • +
    • +
  • +
    • +
+
+ +
+

+ 최근 내가 본 상품 +
    +
+

+
    +
  • 최근 내가 본 상품이 없습니다.
+
+
+ + + + + + +
+ +  -새로고침  -다음페이지   +
+ + + + + + + +
+ + 1 2 3 4 5 6 7 8 9 10 다음 +
+ + + + + + + + + + + + +
+ + + + + +
+ + + + + + + + + +
+
+ + + + +
+ +
+ + +
+ + + + + + + +
+
+ + + +
+ +
+ + 사이트소개 + 운영참여 + + + 게시중단요청 + + 이용약관 + 개인정보처리방침 + 청소년보호정책 + 불법촬영물등 신고 + 광고/제휴 + 메일문의 + + + 뽐뿌채용 + + 다크모드 +
+ + + + + +
+
+
+

뽐뿌는 쇼핑 정보 커뮤니티입니다. 상품 주문, 배송 및 환불의 의무와 책임은 각 판매 업체에 있습니다.

+

(주)뽐뿌커뮤니케이션 / 대표이사 : 고재성 / 사업자등록번호 : 117-81-66008 / 통신판매업신고번호 : 제2015-서울영등포-1441호

+

주소 : 서울특별시 영등포구 당산로 41길 당산 SK V1센터 W동 1411호 + / 팩스 : 02-6280-7029 / 사업자회원 FAQ

+ +
+ + Copyright 2005-2024 ppomppu. All rights reserved +
+ +
+
+ + + + + + + + + + + + + + + + + diff --git a/support/build.gradle b/support/build.gradle index 81d2bcc..53479a9 100644 --- a/support/build.gradle +++ b/support/build.gradle @@ -3,7 +3,7 @@ dependencies { // https://projectreactor.io/docs/core/release/reference/#debug-activate annotationProcessor 'org.projectlombok:lombok' - implementation 'org.springframework.boot:spring-boot-starter-webflux' + implementation 'org.springframework.boot:spring-boot-starter-web' } diff --git a/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/webclient/PpomppuNotifierWebClientConfiguration.java b/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/webclient/PpomppuNotifierWebClientConfiguration.java deleted file mode 100644 index b907331..0000000 --- a/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/webclient/PpomppuNotifierWebClientConfiguration.java +++ /dev/null @@ -1,19 +0,0 @@ -package com.myoa.engineering.crawl.ppomppu.support.webclient; - -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.context.annotation.ComponentScan; -import org.springframework.context.annotation.Configuration; - -/** - * PpomppuNotifierWebClientConfiguration - * @author Shin Woo-jin (woo-jin.shin@linecorp.com) - * @since 2021-11-18 - * - */ -@Configuration -@ConditionalOnProperty(value = "webclient.init", havingValue = "true") -@ComponentScan(basePackageClasses = WebClientBaseScan.class) -public class PpomppuNotifierWebClientConfiguration { - - -} diff --git a/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/webclient/WebClientBaseScan.java b/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/webclient/WebClientBaseScan.java deleted file mode 100644 index 00acb82..0000000 --- a/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/webclient/WebClientBaseScan.java +++ /dev/null @@ -1,4 +0,0 @@ -package com.myoa.engineering.crawl.ppomppu.support.webclient; - -public interface WebClientBaseScan { -} diff --git a/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/dto/APIResponse.java b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/dto/APIResponse.java similarity index 96% rename from support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/dto/APIResponse.java rename to support/src/main/java/com/myoa/engineering/crawl/shopping/support/dto/APIResponse.java index 94df264..3c20af2 100644 --- a/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/dto/APIResponse.java +++ b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/dto/APIResponse.java @@ -1,4 +1,4 @@ -package com.myoa.engineering.crawl.ppomppu.support.dto; +package com.myoa.engineering.crawl.shopping.support.dto; import java.io.Serializable; import java.util.Map; diff --git a/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/dto/APIResponseError.java b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/dto/APIResponseError.java similarity index 96% rename from support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/dto/APIResponseError.java rename to support/src/main/java/com/myoa/engineering/crawl/shopping/support/dto/APIResponseError.java index 38801f9..a73502f 100644 --- a/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/dto/APIResponseError.java +++ b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/dto/APIResponseError.java @@ -1,4 +1,4 @@ -package com.myoa.engineering.crawl.ppomppu.support.dto; +package com.myoa.engineering.crawl.shopping.support.dto; import java.io.Serializable; import java.util.HashMap; diff --git a/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/dto/BlockMessageDTO.java b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/dto/BlockMessageDTO.java similarity index 100% rename from support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/dto/BlockMessageDTO.java rename to support/src/main/java/com/myoa/engineering/crawl/shopping/support/dto/BlockMessageDTO.java diff --git a/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/dto/SimpleMessageDTO.java b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/dto/SimpleMessageDTO.java similarity index 93% rename from support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/dto/SimpleMessageDTO.java rename to support/src/main/java/com/myoa/engineering/crawl/shopping/support/dto/SimpleMessageDTO.java index 1a2c026..033e97e 100644 --- a/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/dto/SimpleMessageDTO.java +++ b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/dto/SimpleMessageDTO.java @@ -1,4 +1,4 @@ - package com.myoa.engineering.crawl.ppomppu.support.dto; + package com.myoa.engineering.crawl.shopping.support.dto; import java.io.Serializable; import java.time.Instant; diff --git a/support/src/main/java/com/myoa/engineering/crawl/shopping/support/dto/constant/CrawlTarget.java b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/dto/constant/CrawlTarget.java new file mode 100644 index 0000000..11ba3e5 --- /dev/null +++ b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/dto/constant/CrawlTarget.java @@ -0,0 +1,13 @@ +package com.myoa.engineering.crawl.shopping.support.dto.constant; + +import lombok.AllArgsConstructor; +import lombok.Getter; + +@Getter +@AllArgsConstructor +public enum CrawlTarget { + PPOMPPU_DOMESTIC, + PPOMPPU_OVERSEA, + FMKOREA, + ; +} diff --git a/support/src/main/java/com/myoa/engineering/crawl/shopping/support/dto/constant/PpomppuBoardName.java b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/dto/constant/PpomppuBoardName.java new file mode 100644 index 0000000..cd6ac23 --- /dev/null +++ b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/dto/constant/PpomppuBoardName.java @@ -0,0 +1,62 @@ +package com.myoa.engineering.crawl.shopping.support.dto.constant; + +import lombok.AllArgsConstructor; +import lombok.Getter; + +import java.util.Arrays; + +/** + * PpomppuBoardName + * + * @author Shin Woo-jin (woo-jin.shin@linecorp.com) + * @since 2021-09-05 + */ +@Getter +@AllArgsConstructor +public enum PpomppuBoardName { + UNKNOWN("", "UNKNOWN", false, false), + PPOMPPU_DOMESTIC_ALL("id=ppomppu", "전체", false, true), + PPOMPPU_DOMESTIC_ETC("id=ppomppu&category=1", "기타", true, true), + PPOMPPU_DOMESTIC_COMPUTER("id=ppomppu&category=4", "컴퓨터", true, true), + PPOMPPU_DOMESTIC_DIGITAL("id=ppomppu&category=5", "디지털", true, true), + PPOMPPU_DOMESTIC_FOOD("id=ppomppu&category=6", "식품/건강", true, true), + PPOMPPU_DOMESTIC_BOOK("id=ppomppu&category=8", "서적", true, true), + PPOMPPU_DOMESTIC_APPLIANCES("id=ppomppu&category=9", "가전/가구", true, true), + PPOMPPU_DOMESTIC_PARENTING("id=ppomppu&category=10", "육아", true, true), + PPOMPPU_DOMESTIC_GIFTCARD("id=ppomppu&category=11", "상품권", true, true), + PPOMPPU_DOMESTIC_CLOTHES("id=ppomppu&category=12", "의류/잡화", true, true), + PPOMPPU_DOMESTIC_COSMETIC("id=ppomppu&category=13", "화장품", true, true), + PPOMPPU_DOMESTIC_OUTDOOR("id=ppomppu&category=15", "등산/캠핑", true, true), + PPOMPPU_OVERSEA_ALL("id=ppomppu4", "전체", false, false), + PPOMPPU_OVERSEA_ETC("id=ppomppu4&category=1", "기타", true, false), + PPOMPPU_OVERSEA_APPLIANCES("id=ppomppu4&category=7", "가전", true, false), + PPOMPPU_OVERSEA_TVAV("id=ppomppu4&category=8", "TV/영상", true, false), + PPOMPPU_OVERSEA_COMPUTER("id=ppomppu4&category=3", "컴퓨터", true, false), + PPOMPPU_OVERSEA_DIGITAL("id=ppomppu4&category=4", "디지털", true, false), + PPOMPPU_OVERSEA_MOBILEACCESSORY("id=ppomppu4&category=9", "액세서리", false, true), + PPOMPPU_OVERSEA_CLOTHES("id=ppomppu4&category=5", "의류/잡화", true, false), + PPOMPPU_OVERSEA_WATCH("id=ppomppu4&category=2", "시계", true, false), + PPOMPPU_OVERSEA_SHOES("id=ppomppu4&category=11", "신발", true, false), + PPOMPPU_OVERSEA_FOOD("id=ppomppu4&category=10", "식품/건강", true, false), + PPOMPPU_OVERSEA_PARENTING("id=ppomppu4&category=6", "육아", true, false), + ; + + private String resourcePath; + private String menuName; + private boolean crawlWithDefaultTimer; + private boolean isDomestic; + + public static final String PPOMPPU_URL = "https://www.ppomppu.co.kr"; + + public static String ofViewPageUrl(String articleUrl) { + return PPOMPPU_URL + "/zboard/" + articleUrl; + } + + public static PpomppuBoardName ofBoardName(String boardName, boolean isDomestic) { + return Arrays.stream(values()) + .filter(e -> e.getMenuName().equals(boardName) && e.isDomestic() == isDomestic) + .findFirst() + .orElse(UNKNOWN); + } + +} diff --git a/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/util/DateUtil.java b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/util/DateUtil.java similarity index 100% rename from support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/util/DateUtil.java rename to support/src/main/java/com/myoa/engineering/crawl/shopping/support/util/DateUtil.java diff --git a/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/util/NumberUtil.java b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/util/NumberUtil.java similarity index 52% rename from support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/util/NumberUtil.java rename to support/src/main/java/com/myoa/engineering/crawl/shopping/support/util/NumberUtil.java index a6a0031..1b36154 100644 --- a/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/util/NumberUtil.java +++ b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/util/NumberUtil.java @@ -1,4 +1,4 @@ -package com.myoa.engineering.crawl.ppomppu.support.util; +package com.myoa.engineering.crawl.shopping.support.util; public final class NumberUtil { diff --git a/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/util/ObjectMapperFactory.java b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/util/ObjectMapperFactory.java similarity index 98% rename from support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/util/ObjectMapperFactory.java rename to support/src/main/java/com/myoa/engineering/crawl/shopping/support/util/ObjectMapperFactory.java index df57df1..f1404d0 100644 --- a/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/util/ObjectMapperFactory.java +++ b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/util/ObjectMapperFactory.java @@ -1,4 +1,4 @@ -package com.myoa.engineering.crawl.ppomppu.support.util; +package com.myoa.engineering.crawl.shopping.support.util; import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.core.JsonParser.Feature; diff --git a/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/util/ObjectUtil.java b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/util/ObjectUtil.java similarity index 79% rename from support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/util/ObjectUtil.java rename to support/src/main/java/com/myoa/engineering/crawl/shopping/support/util/ObjectUtil.java index e2d9863..49322ac 100644 --- a/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/util/ObjectUtil.java +++ b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/util/ObjectUtil.java @@ -1,4 +1,4 @@ -package com.myoa.engineering.crawl.ppomppu.support.util; +package com.myoa.engineering.crawl.shopping.support.util; public final class ObjectUtil { diff --git a/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/util/WebRequestUtil.java b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/util/WebRequestUtil.java similarity index 88% rename from support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/util/WebRequestUtil.java rename to support/src/main/java/com/myoa/engineering/crawl/shopping/support/util/WebRequestUtil.java index 9204689..00675df 100644 --- a/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/util/WebRequestUtil.java +++ b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/util/WebRequestUtil.java @@ -1,4 +1,4 @@ -package com.myoa.engineering.crawl.ppomppu.support.util; +package com.myoa.engineering.crawl.shopping.support.util; /** * WebUtil diff --git a/support/src/main/java/com/myoa/engineering/crawl/shopping/support/webclient/WebClientBaseScan.java b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/webclient/WebClientBaseScan.java new file mode 100644 index 0000000..0695933 --- /dev/null +++ b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/webclient/WebClientBaseScan.java @@ -0,0 +1,4 @@ +package com.myoa.engineering.crawl.shopping.support.webclient; + +public interface WebClientBaseScan { +} diff --git a/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/webclient/factory/WebClientFilterFactory.java b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/webclient/factory/WebClientFilterFactory.java similarity index 81% rename from support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/webclient/factory/WebClientFilterFactory.java rename to support/src/main/java/com/myoa/engineering/crawl/shopping/support/webclient/factory/WebClientFilterFactory.java index 89d1ec9..cd8c29e 100644 --- a/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/webclient/factory/WebClientFilterFactory.java +++ b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/webclient/factory/WebClientFilterFactory.java @@ -1,10 +1,6 @@ -package com.myoa.engineering.crawl.ppomppu.support.webclient.factory; +package com.myoa.engineering.crawl.shopping.support.webclient.factory; import lombok.extern.slf4j.Slf4j; -import org.springframework.web.reactive.function.client.ClientRequest; -import org.springframework.web.reactive.function.client.ClientResponse; -import org.springframework.web.reactive.function.client.ExchangeFilterFunction; -import reactor.core.publisher.Mono; /** * WebClientFilterFactory @@ -15,6 +11,7 @@ import reactor.core.publisher.Mono; @Slf4j public final class WebClientFilterFactory { + /* private WebClientFilterFactory() {} public static ExchangeFilterFunction logRequest() { @@ -44,5 +41,6 @@ public final class WebClientFilterFactory { } return Mono.just(clientResponse); } + */ } diff --git a/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/webclient/factory/WebFluxExchangeStragiesFactory.java b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/webclient/factory/WebFluxExchangeStragiesFactory.java similarity index 82% rename from support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/webclient/factory/WebFluxExchangeStragiesFactory.java rename to support/src/main/java/com/myoa/engineering/crawl/shopping/support/webclient/factory/WebFluxExchangeStragiesFactory.java index 6558638..a14ab39 100644 --- a/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/webclient/factory/WebFluxExchangeStragiesFactory.java +++ b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/webclient/factory/WebFluxExchangeStragiesFactory.java @@ -1,12 +1,4 @@ -package com.myoa.engineering.crawl.ppomppu.support.webclient.factory; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.myoa.engineering.crawl.ppomppu.support.util.ObjectMapperFactory; - -import org.springframework.http.codec.json.Jackson2JsonDecoder; -import org.springframework.http.codec.json.Jackson2JsonEncoder; -import org.springframework.util.MimeTypeUtils; -import org.springframework.web.reactive.function.client.ExchangeStrategies; +package com.myoa.engineering.crawl.shopping.support.webclient.factory; /** * WebFluxExchangeStragiesFactory @@ -15,7 +7,7 @@ import org.springframework.web.reactive.function.client.ExchangeStrategies; * @since 2021-09-08 */ public final class WebFluxExchangeStragiesFactory { - +/* private WebFluxExchangeStragiesFactory() {} public static ExchangeStrategies ofDefault() { @@ -46,5 +38,5 @@ public final class WebFluxExchangeStragiesFactory { }) .build(); } - +*/ } diff --git a/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/webclient/properties/WebClientProperties.java b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/webclient/properties/WebClientProperties.java similarity index 93% rename from support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/webclient/properties/WebClientProperties.java rename to support/src/main/java/com/myoa/engineering/crawl/shopping/support/webclient/properties/WebClientProperties.java index 64b89f8..6b893f8 100644 --- a/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/webclient/properties/WebClientProperties.java +++ b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/webclient/properties/WebClientProperties.java @@ -1,4 +1,4 @@ -package com.myoa.engineering.crawl.ppomppu.support.webclient.properties; +package com.myoa.engineering.crawl.shopping.support.webclient.properties; import java.util.ArrayList; import java.util.List; diff --git a/support/src/main/resources/development/database.yml b/support/src/main/resources/development/database.yml deleted file mode 100644 index a4af932..0000000 --- a/support/src/main/resources/development/database.yml +++ /dev/null @@ -1,41 +0,0 @@ -spring: - jpa: - open-in-view: false - hibernate: - ddl-auto: create - datasource: - # driver-class-name: com.mysql.cj.jdbc.Driver - driver-class-name: org.h2.Driver - url: jdbc:h2:mem:ppomppu-local;DB_CLOSE_DELAY=-1 - hikari: - minimum:idle: 5 - maximum-pool-size: 10 - idle-timeout: 600000 - validation-timeout: 5000 - connection-timeout: 5000 - max-lifetime: 1800000 - auto-commit: false - h2: - console: - enabled: true - path: /h2 - port: 20082 - -datasource: - init: true - units: - - unit-name: ppomppu - schema-name: ppomppu-local - db-connection-url: jdbc:h2:mem:ppomppu-local - simple-connection-url: true - - -hibernate: - units: - - unit-name: ppomppu - dialect: org.hibernate.dialect.H2Dialect - format-sql: true - show-sql: true - hbm2ddl-auto: create - disable-auto-commit: true - diff --git a/support/src/main/resources/logback/logback-development.xml b/support/src/main/resources/logback/logback-development.xml index 458e3d8..ed84966 100644 --- a/support/src/main/resources/logback/logback-development.xml +++ b/support/src/main/resources/logback/logback-development.xml @@ -10,8 +10,6 @@ - - diff --git a/support/src/main/resources/logback/logback-production.xml b/support/src/main/resources/logback/logback-production.xml index f824e41..df845f5 100644 --- a/support/src/main/resources/logback/logback-production.xml +++ b/support/src/main/resources/logback/logback-production.xml @@ -10,8 +10,6 @@ - -