MyoaEngineering
/
ShoppingCrawler
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

[NO-ISSUE] Implement v2

This commit is contained in:
woozu-shin 2024-04-30 22:34:49 +09:00
parent 0524a18ee5
commit 0c4be3cc05
147 changed files with 3873 additions and 546 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
build.gradle
View File

@ -1,13 +1,14 @@
plugins {
id 'org.springframework.boot' version '2.5.4'
id 'io.spring.dependency-management' version '1.0.11.RELEASE'
id 'java'
id 'idea'
id 'org.springframework.boot' version '3.2.5'
id 'io.spring.dependency-management' version '1.1.4'
}
group = 'com.myoa.engineering.crawl.ppomppu'
version = '1.0.1'
sourceCompatibility = '11'
sourceCompatibility = JavaVersion.VERSION_21
targetCompatibility = JavaVersion.VERSION_21
configurations {
compileOnly {
@ -20,8 +21,8 @@ repositories {
}
allprojects {
group = 'com.myoa.engineering.crawl.ppomppu'
version = '1.0.1'
group = 'com.myoa.engineering.crawl.shopping'
version = '2.0.0'
apply plugin: 'java'
apply plugin: 'idea'
@ -36,7 +37,7 @@ allprojects {
}
ext {
set('springCloudVersion', "2020.0.4")
set('springCloudVersion', "2023.0.1")
}
dependencyManagement {

3
gradle/wrapper/gradle-wrapper.properties vendored
View File

@ -1,5 +1,6 @@
#Sun Apr 28 23:47:38 KST 2024
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-7.1.1-bin.zip
distributionUrl=https\://services.gradle.org/distributions/gradle-8.5-bin.zip
zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists

0
gradlew vendored Normal file → Executable file
View File

BIN
processor-1.0.1.jar
View File

Binary file not shown.

7
processor/build.gradle
View File

@ -12,14 +12,15 @@ dependencies {
implementation 'org.springframework.boot:spring-boot-starter-jdbc'
implementation 'org.springframework.cloud:spring-cloud-starter-config'
implementation 'org.springframework.boot:spring-boot-starter-actuator'
implementation 'com.rometools:rome:1.16.0'
implementation 'org.jsoup:jsoup:1.14.2'
implementation 'com.h2database:h2:1.4.200'
implementation 'com.rometools:rome:2.1.0'
implementation 'org.jsoup:jsoup:1.15.3'
implementation 'com.h2database:h2:2.2.220'
annotationProcessor 'org.springframework.boot:spring-boot-configuration-processor'
annotationProcessor 'org.projectlombok:lombok'
testImplementation 'org.springframework.boot:spring-boot-starter-test'
testImplementation("org.assertj:assertj-core:3.24.2")
}
test {

2
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/ProcessorApplication.java
View File

@ -4,7 +4,7 @@ import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.context.annotation.Import;
import com.myoa.engineering.crawl.ppomppu.support.webclient.PpomppuNotifierWebClientConfiguration;
import com.myoa.engineering.crawl.shopping.support.webclient.PpomppuNotifierWebClientConfiguration;
/**
* ProcessorApplication

5
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/controller/CrawlAPIController.java
View File

@ -1,6 +1,5 @@
package com.myoa.engineering.crawl.ppomppu.processor.controller;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
@ -10,8 +9,8 @@ import com.myoa.engineering.crawl.ppomppu.processor.dto.FeedParsedResult;
import com.myoa.engineering.crawl.ppomppu.processor.service.MessageSenderService;
import com.myoa.engineering.crawl.ppomppu.processor.service.PpomppuArticleService;
import com.myoa.engineering.crawl.ppomppu.processor.service.PpomppuFeedService;
import com.myoa.engineering.crawl.ppomppu.support.dto.APIResponse;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import com.myoa.engineering.crawl.shopping.support.dto.APIResponse;
import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName;
import lombok.extern.slf4j.Slf4j;
import reactor.core.publisher.Mono;

32
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/SubscribedBoard.java
View File

@ -1,32 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.processor.domain;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.EnumType;
import javax.persistence.Enumerated;
import javax.persistence.GeneratedValue;
import javax.persistence.GenerationType;
import javax.persistence.Id;
import javax.persistence.Table;
import lombok.Getter;
import lombok.NoArgsConstructor;
@Getter
@NoArgsConstructor
@Entity
@Table(name = "subscribed_board")
public class SubscribedBoard extends Auditable {
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
private Long id;
@Column
private Long userId;
@Column
@Enumerated(EnumType.STRING)
private PpomppuBoardName boardName;
}

4
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/repository/BaseScanRepository.java
View File

@ -1,4 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository;
public interface BaseScanRepository {
}

4
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/PpomppuArticleParser.java → processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/parser/PpomppuArticleParser.java
View File

@ -1,4 +1,4 @@
package com.myoa.engineering.crawl.ppomppu.processor.dto;
package com.myoa.engineering.crawl.ppomppu.processor.parser;
import java.time.Instant;
import java.time.ZoneId;
@ -8,7 +8,7 @@ import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName;
/**
* PpomppuArticleTransformer

1100
processor/src/test/resources/testdata/zboard/file1.html vendored Normal file
View File

File diff suppressed because it is too large Load Diff

0
receiver/build.gradle → reader/build.gradle
View File

10
receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/ReceiverApplication.java → reader/src/main/java/com/myoa/engineering/crawl/ppomppu/reader/ReaderApplication.java
View File

@ -1,12 +1,10 @@
package com.myoa.engineering.crawl.ppomppu.receiver;
package com.myoa.engineering.crawl.ppomppu.reader;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.annotation.Import;
import com.myoa.engineering.crawl.ppomppu.receiver.configuration.properties.TelegramBotProperties;
import com.myoa.engineering.crawl.ppomppu.support.webclient.PpomppuNotifierWebClientConfiguration;
import com.myoa.engineering.crawl.shopping.support.webclient.PpomppuNotifierWebClientConfiguration;
/**
* ReceiverApplication
@ -17,9 +15,9 @@ import com.myoa.engineering.crawl.ppomppu.support.webclient.PpomppuNotifierWebCl
@Import({ PpomppuNotifierWebClientConfiguration.class})
@SpringBootApplication
// @EnableConfigurationProperties({ TelegramBotProperties.class })
public class ReceiverApplication {
public class ReaderApplication {
public static void main(String[] args) {
SpringApplication.run(ReceiverApplication.class, args);
SpringApplication.run(ReaderApplication.class, args);
}
}

10
receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/configuration/TelegramBotConfiguration.java → reader/src/main/java/com/myoa/engineering/crawl/ppomppu/reader/configuration/TelegramBotConfiguration.java
View File

@ -1,4 +1,4 @@
package com.myoa.engineering.crawl.ppomppu.receiver.configuration;
package com.myoa.engineering.crawl.ppomppu.reader.configuration;
import java.util.List;
@ -8,10 +8,10 @@ import org.telegram.telegrambots.meta.TelegramBotsApi;
import org.telegram.telegrambots.meta.exceptions.TelegramApiException;
import org.telegram.telegrambots.updatesreceivers.DefaultBotSession;
import com.myoa.engineering.crawl.ppomppu.receiver.configuration.properties.TelegramBotProperties;
import com.myoa.engineering.crawl.ppomppu.receiver.configuration.properties.TelegramBotProperties.TelegramBotPropertiesUnit;
import com.myoa.engineering.crawl.ppomppu.receiver.dispatch.MessageDispatcher;
import com.myoa.engineering.crawl.ppomppu.receiver.handler.message.MessageHandler;
import com.myoa.engineering.crawl.ppomppu.reader.configuration.properties.TelegramBotProperties;
import com.myoa.engineering.crawl.ppomppu.reader.configuration.properties.TelegramBotProperties.TelegramBotPropertiesUnit;
import com.myoa.engineering.crawl.ppomppu.reader.dispatch.MessageDispatcher;
import com.myoa.engineering.crawl.ppomppu.reader.handler.message.MessageHandler;
/**
* TelegramBotConfiguration

5
receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/configuration/properties/TelegramBotProperties.java → reader/src/main/java/com/myoa/engineering/crawl/ppomppu/reader/configuration/properties/TelegramBotProperties.java
View File

@ -1,16 +1,13 @@
package com.myoa.engineering.crawl.ppomppu.receiver.configuration.properties;
package com.myoa.engineering.crawl.ppomppu.reader.configuration.properties;
import java.util.ArrayList;
import java.util.List;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.boot.context.properties.ConstructorBinding;
import org.springframework.stereotype.Component;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
/**

6
receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/controller/v1/EventAPIController.java → reader/src/main/java/com/myoa/engineering/crawl/ppomppu/reader/controller/v1/EventAPIController.java
View File

@ -1,12 +1,12 @@
package com.myoa.engineering.crawl.ppomppu.receiver.controller.v1;
package com.myoa.engineering.crawl.ppomppu.reader.controller.v1;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import com.myoa.engineering.crawl.ppomppu.receiver.service.ProcessorAPIService;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import com.myoa.engineering.crawl.ppomppu.reader.service.ProcessorAPIService;
import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName;
import reactor.core.publisher.Mono;

4
receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/dispatch/MessageDispatcher.java → reader/src/main/java/com/myoa/engineering/crawl/ppomppu/reader/dispatch/MessageDispatcher.java
View File

@ -1,4 +1,4 @@
package com.myoa.engineering.crawl.ppomppu.receiver.dispatch;
package com.myoa.engineering.crawl.ppomppu.reader.dispatch;
import java.util.List;
@ -6,7 +6,7 @@ import org.telegram.telegrambots.bots.TelegramLongPollingBot;
import org.telegram.telegrambots.meta.api.objects.Message;
import org.telegram.telegrambots.meta.api.objects.Update;
import com.myoa.engineering.crawl.ppomppu.receiver.handler.message.MessageHandler;
import com.myoa.engineering.crawl.ppomppu.reader.handler.message.MessageHandler;
import lombok.extern.slf4j.Slf4j;

4
receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/HelloWorldMessageHandler.java → reader/src/main/java/com/myoa/engineering/crawl/ppomppu/reader/handler/message/HelloWorldMessageHandler.java
View File

@ -1,6 +1,6 @@
package com.myoa.engineering.crawl.ppomppu.receiver.handler.message;
package com.myoa.engineering.crawl.ppomppu.reader.handler.message;
import com.myoa.engineering.crawl.ppomppu.support.util.ObjectUtil;
import com.myoa.engineering.crawl.shopping.support.util.ObjectUtil;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import org.telegram.telegrambots.meta.api.objects.Message;

2
receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/ImageMessageHandler.java → reader/src/main/java/com/myoa/engineering/crawl/ppomppu/reader/handler/message/ImageMessageHandler.java
View File

@ -1,4 +1,4 @@
package com.myoa.engineering.crawl.ppomppu.receiver.handler.message;
package com.myoa.engineering.crawl.ppomppu.reader.handler.message;
/**
* ImageHandler

2
receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/MessageHandler.java → reader/src/main/java/com/myoa/engineering/crawl/ppomppu/reader/handler/message/MessageHandler.java
View File

@ -1,4 +1,4 @@
package com.myoa.engineering.crawl.ppomppu.receiver.handler.message;
package com.myoa.engineering.crawl.ppomppu.reader.handler.message;
import org.telegram.telegrambots.meta.api.objects.Message;

4
receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/TextMessageHandler.java → reader/src/main/java/com/myoa/engineering/crawl/ppomppu/reader/handler/message/TextMessageHandler.java
View File

@ -1,6 +1,6 @@
package com.myoa.engineering.crawl.ppomppu.receiver.handler.message;
package com.myoa.engineering.crawl.ppomppu.reader.handler.message;
import com.myoa.engineering.crawl.ppomppu.support.util.ObjectUtil;
import com.myoa.engineering.crawl.shopping.support.util.ObjectUtil;
import org.telegram.telegrambots.meta.api.objects.Message;
/**

4
receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/text/CommandHandler.java → reader/src/main/java/com/myoa/engineering/crawl/ppomppu/reader/handler/message/text/CommandHandler.java
View File

@ -1,11 +1,11 @@
package com.myoa.engineering.crawl.ppomppu.receiver.handler.message.text;
package com.myoa.engineering.crawl.ppomppu.reader.handler.message.text;
import java.util.List;
import org.springframework.stereotype.Component;
import org.telegram.telegrambots.meta.api.objects.Message;
import com.myoa.engineering.crawl.ppomppu.receiver.handler.message.TextMessageHandler;
import com.myoa.engineering.crawl.ppomppu.reader.handler.message.TextMessageHandler;
import lombok.extern.slf4j.Slf4j;

4
receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/text/CommonTextHandler.java → reader/src/main/java/com/myoa/engineering/crawl/ppomppu/reader/handler/message/text/CommonTextHandler.java
View File

@ -1,9 +1,9 @@
package com.myoa.engineering.crawl.ppomppu.receiver.handler.message.text;
package com.myoa.engineering.crawl.ppomppu.reader.handler.message.text;
import org.springframework.stereotype.Component;
import org.telegram.telegrambots.meta.api.objects.Message;
import com.myoa.engineering.crawl.ppomppu.receiver.handler.message.TextMessageHandler;
import com.myoa.engineering.crawl.ppomppu.reader.handler.message.TextMessageHandler;
import lombok.extern.slf4j.Slf4j;

2
receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/text/EmptyTextCommandProcessor.java → reader/src/main/java/com/myoa/engineering/crawl/ppomppu/reader/handler/message/text/EmptyTextCommandProcessor.java
View File

@ -1,4 +1,4 @@
package com.myoa.engineering.crawl.ppomppu.receiver.handler.message.text;
package com.myoa.engineering.crawl.ppomppu.reader.handler.message.text;
import org.springframework.stereotype.Component;
import org.telegram.telegrambots.meta.api.objects.Message;

2
receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/text/StartTextCommandProcessor.java → reader/src/main/java/com/myoa/engineering/crawl/ppomppu/reader/handler/message/text/StartTextCommandProcessor.java
View File

@ -1,4 +1,4 @@
package com.myoa.engineering.crawl.ppomppu.receiver.handler.message.text;
package com.myoa.engineering.crawl.ppomppu.reader.handler.message.text;
import org.springframework.stereotype.Component;
import org.telegram.telegrambots.meta.api.objects.Message;

2
receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/text/TextCommandCode.java → reader/src/main/java/com/myoa/engineering/crawl/ppomppu/reader/handler/message/text/TextCommandCode.java
View File

@ -1,4 +1,4 @@
package com.myoa.engineering.crawl.ppomppu.receiver.handler.message.text;
package com.myoa.engineering.crawl.ppomppu.reader.handler.message.text;
import java.util.Arrays;

2
receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/text/TextCommandProcessor.java → reader/src/main/java/com/myoa/engineering/crawl/ppomppu/reader/handler/message/text/TextCommandProcessor.java
View File

@ -1,4 +1,4 @@
package com.myoa.engineering.crawl.ppomppu.receiver.handler.message.text;
package com.myoa.engineering.crawl.ppomppu.reader.handler.message.text;
import org.telegram.telegrambots.meta.api.objects.Message;

12
receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/infrastructure/client/ProcessorAPIWebClient.java → reader/src/main/java/com/myoa/engineering/crawl/ppomppu/reader/infrastructure/client/ProcessorAPIWebClient.java
View File

@ -1,15 +1,15 @@
package com.myoa.engineering.crawl.ppomppu.receiver.infrastructure.client;
package com.myoa.engineering.crawl.ppomppu.reader.infrastructure.client;
import org.springframework.core.ParameterizedTypeReference;
import org.springframework.stereotype.Component;
import org.springframework.web.reactive.function.client.WebClient;
import org.springframework.web.reactive.function.client.WebClientRequestException;
import com.myoa.engineering.crawl.ppomppu.receiver.dto.constant.WebClientPropertiesUnitName;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebClientFilterFactory;
import com.myoa.engineering.crawl.ppomppu.support.webclient.properties.WebClientProperties;
import com.myoa.engineering.crawl.ppomppu.support.webclient.properties.WebClientProperties.WebClientPropertiesUnit;
import com.myoa.engineering.crawl.ppomppu.reader.dto.constant.WebClientPropertiesUnitName;
import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName;
import com.myoa.engineering.crawl.shopping.support.webclient.factory.WebClientFilterFactory;
import com.myoa.engineering.crawl.shopping.support.webclient.properties.WebClientProperties;
import com.myoa.engineering.crawl.shopping.support.webclient.properties.WebClientProperties.WebClientPropertiesUnit;
import lombok.extern.slf4j.Slf4j;
import reactor.core.publisher.Mono;

6
receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/scheduler/ParseEventEmitter.java → reader/src/main/java/com/myoa/engineering/crawl/ppomppu/reader/scheduler/ParseEventEmitter.java
View File

@ -1,11 +1,11 @@
package com.myoa.engineering.crawl.ppomppu.receiver.scheduler;
package com.myoa.engineering.crawl.ppomppu.reader.scheduler;
import org.springframework.scheduling.annotation.EnableScheduling;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import com.myoa.engineering.crawl.ppomppu.receiver.service.ProcessorAPIService;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import com.myoa.engineering.crawl.ppomppu.reader.service.ProcessorAPIService;
import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName;
import lombok.extern.slf4j.Slf4j;

6
receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/service/ProcessorAPIService.java → reader/src/main/java/com/myoa/engineering/crawl/ppomppu/reader/service/ProcessorAPIService.java
View File

@ -1,9 +1,9 @@
package com.myoa.engineering.crawl.ppomppu.receiver.service;
package com.myoa.engineering.crawl.ppomppu.reader.service;
import org.springframework.stereotype.Service;
import com.myoa.engineering.crawl.ppomppu.receiver.infrastructure.client.ProcessorAPIWebClient;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import com.myoa.engineering.crawl.ppomppu.reader.infrastructure.client.ProcessorAPIWebClient;
import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName;
import lombok.extern.slf4j.Slf4j;
import reactor.core.publisher.Mono;

0
receiver/src/main/resources/application-development.yml → reader/src/main/resources/application-development.yml
View File

0
receiver/src/main/resources/application-local.yml → reader/src/main/resources/application-local.yml
View File

0
receiver/src/main/resources/application-production.yml → reader/src/main/resources/application-production.yml
View File

0
receiver/src/main/resources/application.yml → reader/src/main/resources/application.yml
View File

0
receiver/src/main/resources/logback-spring.xml → reader/src/main/resources/logback-spring.xml
View File

0
receiver/src/main/resources/logback/component/logback-file.xml → reader/src/main/resources/logback/component/logback-file.xml
View File

0
receiver/src/main/resources/logback/logback-development.xml → reader/src/main/resources/logback/logback-development.xml
View File

0
receiver/src/main/resources/logback/logback-production.xml → reader/src/main/resources/logback/logback-production.xml
View File

BIN
receiver-1.0.1.jar
View File

Binary file not shown.

25
receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/dto/FeedParsedResult.java
View File

@ -1,25 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.receiver.dto;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import java.io.Serializable;
import java.time.Instant;
import lombok.Getter;
import lombok.NoArgsConstructor;
/**
* FeedParsedResult
*
* @author Shin Woo-jin (woozu.shin@kakaoent.com)
* @since 2021-09-08
*/
@Getter
@NoArgsConstructor
public class FeedParsedResult implements Serializable {
private static final long serialVersionUID = -3771310078623481348L;
private PpomppuBoardName boardName;
private Instant requestedAt;
private Instant processedAt;
}

19
receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/dto/constant/WebClientPropertiesUnitName.java
View File

@ -1,19 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.receiver.dto.constant;
import lombok.AllArgsConstructor;
import lombok.Getter;
/**
* WebClientPropertiesUnitName
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-11-18
*
*/
@Getter
@AllArgsConstructor
public enum WebClientPropertiesUnitName {
PPOMPPU_NOTIFIER_PROCESSOR_API("ppn-processor-api"),
;
private String unitName;
}

BIN
sender-1.0.1.jar
View File

Binary file not shown.

22
sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/SenderApplication.java
View File

@ -1,22 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.sender;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.context.annotation.Import;
import com.myoa.engineering.crawl.ppomppu.support.webclient.PpomppuNotifierWebClientConfiguration;
/**
* SenderApplication
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-08-20
*
*/
@Import({ PpomppuNotifierWebClientConfiguration.class })
@SpringBootApplication
public class SenderApplication {
public static void main(String[] args) {
SpringApplication.run(SenderApplication.class, args);
}
}

17
sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/infrastructure/client/MessageSender.java
View File

@ -1,17 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.sender.infrastructure.client;
import com.myoa.engineering.crawl.ppomppu.sender.dto.MessageDTO;
import reactor.core.publisher.Mono;
/**
* MessageSender
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-11-14
*
*/
public interface MessageSender<T extends MessageDTO> {
Mono<String> sendMessage(T message);
}

16
settings.gradle
View File

@ -1,14 +1,14 @@
rootProject.name = 'PpomppuNotifier'
include 'receiver'
project(':receiver').projectDir = "$rootDir/receiver" as File
include 'processor'
project(':processor').projectDir = "$rootDir/processor" as File
include 'sender'
project(':sender').projectDir = "$rootDir/sender" as File
//include 'reader'
//project(':reader').projectDir = "$rootDir/reader" as File
//include 'processor'
//project(':processor').projectDir = "$rootDir/processor" as File
//include 'writer'
//project(':writer').projectDir = "$rootDir/writer" as File
include 'shopping-crawler'
project(':shopping-crawler').projectDir = "$rootDir/shopping-crawler" as File
include 'support'
project(':support').projectDir = "$rootDir/support" as File

BIN
shopping-crawler/.jqwik-database Normal file
View File

Binary file not shown.

42
shopping-crawler/build.gradle Normal file
View File

@ -0,0 +1,42 @@
dependencies {
developmentOnly 'org.springframework.boot:spring-boot-devtools'
runtimeOnly 'com.h2database:h2'
runtimeOnly 'com.mysql:mysql-connector-j'
compileOnly 'org.projectlombok:lombok'
implementation project(':support')
// https://projectreactor.io/docs/core/release/reference/#debug-activate
implementation("org.springframework.boot:spring-boot-starter-web") {
exclude group: "org.springframework.boot", module: "spring-boot-starter-tomcat"
}
implementation("org.springframework.boot:spring-boot-starter-undertow") {
exclude group: "io.undertow", module: "undertow-websockets-jsr"
}
implementation 'org.springframework.boot:spring-boot-starter-data-jpa'
implementation 'org.springframework.boot:spring-boot-configuration-processor'
implementation 'org.springframework.cloud:spring-cloud-starter-config'
implementation 'org.springframework.boot:spring-boot-starter-actuator'
implementation 'com.rometools:rome:2.1.0'
implementation 'org.jsoup:jsoup:1.17.2'
implementation 'com.h2database:h2:2.2.224'
implementation "org.springframework.cloud:spring-cloud-starter-openfeign"
implementation "io.github.openfeign:feign-hc5"
implementation 'org.ahocorasick:ahocorasick:0.6.3'
annotationProcessor 'org.springframework.boot:spring-boot-configuration-processor'
annotationProcessor 'org.projectlombok:lombok'
testImplementation 'org.springframework.boot:spring-boot-starter-test'
testImplementation('org.assertj:assertj-core:3.25.3')
testImplementation("org.jeasy:easy-random-core:5.0.0")
testCompileOnly 'org.projectlombok:lombok'
testAnnotationProcessor('org.projectlombok:lombok')
}
test {
useJUnitPlatform()
testLogging {
events "passed", "skipped", "failed"
}
}

19
shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/ShoppingCrawlerApplication.java Normal file
View File

@ -0,0 +1,19 @@
package com.myoa.engineering.crawl.shopping;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.cloud.openfeign.EnableFeignClients;
import org.springframework.scheduling.annotation.EnableScheduling;
@EnableScheduling
@EnableFeignClients
@EnableConfigurationProperties
@SpringBootApplication
public class ShoppingCrawlerApplication {
public static void main(String[] args) {
SpringApplication.run(ShoppingCrawlerApplication.class, args);
}
}

5
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/H2ConsoleConfiguration.java → shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/datasource/H2ConsoleConfiguration.java
View File

@ -1,6 +1,5 @@
package com.myoa.engineering.crawl.ppomppu.processor.configuration;
package com.myoa.engineering.crawl.shopping.configuration.datasource;
import java.sql.SQLException;
import lombok.extern.slf4j.Slf4j;
import org.h2.tools.Server;
import org.springframework.beans.factory.annotation.Value;
@ -10,6 +9,8 @@ import org.springframework.context.event.ContextClosedEvent;
import org.springframework.context.event.ContextRefreshedEvent;
import org.springframework.context.event.EventListener;
import java.sql.SQLException;
@Slf4j
@Profile({"datasource-local", "datasource-development"})
@Configuration

75
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/PpomppuDatasourceConfiguration.java → shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/datasource/ShoppingCrawlerDatasourceConfiguration.java
View File

@ -1,54 +1,58 @@
package com.myoa.engineering.crawl.ppomppu.processor.configuration;
package com.myoa.engineering.crawl.shopping.configuration.datasource;
import com.myoa.engineering.crawl.ppomppu.processor.configuration.properties.DatasourceProperties;
import com.myoa.engineering.crawl.ppomppu.processor.configuration.properties.DatasourceProperties.DataSourcePropertiesUnit;
import com.myoa.engineering.crawl.ppomppu.processor.configuration.properties.HibernateProperties;
import com.myoa.engineering.crawl.ppomppu.processor.configuration.properties.HikariProperties;
import com.myoa.engineering.crawl.ppomppu.processor.domain.BaseScanDomain;
import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository.BaseScanRepository;
import com.myoa.engineering.crawl.shopping.configuration.datasource.properties.DatasourceProperties;
import com.myoa.engineering.crawl.shopping.configuration.datasource.properties.HibernateProperties;
import com.myoa.engineering.crawl.shopping.configuration.datasource.properties.HikariProperties;
import com.myoa.engineering.crawl.shopping.domain.entity.BaseScanDomain;
import com.myoa.engineering.crawl.shopping.infra.repository.BaseScanRepository;
import com.zaxxer.hikari.HikariConfig;
import com.zaxxer.hikari.HikariDataSource;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
import javax.persistence.EntityManagerFactory;
import javax.sql.DataSource;
import jakarta.persistence.EntityManagerFactory;
import lombok.NonNull;
import org.hibernate.boot.model.naming.CamelCaseToUnderscoresNamingStrategy;
import org.hibernate.boot.model.naming.ImplicitNamingStrategyJpaCompliantImpl;
import org.hibernate.cfg.AvailableSettings;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.boot.orm.jpa.EntityManagerFactoryBuilder;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.data.jpa.repository.config.EnableJpaAuditing;
import org.springframework.data.jpa.repository.config.EnableJpaRepositories;
import org.springframework.orm.jpa.JpaTransactionManager;
import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean;
import org.springframework.transaction.PlatformTransactionManager;
@Configuration
@EnableJpaRepositories(basePackageClasses = BaseScanRepository.class,
entityManagerFactoryRef = "ppomppuNotifierProcessorEntityManagerFactory",
transactionManagerRef = "ppomppuNotifierProcessorTransactionManager"
)
public class PpomppuDatasourceConfiguration {
import javax.sql.DataSource;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
private static final String DATA_SOURCE_UNIT_NAME = "ppn_mysql";
@Configuration
@EnableJpaAuditing
@EnableJpaRepositories(basePackageClasses = BaseScanRepository.class,
entityManagerFactoryRef = "shoppingCrawlerEntityManagerFactory",
transactionManagerRef = "shoppingCrawlerTransactionManager"
)
public class ShoppingCrawlerDatasourceConfiguration {
private static final String DATA_SOURCE_UNIT_NAME = "crawler-shopping";
private final DatasourceProperties dataSourceProeprties;
private final HikariProperties hikariProperties;
private final HibernateProperties hibernateProperties;
public PpomppuDatasourceConfiguration(DatasourceProperties dataSourceProeprties,
HikariProperties hikariProperties,
HibernateProperties hibernateProperties) {
public ShoppingCrawlerDatasourceConfiguration(DatasourceProperties dataSourceProeprties,
HikariProperties hikariProperties,
HibernateProperties hibernateProperties) {
this.dataSourceProeprties = dataSourceProeprties;
this.hikariProperties = hikariProperties;
this.hibernateProperties = hibernateProperties;
}
@Bean(name = "ppomppuNotifierProcessorDataSource")
@Bean(name = "shoppingCrawlerDataSource")
public DataSource dataSource() {
DataSourcePropertiesUnit dataSourcePropertiesUnit = dataSourceProeprties.find(DATA_SOURCE_UNIT_NAME);
DatasourceProperties.DataSourcePropertiesUnit dataSourcePropertiesUnit = dataSourceProeprties.find(DATA_SOURCE_UNIT_NAME);
final HikariConfig hikariConfig = new HikariConfig();
hikariConfig.setJdbcUrl(dataSourcePropertiesUnit.toCompletedJdbcUrl());
@ -69,19 +73,19 @@ public class PpomppuDatasourceConfiguration {
return dataSource;
}
@Bean("ppomppuNotifierProcessorEntityManagerFactory")
@Bean("shoppingCrawlerEntityManagerFactory")
public LocalContainerEntityManagerFactoryBean entityManagerFactory(
EntityManagerFactoryBuilder builder,
@Qualifier("ppomppuNotifierProcessorDataSource") DataSource dataSource) {
EntityManagerFactoryBuilder builder,
@Qualifier("shoppingCrawlerDataSource") DataSource dataSource) {
return builder.dataSource(dataSource)
.packages(BaseScanDomain.class)
.properties(getPropsMap(DATA_SOURCE_UNIT_NAME))
.build();
}
@Bean("ppomppuNotifierProcessorTransactionManager")
@Bean("shoppingCrawlerTransactionManager")
public PlatformTransactionManager transactionManager(
@Qualifier("ppomppuNotifierProcessorEntityManagerFactory") EntityManagerFactory entityManagerFactory) {
@Qualifier("shoppingCrawlerEntityManagerFactory") EntityManagerFactory entityManagerFactory) {
return new JpaTransactionManager(entityManagerFactory);
}
@ -91,12 +95,9 @@ public class PpomppuDatasourceConfiguration {
properties.put(AvailableSettings.FORMAT_SQL, hibernateProperties.getFormatSql());
properties.put(AvailableSettings.SHOW_SQL, hibernateProperties.getShowSql());
properties.put(AvailableSettings.HBM2DDL_AUTO, hibernateProperties.getHbm2ddlAuto());
properties.put(AvailableSettings.CONNECTION_PROVIDER_DISABLES_AUTOCOMMIT,
hibernateProperties.getDisableAutoCommit());
properties.put(AvailableSettings.IMPLICIT_NAMING_STRATEGY,
"org.springframework.boot.orm.jpa.hibernate.SpringImplicitNamingStrategy");
properties.put(AvailableSettings.PHYSICAL_NAMING_STRATEGY,
"org.springframework.boot.orm.jpa.hibernate.SpringPhysicalNamingStrategy");
properties.put(AvailableSettings.CONNECTION_PROVIDER_DISABLES_AUTOCOMMIT, hibernateProperties.getDisableAutoCommit());
properties.put(AvailableSettings.IMPLICIT_NAMING_STRATEGY, ImplicitNamingStrategyJpaCompliantImpl.class.getName());
properties.put(AvailableSettings.PHYSICAL_NAMING_STRATEGY, CamelCaseToUnderscoresNamingStrategy.class.getName());
properties.put(AvailableSettings.GENERATE_STATISTICS, "false");
properties.put(AvailableSettings.GLOBALLY_QUOTED_IDENTIFIERS, "true");
properties.put(AvailableSettings.GLOBALLY_QUOTED_IDENTIFIERS_SKIP_COLUMN_DEFINITIONS, "true");
@ -104,7 +105,7 @@ public class PpomppuDatasourceConfiguration {
properties.put(AvailableSettings.ORDER_INSERTS, "true");
properties.put(AvailableSettings.ORDER_UPDATES, "true");
properties.put(AvailableSettings.BATCH_VERSIONED_DATA, "true");
properties.put(AvailableSettings.USE_NEW_ID_GENERATOR_MAPPINGS, "false");
// properties.put(AvailableSettings.JPA_ID_GENERATOR_GLOBAL_SCOPE_COMPLIANCE, "false");
return properties;
}

27
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/properties/DatasourceProperties.java → shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/datasource/properties/DatasourceProperties.java
View File

@ -1,29 +1,26 @@
package com.myoa.engineering.crawl.ppomppu.processor.configuration.properties;
package com.myoa.engineering.crawl.shopping.configuration.datasource.properties;
import com.myoa.engineering.crawl.ppomppu.support.util.ObjectUtil;
import java.util.List;
import lombok.Getter;
import lombok.Setter;
import com.myoa.engineering.crawl.shopping.support.util.ObjectUtil;
import lombok.Data;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;
import java.util.List;
@Component
@Setter
@Getter
@ConfigurationProperties(prefix = "infra.database")
@Data
@ConfigurationProperties(prefix = "datasource")
public class DatasourceProperties {
private List<DataSourcePropertiesUnit> units;
@Getter
@Setter
@Data
public static class DataSourcePropertiesUnit {
private String unitName;
private String schemaName;
private String connectionParameters;
private String datasourceUrl;
private String dbConnectionUrl;
private Boolean isSimpleConnectionUrl;
private String username;
private String password;
@ -31,9 +28,9 @@ public class DatasourceProperties {
public String toCompletedJdbcUrl() {
if (ObjectUtil.isEmpty(isSimpleConnectionUrl) || isSimpleConnectionUrl == false) {
return String.format("%s/%s?%s", datasourceUrl, schemaName, connectionParameters);
return String.format("%s/%s?%s", dbConnectionUrl, schemaName, connectionParameters);
}
return datasourceUrl;
return dbConnectionUrl;
}
}
@ -42,7 +39,7 @@ public class DatasourceProperties {
.filter(e -> e.getUnitName().equals(unitName))
.findFirst()
.orElseThrow(
() -> new IllegalArgumentException(this.getClass().getName() + ": unitName Not found. " + unitName));
() -> new IllegalArgumentException(this.getClass().getName() + ": unitName Not found. " + unitName));
}
}

8
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/properties/HibernateProperties.java → shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/datasource/properties/HibernateProperties.java
View File

@ -1,14 +1,12 @@
package com.myoa.engineering.crawl.ppomppu.processor.configuration.properties;
package com.myoa.engineering.crawl.shopping.configuration.datasource.properties;
import java.util.List;
import lombok.Getter;
import lombok.Setter;
import org.hibernate.dialect.MySQL8Dialect;
import org.hibernate.dialect.MySQLDialect;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;
import java.util.List;
@Component
@Setter
@Getter

2
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/properties/HikariProperties.java → shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/datasource/properties/HikariProperties.java
View File

@ -1,4 +1,4 @@
package com.myoa.engineering.crawl.ppomppu.processor.configuration.properties;
package com.myoa.engineering.crawl.shopping.configuration.datasource.properties;
import lombok.Getter;
import lombok.Setter;

12
sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/configuration/properties/SlackSecretProperties.java → shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/slack/properties/SlackSecretProperties.java
View File

@ -1,4 +1,4 @@
package com.myoa.engineering.crawl.ppomppu.sender.configuration.properties;
package com.myoa.engineering.crawl.shopping.configuration.slack.properties;
import java.util.List;
import lombok.Data;
@ -7,18 +7,16 @@ import lombok.Setter;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;
@Getter
@Setter
@Data
@Component
@ConfigurationProperties("infra.slack.bot")
@ConfigurationProperties(prefix = "slack.bot")
public class SlackSecretProperties {
private List<SlackSecretPropertiesUnit> units;
@Data
public static class SlackSecretPropertiesUnit {
private String botName;
private String botUnitName;
private String username;
private String iconEmoji;
private String channel;
@ -27,7 +25,7 @@ public class SlackSecretProperties {
public SlackSecretPropertiesUnit find(String botUnitName) {
return units.stream()
.filter(e -> e.getBotName().equals(botUnitName))
.filter(e -> e.getBotUnitName().equals(botUnitName))
.findFirst()
.orElseThrow(() -> new IllegalArgumentException("not found bot unit name : " + botUnitName));
}

22
shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/controller/TestAPIController.java Normal file
View File

@ -0,0 +1,22 @@
package com.myoa.engineering.crawl.shopping.controller;
import com.myoa.engineering.crawl.shopping.crawlhandler.PpomppuCrawlDomesticHandler;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
@RestController
@RequestMapping("/api/v1/exploit")
public class TestAPIController {
private final PpomppuCrawlDomesticHandler ppomppuCrawlDomesticHandler;
public TestAPIController(PpomppuCrawlDomesticHandler ppomppuCrawlDomesticHandler) {
this.ppomppuCrawlDomesticHandler = ppomppuCrawlDomesticHandler;
}
@GetMapping("/triggers")
public void triggerExploit() {
ppomppuCrawlDomesticHandler.handle();
}
}

10
shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/crawlhandler/CrawlHandler.java Normal file
View File

@ -0,0 +1,10 @@
package com.myoa.engineering.crawl.shopping.crawlhandler;
import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget;
public interface CrawlHandler {
CrawlTarget getCrawlTarget();
void handle();
}

18
shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/crawlhandler/FmkoreaCrawlHandler.java Normal file
View File

@ -0,0 +1,18 @@
package com.myoa.engineering.crawl.shopping.crawlhandler;
import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
@Slf4j
@Component
public class FmkoreaCrawlHandler implements CrawlHandler {
@Override
public CrawlTarget getCrawlTarget() {
return CrawlTarget.FMKOREA;
}
@Override
public void handle() {
}
}

58
shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/crawlhandler/PpomppuCrawlDomesticHandler.java Normal file
View File

@ -0,0 +1,58 @@
package com.myoa.engineering.crawl.shopping.crawlhandler;
import com.myoa.engineering.crawl.shopping.crawlhandler.parser.PpomppuArticleParserV2;
import com.myoa.engineering.crawl.shopping.domain.entity.v2.Article;
import com.myoa.engineering.crawl.shopping.infra.client.ppomppu.PpomppuBoardClientV2;
import com.myoa.engineering.crawl.shopping.service.ArticleCommandService;
import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget;
import org.springframework.stereotype.Component;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Stream;
@Component
public class PpomppuCrawlDomesticHandler implements CrawlHandler {
private final PpomppuBoardClientV2 ppomppuBoardClient;
private final PpomppuArticleParserV2 ppomppuArticleParserV2;
private final ArticleCommandService articleCommandService;
public PpomppuCrawlDomesticHandler(PpomppuBoardClientV2 ppomppuBoardClient,
PpomppuArticleParserV2 ppomppuArticleParserV2,
ArticleCommandService articleCommandService) {
this.ppomppuBoardClient = ppomppuBoardClient;
this.ppomppuArticleParserV2 = ppomppuArticleParserV2;
this.articleCommandService = articleCommandService;
}
@Override
public CrawlTarget getCrawlTarget() {
return CrawlTarget.PPOMPPU_DOMESTIC;
}
@Override
public void handle() {
String boardHtmlPage1 = ppomppuBoardClient.getBoardHtml("/zboard/zboard.php", generateRequestParams(1));
List<Article> parsedPage1 = ppomppuArticleParserV2.parse(boardHtmlPage1);
String boardHtmlPage2 = ppomppuBoardClient.getBoardHtml("/zboard/zboard.php", generateRequestParams(2));
List<Article> parsedPage2 = ppomppuArticleParserV2.parse(boardHtmlPage2);
List<Article> merged = Stream.of(parsedPage1, parsedPage2)
.flatMap(List::stream)
.map(e -> e.updateCrawlTarget(getCrawlTarget()))
.toList();
articleCommandService.upsert(merged);
}
private Map<String, String> generateRequestParams(int pageId) {
Map<String, String> params = new HashMap<>();
params.put("id", "ppomppu");
params.put("page", String.valueOf(pageId));
return params;
}
}

115
shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/crawlhandler/parser/PpomppuArticleParserV2.java Normal file
View File

@ -0,0 +1,115 @@
package com.myoa.engineering.crawl.shopping.crawlhandler.parser;
import com.myoa.engineering.crawl.shopping.domain.entity.v2.Article;
import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName;
import com.myoa.engineering.crawl.shopping.util.DateTimeUtils;
import com.myoa.engineering.crawl.shopping.util.NumberUtils;
import io.micrometer.core.instrument.util.StringUtils;
import lombok.extern.slf4j.Slf4j;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.stereotype.Component;
import java.time.ZoneId;
import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@Slf4j
@Component
public final class PpomppuArticleParserV2 {
private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yy.MM.dd HH:mm:ss")
.withZone(ZoneId.of("Asia/Seoul"));
private PpomppuArticleParserV2() {
}
public List<Article> parse(String html) {
Elements trElements = converHtmlToTrElements(html);
return trElements.stream()
.filter(this::isRealArticle)
.map(this::parse)
.toList();
}
private Elements converHtmlToTrElements(String data) {
Document document = Jsoup.parse(data);
Elements trList = document.getElementById("revolution_main_table").getElementsByTag("tr");
return trList;
}
private boolean isRealArticle(Element tr) {
Elements tdList = tr.getElementsByTag("td");
if (tdList.size() != 6) {
return false;
}
if (!hasOnlyNumeric(tdList.get(0))) {
return false;
}
return true;
}
private Pattern pattern_numeric = Pattern.compile("\\d+");
private boolean hasOnlyNumeric(Element td) {
return pattern_numeric.matcher(td.text()).matches();
}
private Article parse(Element tr) {
Elements tdList = tr.getElementsByTag("td");
Long articleId = Long.parseLong(tdList.get(0).text());
String title = tdList.get(2).text();
String articleUrl = parseArticleUrl(tdList.get(2).getElementsByTag("a").attr("href"));
String boardName = parseBoardName(title);
Integer recommended = parseRecommended(tdList.get(4));
Integer hit = NumberUtils.parseInt(tdList.get(5).text(), 0);
ZonedDateTime registeredAt = DateTimeUtils.parse(tdList.get(3).text());
return Article.builder()
.articleId(articleId)
.title(title)
.boardName(boardName)
.articleUrl(articleUrl)
.recommended(recommended)
.hit(hit)
.registeredAt(registeredAt)
.build();
}
public Integer parseRecommended(Element td) {
final String voteString = td.text();
if (StringUtils.isEmpty(voteString)) {
return null;
}
final int voteUp = Integer.parseInt(td.text().split(" - ")[0]);
final int voteDown = Integer.parseInt(td.text().split(" - ")[1]);
int recommended = voteUp - voteDown;
return recommended;
}
public static String parseArticleUrl(String data) {
return PpomppuBoardName.ofViewPageUrl(data);
}
Pattern patternBoardName = Pattern.compile("\\[(.+?)\\]");
public String parseBoardName(String fullTitle) {
Matcher matcher = patternBoardName.matcher(fullTitle);
String lastMatched = null;
while (matcher.find()) {
lastMatched = matcher.group(1);
}
return lastMatched;
}
}

21
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/Auditable.java → shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/Auditable.java
View File

@ -1,17 +1,18 @@
package com.myoa.engineering.crawl.ppomppu.processor.domain;
package com.myoa.engineering.crawl.shopping.domain.entity;
import java.io.Serializable;
import java.time.Instant;
import javax.persistence.Column;
import lombok.Getter;
import org.springframework.data.annotation.CreatedDate;
import org.springframework.data.annotation.LastModifiedDate;
/**
* Auditable
*
* @author Shin Woo-jin (woozu.shin@kakaoent.com)
* @since 2021-09-08
*/
import jakarta.persistence.*;
import org.springframework.data.jpa.domain.support.AuditingEntityListener;
import java.io.Serializable;
import java.time.Instant;
@Getter
@MappedSuperclass
@EntityListeners(AuditingEntityListener.class)
public abstract class Auditable implements Serializable {
private static final long serialVersionUID = -7105030870015828551L;

2
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/BaseScanDomain.java → shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/BaseScanDomain.java
View File

@ -1,4 +1,4 @@
package com.myoa.engineering.crawl.ppomppu.processor.domain;
package com.myoa.engineering.crawl.shopping.domain.entity;
/**
* BaseScanDomain

27
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/PpomppuArticle.java → shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v1/PpomppuArticle.java
View File

@ -1,19 +1,16 @@
package com.myoa.engineering.crawl.ppomppu.processor.domain;
package com.myoa.engineering.crawl.shopping.domain.entity.v1;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import java.time.Instant;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.EnumType;
import javax.persistence.Enumerated;
import javax.persistence.GeneratedValue;
import javax.persistence.GenerationType;
import javax.persistence.Id;
import javax.persistence.Table;
import com.myoa.engineering.crawl.shopping.domain.entity.Auditable;
import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName;
import jakarta.persistence.*;
import lombok.Builder;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.ToString;
import java.time.ZonedDateTime;
@ToString
@Getter
@NoArgsConstructor
@Entity
@ -44,11 +41,11 @@ public class PpomppuArticle extends Auditable {
private Integer recommended;
@Column
private Instant registeredAt;
private ZonedDateTime registeredAt;
@Builder
public PpomppuArticle(Long id, Long articleId, PpomppuBoardName boardName, String articleUrl,
String title, Integer recommended, Integer hit, Instant registeredAt) {
String title, Integer recommended, Integer hit, ZonedDateTime registeredAt) {
this.id = id;
this.articleId = articleId;
this.boardName = boardName;
@ -59,8 +56,4 @@ public class PpomppuArticle extends Auditable {
this.registeredAt = registeredAt;
}
public PpomppuArticle updateBoardName(PpomppuBoardName boardName) {
this.boardName = boardName;
return this;
}
}

17
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/PpomppuBoardFeedStatus.java → shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v1/PpomppuBoardFeedStatus.java
View File

@ -1,19 +1,14 @@
package com.myoa.engineering.crawl.ppomppu.processor.domain;
package com.myoa.engineering.crawl.shopping.domain.entity.v1;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import java.time.Instant;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.EnumType;
import javax.persistence.Enumerated;
import javax.persistence.GeneratedValue;
import javax.persistence.GenerationType;
import javax.persistence.Id;
import javax.persistence.Table;
import com.myoa.engineering.crawl.shopping.domain.entity.Auditable;
import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName;
import jakarta.persistence.*;
import lombok.Builder;
import lombok.Getter;
import lombok.NoArgsConstructor;
import java.time.Instant;
@Getter
@NoArgsConstructor
@Entity

13
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/PublishedHistory.java → shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v1/PublishedHistory.java
View File

@ -1,15 +1,12 @@
package com.myoa.engineering.crawl.ppomppu.processor.domain;
package com.myoa.engineering.crawl.shopping.domain.entity.v1;
import java.time.Instant;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.GeneratedValue;
import javax.persistence.GenerationType;
import javax.persistence.Id;
import javax.persistence.Table;
import com.myoa.engineering.crawl.shopping.domain.entity.Auditable;
import lombok.Getter;
import lombok.NoArgsConstructor;
import jakarta.persistence.*;
import java.time.Instant;
@Getter
@NoArgsConstructor
@Entity

27
shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v1/SubscribedBoard.java Normal file
View File

@ -0,0 +1,27 @@
package com.myoa.engineering.crawl.shopping.domain.entity.v1;
import com.myoa.engineering.crawl.shopping.domain.entity.Auditable;
import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName;
import lombok.Getter;
import lombok.NoArgsConstructor;
import jakarta.persistence.*;
@Getter
@NoArgsConstructor
@Entity
@Table(name = "subscribed_board")
public class SubscribedBoard extends Auditable {
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
private Long id;
@Column
private Long userId;
@Column
@Enumerated(EnumType.STRING)
private PpomppuBoardName boardName;
}

15
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/SubscribedUser.java → shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v1/SubscribedUser.java
View File

@ -1,20 +1,17 @@
package com.myoa.engineering.crawl.ppomppu.processor.domain;
package com.myoa.engineering.crawl.shopping.domain.entity.v1;
import java.time.Instant;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.GeneratedValue;
import javax.persistence.GenerationType;
import javax.persistence.Id;
import javax.persistence.Table;
import com.myoa.engineering.crawl.shopping.domain.entity.Auditable;
import lombok.Getter;
import lombok.NoArgsConstructor;
import jakarta.persistence.*;
import java.time.Instant;
@Getter
@NoArgsConstructor
@Entity
@Table(name = "subscribed_user")
public class SubscribedUser extends Auditable{
public class SubscribedUser extends Auditable {
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)

26
shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v2/AppUser.java Normal file
View File

@ -0,0 +1,26 @@
package com.myoa.engineering.crawl.shopping.domain.entity.v2;
import com.myoa.engineering.crawl.shopping.domain.entity.Auditable;
import jakarta.persistence.*;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Getter;
import lombok.NoArgsConstructor;
@Getter
@Builder
@NoArgsConstructor
@AllArgsConstructor
@Entity
@Table
public class AppUser extends Auditable {
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
private Long id;
@Column
private String name;
@Column
private Boolean enabled;
}

63
shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v2/Article.java Normal file
View File

@ -0,0 +1,63 @@
package com.myoa.engineering.crawl.shopping.domain.entity.v2;
import com.myoa.engineering.crawl.shopping.domain.entity.Auditable;
import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget;
import jakarta.persistence.*;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Getter;
import lombok.NoArgsConstructor;
import java.time.ZonedDateTime;
@Getter
@Builder
@NoArgsConstructor
@AllArgsConstructor
@Entity
@Table
public class Article extends Auditable {
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
private Long id;
@Column
private Long articleId;
@Column
@Enumerated(EnumType.STRING)
private CrawlTarget crawlTarget;
@Column
private String boardName;
@Column
private String articleUrl;
@Column
private String title;
@Column
private Integer hit;
@Column
private Integer recommended;
@Column
private ZonedDateTime registeredAt;
public Article update(Article article) {
this.boardName = article.getBoardName();
this.articleUrl = article.getArticleUrl();
this.title = article.getTitle();
this.hit = article.getHit();
this.recommended = article.getRecommended();
return this;
}
public Article updateCrawlTarget(CrawlTarget crawlTarget) {
this.crawlTarget = crawlTarget;
return this;
}
}

30
shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v2/SubscribedKeyword.java Normal file
View File

@ -0,0 +1,30 @@
package com.myoa.engineering.crawl.shopping.domain.entity.v2;
import com.myoa.engineering.crawl.shopping.domain.entity.Auditable;
import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget;
import jakarta.persistence.*;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Getter;
import lombok.NoArgsConstructor;
@Getter
@Builder
@NoArgsConstructor
@AllArgsConstructor
@Entity
@Table
public class SubscribedKeyword extends Auditable {
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
private Long id;
@Column
private String keyword;
@Column
@Enumerated(EnumType.STRING)
private CrawlTarget crawlTarget;
}

27
shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/model/v2/ArticleModel.java Normal file
View File

@ -0,0 +1,27 @@
package com.myoa.engineering.crawl.shopping.domain.model.v2;
import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Getter;
import lombok.NoArgsConstructor;
import java.time.ZonedDateTime;
@Getter
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class ArticleModel {
private Long id;
private Long articleId;
private CrawlTarget crawlTarget;
private String boardName;
private String articleUrl;
private String title;
private Integer hit;
private Integer recommended;
private ZonedDateTime registeredAt;
}

9
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/FeedParsedResult.java → shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/FeedParsedResult.java
View File

@ -1,12 +1,13 @@
package com.myoa.engineering.crawl.ppomppu.processor.dto;
package com.myoa.engineering.crawl.shopping.dto;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import java.io.Serializable;
import java.time.Instant;
import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName;
import lombok.Builder;
import lombok.Getter;
import lombok.NoArgsConstructor;
import java.io.Serializable;
import java.time.Instant;
/**
* FeedParsedResult
*

2
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/PpomppuArticleDTO.java → shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/PpomppuArticleDTO.java
View File

@ -1,4 +1,4 @@
package com.myoa.engineering.crawl.ppomppu.processor.dto;
package com.myoa.engineering.crawl.shopping.dto;
/**
* PpomppuArticle

30
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/PpomppuArticleTransformer.java → shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/PpomppuArticleTransformer.java
View File

@ -1,38 +1,38 @@
package com.myoa.engineering.crawl.ppomppu.processor.dto;
package com.myoa.engineering.crawl.shopping.dto;
import com.myoa.engineering.crawl.shopping.domain.entity.v1.PpomppuArticle;
import com.myoa.engineering.crawl.shopping.support.dto.SimpleMessageDTO;
import java.time.Instant;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.util.List;
import java.util.function.Function;
import java.util.stream.Collectors;
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
import com.myoa.engineering.crawl.ppomppu.support.dto.SimpleMessageDTO;
/**
* PpomppuArticleTransformer
*
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-11-21
*
*/
public final class PpomppuArticleTransformer {
private PpomppuArticleTransformer() {}
private PpomppuArticleTransformer() {
}
private static final String MESSAGE_FORMAT_V1 = "%s)) `%s` <%s:LINK>";
private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")
.withZone(ZoneId.of("Asia/Seoul"));
/*
public static final Function<PpomppuArticle, SimpleMessageDTO> TRANSFORM_TO_MESSAGE_DTO = entity ->
SimpleMessageDTO.builder()
.requestedAt(Instant.now())
.publishedAt(entity.getRegisteredAt())
.title(String.format(MESSAGE_FORMAT_V1, entity.getBoardName().getMenuName(), entity.getTitle()))
.body(entity.getArticleUrl())
.build();
SimpleMessageDTO.builder()
.requestedAt(Instant.now())
.publishedAt(entity.getRegisteredAt())
.title(String.format(MESSAGE_FORMAT_V1, entity.getBoardName().getMenuName(), entity.getTitle()))
.body(entity.getArticleUrl())
.build();
*/
// https://stackoverflow.com/questions/24882927/using-streams-to-convert-a-list-of-objects-into-a-string-obtained-from-the-tostr
public static SimpleMessageDTO transform(List<PpomppuArticle> articles) {
Instant requestedAt = Instant.now();

2
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/constant/WebClientPropertiesUnitName.java → shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/constant/WebClientPropertiesUnitName.java
View File

@ -1,4 +1,4 @@
package com.myoa.engineering.crawl.ppomppu.processor.dto.constant;
package com.myoa.engineering.crawl.shopping.dto.constant;
import lombok.AllArgsConstructor;
import lombok.Getter;

2
sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/dto/MessageDTO.java → shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/slack/MessageDTO.java
View File

@ -1,4 +1,4 @@
package com.myoa.engineering.crawl.ppomppu.sender.dto;
package com.myoa.engineering.crawl.shopping.dto.slack;
import java.io.Serializable;

2
sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/dto/SlackMessageDTO.java → shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/slack/SlackMessageDTO.java
View File

@ -1,4 +1,4 @@
package com.myoa.engineering.crawl.ppomppu.sender.dto;
package com.myoa.engineering.crawl.shopping.dto.slack;
import com.fasterxml.jackson.annotation.JsonProperty;

12
shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/event/ArticleUpsertEvent.java Normal file
View File

@ -0,0 +1,12 @@
package com.myoa.engineering.crawl.shopping.event;
import org.springframework.context.ApplicationEvent;
public class ArticleUpsertEvent extends ApplicationEvent {
public ArticleUpsertEvent(Object source) {
super(source);
}
}

19
shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/event/ArticleUpsertEventCommand.java Normal file
View File

@ -0,0 +1,19 @@
package com.myoa.engineering.crawl.shopping.event;
import com.myoa.engineering.crawl.shopping.domain.model.v2.ArticleModel;
import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget;
import lombok.Getter;
import java.util.List;
@Getter
@Deprecated
public class ArticleUpsertEventCommand {
private final List<ArticleModel> articles;
private final CrawlTarget crawlTarget;
public ArticleUpsertEventCommand(List<ArticleModel> articles, CrawlTarget crawlTarget) {
this.articles = articles;
this.crawlTarget = crawlTarget;
}
}

23
shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/event/handler/ArticleUpsertEventListener.java Normal file
View File

@ -0,0 +1,23 @@
package com.myoa.engineering.crawl.shopping.event.handler;
import com.myoa.engineering.crawl.shopping.event.ArticleUpsertEvent;
import com.myoa.engineering.crawl.shopping.service.SubscribedKeywordQueryService;
import org.ahocorasick.trie.Trie;
import org.springframework.context.event.EventListener;
import org.springframework.stereotype.Component;
@Component
public class ArticleUpsertEventListener {
private final SubscribedKeywordQueryService subscribedKeywordQueryService;
public ArticleUpsertEventListener(SubscribedKeywordQueryService subscribedKeywordQueryService) {
this.subscribedKeywordQueryService = subscribedKeywordQueryService;
}
@EventListener
public void handleArticleUpsertEvent(ArticleUpsertEvent event) {
System.out.println("event = " + event);
}
}

23
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/client/MessageSenderAPIClient.java → shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/MessageSenderAPIClient.java
View File

@ -1,22 +1,7 @@
package com.myoa.engineering.crawl.ppomppu.processor.infrastructure.client;
import org.springframework.core.ParameterizedTypeReference;
import org.springframework.http.HttpHeaders;
import org.springframework.http.MediaType;
import org.springframework.stereotype.Component;
import org.springframework.web.reactive.function.client.WebClient;
import org.springframework.web.reactive.function.client.WebClientRequestException;
import com.myoa.engineering.crawl.ppomppu.processor.dto.constant.WebClientPropertiesUnitName;
import com.myoa.engineering.crawl.ppomppu.support.dto.SimpleMessageDTO;
import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebClientFilterFactory;
import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebFluxExchangeStragiesFactory;
import com.myoa.engineering.crawl.ppomppu.support.webclient.properties.WebClientProperties;
import com.myoa.engineering.crawl.ppomppu.support.webclient.properties.WebClientProperties.WebClientPropertiesUnit;
package com.myoa.engineering.crawl.shopping.infra.client;
import lombok.extern.slf4j.Slf4j;
import reactor.core.publisher.Mono;
import reactor.core.scheduler.Schedulers;
import org.springframework.stereotype.Component;
/**
* PpomppuNotifierSenderAPIClient
@ -27,7 +12,7 @@ import reactor.core.scheduler.Schedulers;
@Slf4j
@Component
public class MessageSenderAPIClient {
/*
private final WebClient webClient;
public MessageSenderAPIClient(WebClientProperties webClientProperties) {
@ -53,4 +38,6 @@ public class MessageSenderAPIClient {
return Mono.empty();
});
}
*/
}

12
shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/fmkorea/FmkoreaBoardClient.java Normal file
View File

@ -0,0 +1,12 @@
package com.myoa.engineering.crawl.shopping.infra.client.fmkorea;
import org.springframework.cloud.openfeign.FeignClient;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
@FeignClient(value = "fmkorea-board-client", url = "https://fmkorea.com")
public interface FmkoreaBoardClient {
@GetMapping("{boardLink}")
String getBoardHtml(@PathVariable("boardLink") String boardLink);
}

19
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/client/PpomppuBoardFeedRetriever.java → shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/ppomppu/PpomppuBoardClient.java
View File

@ -1,16 +1,7 @@
package com.myoa.engineering.crawl.ppomppu.processor.infrastructure.client;
package com.myoa.engineering.crawl.shopping.infra.client.ppomppu;
import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebClientFilterFactory;
import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebFluxExchangeStragiesFactory;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import lombok.extern.slf4j.Slf4j;
import org.springframework.core.ParameterizedTypeReference;
import org.springframework.stereotype.Component;
import org.springframework.web.reactive.function.client.WebClient;
import org.springframework.web.reactive.function.client.WebClientRequestException;
import reactor.core.publisher.Mono;
import reactor.core.scheduler.Schedulers;
/**
* PpomppuBoardFeedRetriever
@ -20,11 +11,11 @@ import reactor.core.scheduler.Schedulers;
*/
@Slf4j
@Component
public class PpomppuBoardFeedRetriever {
public class PpomppuBoardClient {
/*
private final WebClient webClient;
public PpomppuBoardFeedRetriever(WebClient.Builder webClientBuilder) {
public PpomppuBoardClient(WebClient.Builder webClientBuilder) {
this.webClient = webClientBuilder.baseUrl(PpomppuBoardName.PPOMPPU_URL)
.exchangeStrategies(WebFluxExchangeStragiesFactory.ofTextHtml())
.filter(WebClientFilterFactory.logRequest())
@ -43,5 +34,5 @@ public class PpomppuBoardFeedRetriever {
});
// .doOnNext(e -> log.info("[getHtml] {}", e));
}
*/
}

17
shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/ppomppu/PpomppuBoardClientV2.java Normal file
View File

@ -0,0 +1,17 @@
package com.myoa.engineering.crawl.shopping.infra.client.ppomppu;
import org.springframework.cloud.openfeign.FeignClient;
import org.springframework.cloud.openfeign.SpringQueryMap;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestParam;
import java.util.Map;
@FeignClient(value = "ppomppu-board-client", url = "https://www.ppomppu.co.kr")
public interface PpomppuBoardClientV2 {
@GetMapping("{boardLink}")
String getBoardHtml(@PathVariable("boardLink") String boardLink,
@SpringQueryMap Map<String, String> params);
}

21
sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/infrastructure/client/MongeShoppingBotSlackMessageSender.java → shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/slack/MongeShoppingBotSlackMessageSender.java
View File

@ -1,22 +1,25 @@
package com.myoa.engineering.crawl.ppomppu.sender.infrastructure.client;
package com.myoa.engineering.crawl.shopping.infra.client.slack;
import com.myoa.engineering.crawl.ppomppu.sender.configuration.properties.SlackSecretProperties;
import com.myoa.engineering.crawl.ppomppu.sender.configuration.properties.SlackSecretProperties.SlackSecretPropertiesUnit;
import com.myoa.engineering.crawl.ppomppu.sender.dto.SlackMessageDTO;
import com.myoa.engineering.crawl.shopping.configuration.slack.properties.SlackSecretProperties;
import com.myoa.engineering.crawl.shopping.dto.slack.SlackMessageDTO;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
@Slf4j
@Component
public class MongeShoppingBotSlackMessageSender extends SlackMessageSender {
public class MongeShoppingBotSlackMessageSender {
private static final String SLACK_SECRET_UNIT_NAME = "monge_shopping_bot";
private static final String SLACK_SECRET_UNIT_NAME = "shopping-crawler";
private final SlackSecretPropertiesUnit slackProperties;
private final SlackSecretProperties.SlackSecretPropertiesUnit slackProperties;
private final SlackAPIClient slackAPIClient;
private final String token;
public MongeShoppingBotSlackMessageSender(SlackSecretProperties slackSecretProperties) {
super(slackSecretProperties.find(SLACK_SECRET_UNIT_NAME).getToken());
public MongeShoppingBotSlackMessageSender(SlackAPIClient slackAPIClient,
SlackSecretProperties slackSecretProperties) {
this.slackAPIClient = slackAPIClient;
this.slackProperties = slackSecretProperties.find(SLACK_SECRET_UNIT_NAME);
this.token = slackProperties.getToken();
}
public SlackMessageDTO ofMessageTemplate() {

11
shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/slack/SlackAPIClient.java Normal file
View File

@ -0,0 +1,11 @@
package com.myoa.engineering.crawl.shopping.infra.client.slack;
import org.springframework.cloud.openfeign.FeignClient;
import org.springframework.web.bind.annotation.PostMapping;
@FeignClient(value = "slack-api-client", url = "https://slack.com")
public interface SlackAPIClient {
@PostMapping("/api/v1/messages/sendMessage/messengers/slack")
String sendMessage();
}

18
sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/infrastructure/client/SlackMessageSender.java → shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/slack/SlackMessageSender.java
View File

@ -1,17 +1,6 @@
package com.myoa.engineering.crawl.ppomppu.sender.infrastructure.client;
import org.springframework.http.HttpHeaders;
import org.springframework.http.MediaType;
import org.springframework.web.reactive.function.client.WebClient;
import org.springframework.web.reactive.function.client.WebClientRequestException;
import com.myoa.engineering.crawl.ppomppu.sender.dto.SlackMessageDTO;
import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebClientFilterFactory;
import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebFluxExchangeStragiesFactory;
package com.myoa.engineering.crawl.shopping.infra.client.slack;
import lombok.extern.slf4j.Slf4j;
import reactor.core.publisher.Mono;
import reactor.core.scheduler.Schedulers;
/**
* SlackMessageSender
@ -20,8 +9,9 @@ import reactor.core.scheduler.Schedulers;
* @since 2021-09-08
*/
@Slf4j
public class SlackMessageSender implements MessageSender<SlackMessageDTO> {
public class SlackMessageSender { /* implements MessageSender<SlackMessageDTO> {*/
/*
private static final String SLACK_API_URL = "https://slack.com/api";
private final WebClient webClient;
@ -50,4 +40,6 @@ public class SlackMessageSender implements MessageSender<SlackMessageDTO> {
});
}
*/
}

4
shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/repository/BaseScanRepository.java Normal file
View File

@ -0,0 +1,4 @@
package com.myoa.engineering.crawl.shopping.infra.repository;
public interface BaseScanRepository {
}

4
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/repository/PpomppuArticleRepository.java → shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/repository/v1/PpomppuArticleRepository.java
View File

@ -1,6 +1,6 @@
package com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository;
package com.myoa.engineering.crawl.shopping.infra.repository.v1;
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
import com.myoa.engineering.crawl.shopping.domain.entity.v1.PpomppuArticle;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.stereotype.Repository;

9
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/repository/PpomppuBoardFeedStatusRepository.java → shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/repository/v1/PpomppuBoardFeedStatusRepository.java
View File

@ -1,11 +1,12 @@
package com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository;
package com.myoa.engineering.crawl.shopping.infra.repository.v1;
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuBoardFeedStatus;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import java.util.Optional;
import com.myoa.engineering.crawl.shopping.domain.entity.v1.PpomppuBoardFeedStatus;
import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.stereotype.Repository;
import java.util.Optional;
@Repository
public interface PpomppuBoardFeedStatusRepository extends JpaRepository<PpomppuBoardFeedStatus, Long> {

9
shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/repository/v2/AppUserRepository.java Normal file
View File

@ -0,0 +1,9 @@
package com.myoa.engineering.crawl.shopping.infra.repository.v2;
import com.myoa.engineering.crawl.shopping.domain.entity.v2.AppUser;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.stereotype.Repository;
@Repository
public interface AppUserRepository extends JpaRepository<AppUser, Long> {
}

16
shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/repository/v2/ArticleRepository.java Normal file
View File

@ -0,0 +1,16 @@
package com.myoa.engineering.crawl.shopping.infra.repository.v2;
import com.myoa.engineering.crawl.shopping.domain.entity.v2.Article;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.stereotype.Repository;
import java.util.Collection;
import java.util.List;
import java.util.Optional;
@Repository
public interface ArticleRepository extends JpaRepository<Article, Long> {
Optional<Article> findByArticleId(Long articleId);
List<Article> findByArticleIdIn(Collection<Long> articleId);
}

14
shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/repository/v2/SubscribedKeywordRepository.java Normal file
View File

@ -0,0 +1,14 @@
package com.myoa.engineering.crawl.shopping.infra.repository.v2;
import com.myoa.engineering.crawl.shopping.domain.entity.v2.SubscribedKeyword;
import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.stereotype.Repository;
import java.util.List;
@Repository
public interface SubscribedKeywordRepository extends JpaRepository<SubscribedKeyword, Long> {
List<SubscribedKeyword> findByCrawlTarget(CrawlTarget crawlTarget);
}

27
shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/scheduler/ParseEventEmitter.java Normal file
View File

@ -0,0 +1,27 @@
package com.myoa.engineering.crawl.shopping.scheduler;
import com.myoa.engineering.crawl.shopping.crawlhandler.CrawlHandler;
import lombok.extern.slf4j.Slf4j;
import org.springframework.scheduling.annotation.EnableScheduling;
import org.springframework.stereotype.Component;
import java.util.List;
@Slf4j
@Component
@EnableScheduling
public class ParseEventEmitter {
private final List<CrawlHandler> crawlHandlers;
public ParseEventEmitter(List<CrawlHandler> crawlHandlers) {
this.crawlHandlers = crawlHandlers;
}
// @Scheduled(cron = "0 0/5 * * * ?")
public void emit() {
log.info("[emitDomesticBoard] trigger fired!");
crawlHandlers.forEach(CrawlHandler::handle);
}
}

81
shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/ArticleCommandService.java Normal file
View File

@ -0,0 +1,81 @@
package com.myoa.engineering.crawl.shopping.service;
import com.myoa.engineering.crawl.shopping.domain.entity.v2.Article;
import com.myoa.engineering.crawl.shopping.domain.model.v2.ArticleModel;
import com.myoa.engineering.crawl.shopping.event.ArticleUpsertEvent;
import com.myoa.engineering.crawl.shopping.infra.repository.v2.ArticleRepository;
import jakarta.transaction.Transactional;
import org.springframework.context.ApplicationEventPublisher;
import org.springframework.stereotype.Service;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.function.Function;
import java.util.stream.Collectors;
@Service
public class ArticleCommandService {
private final ArticleRepository articleRepository;
private final ApplicationEventPublisher applicationEventPublisher;
public ArticleCommandService(ArticleRepository articleRepository,
ApplicationEventPublisher applicationEventPublisher) {
this.articleRepository = articleRepository;
this.applicationEventPublisher = applicationEventPublisher;
}
@Transactional
public void upsert(List<Article> articles) {
Map<Long, Article> articleMap = articles.stream()
.collect(Collectors.toMap(Article::getArticleId, e -> e));
List<Article> saved = articleRepository.findByArticleIdIn(articleMap.keySet());
List<Article> updated = saved.stream()
.filter(e -> articleMap.containsKey(e.getArticleId()))
.map(e -> e.update(articleMap.get(e.getArticleId())))
.peek(e -> articleMap.remove(e.getArticleId()))
.sorted(Comparator.comparing(Article::getArticleId))
.toList();
List<Article> newArticles = articleMap.values()
.stream()
.sorted(Comparator.comparing(Article::getArticleId))
.toList();
articleRepository.saveAll(updated);
articleRepository.saveAll(newArticles);
publish(newArticles);
}
private void publish(List<Article> articles) {
List<ArticleModel> articleModels =
articles.stream()
.map(transformer)
.toList();
applicationEventPublisher.publishEvent(new ArticleUpsertEvent(articleModels));
}
@Transactional
public void upsert(Article article) {
Article saved = articleRepository.findByArticleId(article.getArticleId())
.orElse(article);
articleRepository.save(saved);
}
public static Function<Article, ArticleModel> transformer =
article -> ArticleModel.builder()
.id(article.getId())
.articleId(article.getArticleId())
.crawlTarget(article.getCrawlTarget())
.boardName(article.getBoardName())
.articleUrl(article.getArticleUrl())
.title(article.getTitle())
.hit(article.getHit())
.recommended(article.getRecommended())
.registeredAt(article.getRegisteredAt())
.build();
}

18
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/MessageSenderService.java → shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/MessageSenderService.java
View File

@ -1,15 +1,7 @@
package com.myoa.engineering.crawl.ppomppu.processor.service;
import java.util.List;
import org.springframework.stereotype.Service;
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
import com.myoa.engineering.crawl.ppomppu.processor.dto.PpomppuArticleTransformer;
import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.client.MessageSenderAPIClient;
package com.myoa.engineering.crawl.shopping.service;
import lombok.extern.slf4j.Slf4j;
import reactor.core.publisher.Mono;
import org.springframework.stereotype.Service;
/**
* MessageSenderService
@ -21,18 +13,20 @@ import reactor.core.publisher.Mono;
@Service
public class MessageSenderService {
/*
private final MessageSenderAPIClient messageSenderAPIClient;
public MessageSenderService(MessageSenderAPIClient messageSenderAPIClient) {
this.messageSenderAPIClient = messageSenderAPIClient;
}
public Mono<String> sendMessageToSlack(PpomppuArticle article) {
public String sendMessageToSlack(PpomppuArticle article) {
return messageSenderAPIClient.sendMessageToSlack(PpomppuArticleTransformer.TRANSFORM_TO_MESSAGE_DTO.apply(article));
}
public Mono<String> sendMessageToSlack(List<PpomppuArticle> articles) {
public String sendMessageToSlack(List<PpomppuArticle> articles) {
return messageSenderAPIClient.sendMessageToSlack(PpomppuArticleTransformer.transform(articles));
}
*/
}

22
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/PpomppuArticleService.java → shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/PpomppuArticleService.java
View File

@ -1,20 +1,18 @@
package com.myoa.engineering.crawl.ppomppu.processor.service;
package com.myoa.engineering.crawl.shopping.service;
import com.myoa.engineering.crawl.shopping.domain.entity.v1.PpomppuArticle;
import com.myoa.engineering.crawl.shopping.domain.entity.v1.PpomppuBoardFeedStatus;
import com.myoa.engineering.crawl.shopping.infra.repository.v1.PpomppuArticleRepository;
import com.myoa.engineering.crawl.shopping.infra.repository.v1.PpomppuBoardFeedStatusRepository;
import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import java.util.List;
import java.util.Optional;
import java.util.stream.Collectors;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuBoardFeedStatus;
import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository.PpomppuArticleRepository;
import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository.PpomppuBoardFeedStatusRepository;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Service
public class PpomppuArticleService {

16
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/PpomppuFeedService.java → shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/PpomppuFeedService.java
View File

@ -1,16 +1,12 @@
package com.myoa.engineering.crawl.ppomppu.processor.service;
package com.myoa.engineering.crawl.shopping.service;
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
import com.myoa.engineering.crawl.ppomppu.processor.dto.PpomppuArticleParser;
import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.client.PpomppuBoardFeedRetriever;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import java.util.List;
import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName;
import lombok.extern.slf4j.Slf4j;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Element;
import org.springframework.stereotype.Component;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
import java.util.List;
/**
* PpomppuFeedService
@ -21,7 +17,7 @@ import reactor.core.publisher.Mono;
@Slf4j
@Component
public class PpomppuFeedService {
/*
private final PpomppuBoardFeedRetriever ppomppuBoardFeedRetriever;
public PpomppuFeedService(PpomppuBoardFeedRetriever ppomppuBoardFeedRetriever) {
@ -56,4 +52,6 @@ public class PpomppuFeedService {
private PpomppuArticle convertFromElement(Element element) {
return PpomppuArticleParser.toArticle(element.getElementsByTag("td"));
}
*/
}

26
shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/SubscribedKeywordQueryService.java Normal file
View File

@ -0,0 +1,26 @@
package com.myoa.engineering.crawl.shopping.service;
import com.myoa.engineering.crawl.shopping.domain.entity.v2.SubscribedKeyword;
import com.myoa.engineering.crawl.shopping.infra.repository.v2.SubscribedKeywordRepository;
import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget;
import org.springframework.stereotype.Service;
import java.util.List;
@Service
public class SubscribedKeywordQueryService {
private final SubscribedKeywordRepository subscribedKeywordRepository;
public SubscribedKeywordQueryService(SubscribedKeywordRepository subscribedKeywordRepository) {
this.subscribedKeywordRepository = subscribedKeywordRepository;
}
public List<SubscribedKeyword> findAll() {
return subscribedKeywordRepository.findAll();
}
public List<SubscribedKeyword> findByCrawlTarget(CrawlTarget crawlTarget) {
return subscribedKeywordRepository.findByCrawlTarget(crawlTarget);
}
}

27
shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/util/DateTimeUtils.java Normal file
View File

@ -0,0 +1,27 @@
package com.myoa.engineering.crawl.shopping.util;
import java.time.*;
import java.time.format.DateTimeFormatter;
public final class DateTimeUtils {
private static final DateTimeFormatter FORMATTER_HHMMss = DateTimeFormatter.ofPattern("HH:mm:ss");
private static final ZoneId ZONE_ASIA_SEOUL = ZoneId.of("Asia/Seoul");
private DateTimeUtils() {
}
public static ZonedDateTime parse(String HHMMss) {
try {
LocalTime time = LocalTime.parse(HHMMss, FORMATTER_HHMMss);
LocalDateTime dateTime = LocalDateTime.of(LocalDate.now(), time);
if (dateTime.isAfter(LocalDateTime.now())) {
dateTime = dateTime.minusDays(1);
}
return dateTime.atZone(ZONE_ASIA_SEOUL);
} catch (Exception e) {
return null;
}
}
}

32
shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/util/NumberUtils.java Normal file
View File

@ -0,0 +1,32 @@
package com.myoa.engineering.crawl.shopping.util;
public final class NumberUtils {
private NumberUtils() {
}
public static Integer parseInt(String value) {
return parseInt(value, null);
}
public static Integer parseInt(String value, Integer defaultValue) {
try {
return Integer.parseInt(value);
} catch (NumberFormatException e) {
return defaultValue;
}
}
public static Long parseLong(String value) {
return parseLong(value, null);
}
public static Long parseLong(String value, Long defaultValue) {
try {
return Long.parseLong(value);
} catch (NumberFormatException e) {
return defaultValue;
}
}
}

Some files were not shown because too many files have changed in this diff Show More