[NO-ISSUE] Implement v2
This commit is contained in:
parent
0524a18ee5
commit
0c4be3cc05
13
build.gradle
13
build.gradle
|
@ -1,13 +1,14 @@
|
|||
plugins {
|
||||
id 'org.springframework.boot' version '2.5.4'
|
||||
id 'io.spring.dependency-management' version '1.0.11.RELEASE'
|
||||
id 'java'
|
||||
id 'idea'
|
||||
id 'org.springframework.boot' version '3.2.5'
|
||||
id 'io.spring.dependency-management' version '1.1.4'
|
||||
}
|
||||
|
||||
group = 'com.myoa.engineering.crawl.ppomppu'
|
||||
version = '1.0.1'
|
||||
sourceCompatibility = '11'
|
||||
sourceCompatibility = JavaVersion.VERSION_21
|
||||
targetCompatibility = JavaVersion.VERSION_21
|
||||
|
||||
configurations {
|
||||
compileOnly {
|
||||
|
@ -20,8 +21,8 @@ repositories {
|
|||
}
|
||||
|
||||
allprojects {
|
||||
group = 'com.myoa.engineering.crawl.ppomppu'
|
||||
version = '1.0.1'
|
||||
group = 'com.myoa.engineering.crawl.shopping'
|
||||
version = '2.0.0'
|
||||
|
||||
apply plugin: 'java'
|
||||
apply plugin: 'idea'
|
||||
|
@ -36,7 +37,7 @@ allprojects {
|
|||
}
|
||||
|
||||
ext {
|
||||
set('springCloudVersion', "2020.0.4")
|
||||
set('springCloudVersion', "2023.0.1")
|
||||
}
|
||||
|
||||
dependencyManagement {
|
||||
|
|
|
@ -1,5 +1,6 @@
|
|||
#Sun Apr 28 23:47:38 KST 2024
|
||||
distributionBase=GRADLE_USER_HOME
|
||||
distributionPath=wrapper/dists
|
||||
distributionUrl=https\://services.gradle.org/distributions/gradle-7.1.1-bin.zip
|
||||
distributionUrl=https\://services.gradle.org/distributions/gradle-8.5-bin.zip
|
||||
zipStoreBase=GRADLE_USER_HOME
|
||||
zipStorePath=wrapper/dists
|
||||
|
|
Binary file not shown.
|
@ -12,14 +12,15 @@ dependencies {
|
|||
implementation 'org.springframework.boot:spring-boot-starter-jdbc'
|
||||
implementation 'org.springframework.cloud:spring-cloud-starter-config'
|
||||
implementation 'org.springframework.boot:spring-boot-starter-actuator'
|
||||
implementation 'com.rometools:rome:1.16.0'
|
||||
implementation 'org.jsoup:jsoup:1.14.2'
|
||||
implementation 'com.h2database:h2:1.4.200'
|
||||
implementation 'com.rometools:rome:2.1.0'
|
||||
implementation 'org.jsoup:jsoup:1.15.3'
|
||||
implementation 'com.h2database:h2:2.2.220'
|
||||
|
||||
annotationProcessor 'org.springframework.boot:spring-boot-configuration-processor'
|
||||
annotationProcessor 'org.projectlombok:lombok'
|
||||
|
||||
testImplementation 'org.springframework.boot:spring-boot-starter-test'
|
||||
testImplementation("org.assertj:assertj-core:3.24.2")
|
||||
}
|
||||
|
||||
test {
|
||||
|
|
|
@ -4,7 +4,7 @@ import org.springframework.boot.SpringApplication;
|
|||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||
import org.springframework.context.annotation.Import;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.support.webclient.PpomppuNotifierWebClientConfiguration;
|
||||
import com.myoa.engineering.crawl.shopping.support.webclient.PpomppuNotifierWebClientConfiguration;
|
||||
|
||||
/**
|
||||
* ProcessorApplication
|
||||
|
|
|
@ -1,6 +1,5 @@
|
|||
package com.myoa.engineering.crawl.ppomppu.processor.controller;
|
||||
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
|
@ -10,8 +9,8 @@ import com.myoa.engineering.crawl.ppomppu.processor.dto.FeedParsedResult;
|
|||
import com.myoa.engineering.crawl.ppomppu.processor.service.MessageSenderService;
|
||||
import com.myoa.engineering.crawl.ppomppu.processor.service.PpomppuArticleService;
|
||||
import com.myoa.engineering.crawl.ppomppu.processor.service.PpomppuFeedService;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.dto.APIResponse;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
|
||||
import com.myoa.engineering.crawl.shopping.support.dto.APIResponse;
|
||||
import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import reactor.core.publisher.Mono;
|
||||
|
|
|
@ -1,32 +0,0 @@
|
|||
package com.myoa.engineering.crawl.ppomppu.processor.domain;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
|
||||
import javax.persistence.Column;
|
||||
import javax.persistence.Entity;
|
||||
import javax.persistence.EnumType;
|
||||
import javax.persistence.Enumerated;
|
||||
import javax.persistence.GeneratedValue;
|
||||
import javax.persistence.GenerationType;
|
||||
import javax.persistence.Id;
|
||||
import javax.persistence.Table;
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Getter
|
||||
@NoArgsConstructor
|
||||
@Entity
|
||||
@Table(name = "subscribed_board")
|
||||
public class SubscribedBoard extends Auditable {
|
||||
|
||||
@Id
|
||||
@GeneratedValue(strategy = GenerationType.IDENTITY)
|
||||
private Long id;
|
||||
|
||||
@Column
|
||||
private Long userId;
|
||||
|
||||
@Column
|
||||
@Enumerated(EnumType.STRING)
|
||||
private PpomppuBoardName boardName;
|
||||
|
||||
}
|
|
@ -1,4 +0,0 @@
|
|||
package com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository;
|
||||
|
||||
public interface BaseScanRepository {
|
||||
}
|
|
@ -1,4 +1,4 @@
|
|||
package com.myoa.engineering.crawl.ppomppu.processor.dto;
|
||||
package com.myoa.engineering.crawl.ppomppu.processor.parser;
|
||||
|
||||
import java.time.Instant;
|
||||
import java.time.ZoneId;
|
||||
|
@ -8,7 +8,7 @@ import org.jsoup.nodes.Element;
|
|||
import org.jsoup.select.Elements;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
|
||||
import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName;
|
||||
|
||||
/**
|
||||
* PpomppuArticleTransformer
|
File diff suppressed because it is too large
Load Diff
|
@ -1,12 +1,10 @@
|
|||
package com.myoa.engineering.crawl.ppomppu.receiver;
|
||||
package com.myoa.engineering.crawl.ppomppu.reader;
|
||||
|
||||
import org.springframework.boot.SpringApplication;
|
||||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||
import org.springframework.boot.context.properties.EnableConfigurationProperties;
|
||||
import org.springframework.context.annotation.Import;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.receiver.configuration.properties.TelegramBotProperties;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.webclient.PpomppuNotifierWebClientConfiguration;
|
||||
import com.myoa.engineering.crawl.shopping.support.webclient.PpomppuNotifierWebClientConfiguration;
|
||||
|
||||
/**
|
||||
* ReceiverApplication
|
||||
|
@ -17,9 +15,9 @@ import com.myoa.engineering.crawl.ppomppu.support.webclient.PpomppuNotifierWebCl
|
|||
@Import({ PpomppuNotifierWebClientConfiguration.class})
|
||||
@SpringBootApplication
|
||||
// @EnableConfigurationProperties({ TelegramBotProperties.class })
|
||||
public class ReceiverApplication {
|
||||
public class ReaderApplication {
|
||||
|
||||
public static void main(String[] args) {
|
||||
SpringApplication.run(ReceiverApplication.class, args);
|
||||
SpringApplication.run(ReaderApplication.class, args);
|
||||
}
|
||||
}
|
|
@ -1,4 +1,4 @@
|
|||
package com.myoa.engineering.crawl.ppomppu.receiver.configuration;
|
||||
package com.myoa.engineering.crawl.ppomppu.reader.configuration;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
|
@ -8,10 +8,10 @@ import org.telegram.telegrambots.meta.TelegramBotsApi;
|
|||
import org.telegram.telegrambots.meta.exceptions.TelegramApiException;
|
||||
import org.telegram.telegrambots.updatesreceivers.DefaultBotSession;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.receiver.configuration.properties.TelegramBotProperties;
|
||||
import com.myoa.engineering.crawl.ppomppu.receiver.configuration.properties.TelegramBotProperties.TelegramBotPropertiesUnit;
|
||||
import com.myoa.engineering.crawl.ppomppu.receiver.dispatch.MessageDispatcher;
|
||||
import com.myoa.engineering.crawl.ppomppu.receiver.handler.message.MessageHandler;
|
||||
import com.myoa.engineering.crawl.ppomppu.reader.configuration.properties.TelegramBotProperties;
|
||||
import com.myoa.engineering.crawl.ppomppu.reader.configuration.properties.TelegramBotProperties.TelegramBotPropertiesUnit;
|
||||
import com.myoa.engineering.crawl.ppomppu.reader.dispatch.MessageDispatcher;
|
||||
import com.myoa.engineering.crawl.ppomppu.reader.handler.message.MessageHandler;
|
||||
|
||||
/**
|
||||
* TelegramBotConfiguration
|
|
@ -1,16 +1,13 @@
|
|||
package com.myoa.engineering.crawl.ppomppu.receiver.configuration.properties;
|
||||
package com.myoa.engineering.crawl.ppomppu.reader.configuration.properties;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
import org.springframework.boot.context.properties.ConstructorBinding;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.Setter;
|
||||
|
||||
/**
|
|
@ -1,12 +1,12 @@
|
|||
package com.myoa.engineering.crawl.ppomppu.receiver.controller.v1;
|
||||
package com.myoa.engineering.crawl.ppomppu.reader.controller.v1;
|
||||
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.receiver.service.ProcessorAPIService;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
|
||||
import com.myoa.engineering.crawl.ppomppu.reader.service.ProcessorAPIService;
|
||||
import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName;
|
||||
|
||||
import reactor.core.publisher.Mono;
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
package com.myoa.engineering.crawl.ppomppu.receiver.dispatch;
|
||||
package com.myoa.engineering.crawl.ppomppu.reader.dispatch;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
|
@ -6,7 +6,7 @@ import org.telegram.telegrambots.bots.TelegramLongPollingBot;
|
|||
import org.telegram.telegrambots.meta.api.objects.Message;
|
||||
import org.telegram.telegrambots.meta.api.objects.Update;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.receiver.handler.message.MessageHandler;
|
||||
import com.myoa.engineering.crawl.ppomppu.reader.handler.message.MessageHandler;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
package com.myoa.engineering.crawl.ppomppu.receiver.handler.message;
|
||||
package com.myoa.engineering.crawl.ppomppu.reader.handler.message;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.support.util.ObjectUtil;
|
||||
import com.myoa.engineering.crawl.shopping.support.util.ObjectUtil;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.telegram.telegrambots.meta.api.objects.Message;
|
|
@ -1,4 +1,4 @@
|
|||
package com.myoa.engineering.crawl.ppomppu.receiver.handler.message;
|
||||
package com.myoa.engineering.crawl.ppomppu.reader.handler.message;
|
||||
|
||||
/**
|
||||
* ImageHandler
|
|
@ -1,4 +1,4 @@
|
|||
package com.myoa.engineering.crawl.ppomppu.receiver.handler.message;
|
||||
package com.myoa.engineering.crawl.ppomppu.reader.handler.message;
|
||||
|
||||
import org.telegram.telegrambots.meta.api.objects.Message;
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
package com.myoa.engineering.crawl.ppomppu.receiver.handler.message;
|
||||
package com.myoa.engineering.crawl.ppomppu.reader.handler.message;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.support.util.ObjectUtil;
|
||||
import com.myoa.engineering.crawl.shopping.support.util.ObjectUtil;
|
||||
import org.telegram.telegrambots.meta.api.objects.Message;
|
||||
|
||||
/**
|
|
@ -1,11 +1,11 @@
|
|||
package com.myoa.engineering.crawl.ppomppu.receiver.handler.message.text;
|
||||
package com.myoa.engineering.crawl.ppomppu.reader.handler.message.text;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.telegram.telegrambots.meta.api.objects.Message;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.receiver.handler.message.TextMessageHandler;
|
||||
import com.myoa.engineering.crawl.ppomppu.reader.handler.message.TextMessageHandler;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
|
@ -1,9 +1,9 @@
|
|||
package com.myoa.engineering.crawl.ppomppu.receiver.handler.message.text;
|
||||
package com.myoa.engineering.crawl.ppomppu.reader.handler.message.text;
|
||||
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.telegram.telegrambots.meta.api.objects.Message;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.receiver.handler.message.TextMessageHandler;
|
||||
import com.myoa.engineering.crawl.ppomppu.reader.handler.message.TextMessageHandler;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
package com.myoa.engineering.crawl.ppomppu.receiver.handler.message.text;
|
||||
package com.myoa.engineering.crawl.ppomppu.reader.handler.message.text;
|
||||
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.telegram.telegrambots.meta.api.objects.Message;
|
|
@ -1,4 +1,4 @@
|
|||
package com.myoa.engineering.crawl.ppomppu.receiver.handler.message.text;
|
||||
package com.myoa.engineering.crawl.ppomppu.reader.handler.message.text;
|
||||
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.telegram.telegrambots.meta.api.objects.Message;
|
|
@ -1,4 +1,4 @@
|
|||
package com.myoa.engineering.crawl.ppomppu.receiver.handler.message.text;
|
||||
package com.myoa.engineering.crawl.ppomppu.reader.handler.message.text;
|
||||
|
||||
import java.util.Arrays;
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
package com.myoa.engineering.crawl.ppomppu.receiver.handler.message.text;
|
||||
package com.myoa.engineering.crawl.ppomppu.reader.handler.message.text;
|
||||
|
||||
import org.telegram.telegrambots.meta.api.objects.Message;
|
||||
|
|
@ -1,15 +1,15 @@
|
|||
package com.myoa.engineering.crawl.ppomppu.receiver.infrastructure.client;
|
||||
package com.myoa.engineering.crawl.ppomppu.reader.infrastructure.client;
|
||||
|
||||
import org.springframework.core.ParameterizedTypeReference;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.web.reactive.function.client.WebClient;
|
||||
import org.springframework.web.reactive.function.client.WebClientRequestException;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.receiver.dto.constant.WebClientPropertiesUnitName;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebClientFilterFactory;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.webclient.properties.WebClientProperties;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.webclient.properties.WebClientProperties.WebClientPropertiesUnit;
|
||||
import com.myoa.engineering.crawl.ppomppu.reader.dto.constant.WebClientPropertiesUnitName;
|
||||
import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName;
|
||||
import com.myoa.engineering.crawl.shopping.support.webclient.factory.WebClientFilterFactory;
|
||||
import com.myoa.engineering.crawl.shopping.support.webclient.properties.WebClientProperties;
|
||||
import com.myoa.engineering.crawl.shopping.support.webclient.properties.WebClientProperties.WebClientPropertiesUnit;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import reactor.core.publisher.Mono;
|
|
@ -1,11 +1,11 @@
|
|||
package com.myoa.engineering.crawl.ppomppu.receiver.scheduler;
|
||||
package com.myoa.engineering.crawl.ppomppu.reader.scheduler;
|
||||
|
||||
import org.springframework.scheduling.annotation.EnableScheduling;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.receiver.service.ProcessorAPIService;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
|
||||
import com.myoa.engineering.crawl.ppomppu.reader.service.ProcessorAPIService;
|
||||
import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
|
@ -1,9 +1,9 @@
|
|||
package com.myoa.engineering.crawl.ppomppu.receiver.service;
|
||||
package com.myoa.engineering.crawl.ppomppu.reader.service;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.receiver.infrastructure.client.ProcessorAPIWebClient;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
|
||||
import com.myoa.engineering.crawl.ppomppu.reader.infrastructure.client.ProcessorAPIWebClient;
|
||||
import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import reactor.core.publisher.Mono;
|
Binary file not shown.
|
@ -1,25 +0,0 @@
|
|||
package com.myoa.engineering.crawl.ppomppu.receiver.dto;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
|
||||
import java.io.Serializable;
|
||||
import java.time.Instant;
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
/**
|
||||
* FeedParsedResult
|
||||
*
|
||||
* @author Shin Woo-jin (woozu.shin@kakaoent.com)
|
||||
* @since 2021-09-08
|
||||
*/
|
||||
@Getter
|
||||
@NoArgsConstructor
|
||||
public class FeedParsedResult implements Serializable {
|
||||
|
||||
private static final long serialVersionUID = -3771310078623481348L;
|
||||
|
||||
private PpomppuBoardName boardName;
|
||||
private Instant requestedAt;
|
||||
private Instant processedAt;
|
||||
|
||||
}
|
|
@ -1,19 +0,0 @@
|
|||
package com.myoa.engineering.crawl.ppomppu.receiver.dto.constant;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Getter;
|
||||
|
||||
/**
|
||||
* WebClientPropertiesUnitName
|
||||
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
|
||||
* @since 2021-11-18
|
||||
*
|
||||
*/
|
||||
@Getter
|
||||
@AllArgsConstructor
|
||||
public enum WebClientPropertiesUnitName {
|
||||
PPOMPPU_NOTIFIER_PROCESSOR_API("ppn-processor-api"),
|
||||
;
|
||||
|
||||
private String unitName;
|
||||
}
|
BIN
sender-1.0.1.jar
BIN
sender-1.0.1.jar
Binary file not shown.
|
@ -1,22 +0,0 @@
|
|||
package com.myoa.engineering.crawl.ppomppu.sender;
|
||||
|
||||
import org.springframework.boot.SpringApplication;
|
||||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||
import org.springframework.context.annotation.Import;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.support.webclient.PpomppuNotifierWebClientConfiguration;
|
||||
|
||||
/**
|
||||
* SenderApplication
|
||||
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
|
||||
* @since 2021-08-20
|
||||
*
|
||||
*/
|
||||
@Import({ PpomppuNotifierWebClientConfiguration.class })
|
||||
@SpringBootApplication
|
||||
public class SenderApplication {
|
||||
|
||||
public static void main(String[] args) {
|
||||
SpringApplication.run(SenderApplication.class, args);
|
||||
}
|
||||
}
|
|
@ -1,17 +0,0 @@
|
|||
package com.myoa.engineering.crawl.ppomppu.sender.infrastructure.client;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.sender.dto.MessageDTO;
|
||||
|
||||
import reactor.core.publisher.Mono;
|
||||
|
||||
/**
|
||||
* MessageSender
|
||||
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
|
||||
* @since 2021-11-14
|
||||
*
|
||||
*/
|
||||
public interface MessageSender<T extends MessageDTO> {
|
||||
|
||||
Mono<String> sendMessage(T message);
|
||||
|
||||
}
|
|
@ -1,14 +1,14 @@
|
|||
rootProject.name = 'PpomppuNotifier'
|
||||
|
||||
include 'receiver'
|
||||
project(':receiver').projectDir = "$rootDir/receiver" as File
|
||||
|
||||
include 'processor'
|
||||
project(':processor').projectDir = "$rootDir/processor" as File
|
||||
|
||||
include 'sender'
|
||||
project(':sender').projectDir = "$rootDir/sender" as File
|
||||
//include 'reader'
|
||||
//project(':reader').projectDir = "$rootDir/reader" as File
|
||||
//include 'processor'
|
||||
//project(':processor').projectDir = "$rootDir/processor" as File
|
||||
//include 'writer'
|
||||
//project(':writer').projectDir = "$rootDir/writer" as File
|
||||
|
||||
include 'shopping-crawler'
|
||||
project(':shopping-crawler').projectDir = "$rootDir/shopping-crawler" as File
|
||||
|
||||
include 'support'
|
||||
project(':support').projectDir = "$rootDir/support" as File
|
||||
|
|
Binary file not shown.
|
@ -0,0 +1,42 @@
|
|||
dependencies {
|
||||
developmentOnly 'org.springframework.boot:spring-boot-devtools'
|
||||
runtimeOnly 'com.h2database:h2'
|
||||
runtimeOnly 'com.mysql:mysql-connector-j'
|
||||
compileOnly 'org.projectlombok:lombok'
|
||||
|
||||
implementation project(':support')
|
||||
// https://projectreactor.io/docs/core/release/reference/#debug-activate
|
||||
implementation("org.springframework.boot:spring-boot-starter-web") {
|
||||
exclude group: "org.springframework.boot", module: "spring-boot-starter-tomcat"
|
||||
}
|
||||
|
||||
implementation("org.springframework.boot:spring-boot-starter-undertow") {
|
||||
exclude group: "io.undertow", module: "undertow-websockets-jsr"
|
||||
}
|
||||
implementation 'org.springframework.boot:spring-boot-starter-data-jpa'
|
||||
implementation 'org.springframework.boot:spring-boot-configuration-processor'
|
||||
implementation 'org.springframework.cloud:spring-cloud-starter-config'
|
||||
implementation 'org.springframework.boot:spring-boot-starter-actuator'
|
||||
implementation 'com.rometools:rome:2.1.0'
|
||||
implementation 'org.jsoup:jsoup:1.17.2'
|
||||
implementation 'com.h2database:h2:2.2.224'
|
||||
implementation "org.springframework.cloud:spring-cloud-starter-openfeign"
|
||||
implementation "io.github.openfeign:feign-hc5"
|
||||
implementation 'org.ahocorasick:ahocorasick:0.6.3'
|
||||
|
||||
annotationProcessor 'org.springframework.boot:spring-boot-configuration-processor'
|
||||
annotationProcessor 'org.projectlombok:lombok'
|
||||
|
||||
testImplementation 'org.springframework.boot:spring-boot-starter-test'
|
||||
testImplementation('org.assertj:assertj-core:3.25.3')
|
||||
testImplementation("org.jeasy:easy-random-core:5.0.0")
|
||||
testCompileOnly 'org.projectlombok:lombok'
|
||||
testAnnotationProcessor('org.projectlombok:lombok')
|
||||
}
|
||||
|
||||
test {
|
||||
useJUnitPlatform()
|
||||
testLogging {
|
||||
events "passed", "skipped", "failed"
|
||||
}
|
||||
}
|
|
@ -0,0 +1,19 @@
|
|||
package com.myoa.engineering.crawl.shopping;
|
||||
|
||||
import org.springframework.boot.SpringApplication;
|
||||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||
import org.springframework.boot.context.properties.EnableConfigurationProperties;
|
||||
import org.springframework.cloud.openfeign.EnableFeignClients;
|
||||
import org.springframework.scheduling.annotation.EnableScheduling;
|
||||
|
||||
@EnableScheduling
|
||||
@EnableFeignClients
|
||||
@EnableConfigurationProperties
|
||||
@SpringBootApplication
|
||||
public class ShoppingCrawlerApplication {
|
||||
|
||||
public static void main(String[] args) {
|
||||
SpringApplication.run(ShoppingCrawlerApplication.class, args);
|
||||
}
|
||||
|
||||
}
|
|
@ -1,6 +1,5 @@
|
|||
package com.myoa.engineering.crawl.ppomppu.processor.configuration;
|
||||
package com.myoa.engineering.crawl.shopping.configuration.datasource;
|
||||
|
||||
import java.sql.SQLException;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.h2.tools.Server;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
|
@ -10,6 +9,8 @@ import org.springframework.context.event.ContextClosedEvent;
|
|||
import org.springframework.context.event.ContextRefreshedEvent;
|
||||
import org.springframework.context.event.EventListener;
|
||||
|
||||
import java.sql.SQLException;
|
||||
|
||||
@Slf4j
|
||||
@Profile({"datasource-local", "datasource-development"})
|
||||
@Configuration
|
|
@ -1,54 +1,58 @@
|
|||
package com.myoa.engineering.crawl.ppomppu.processor.configuration;
|
||||
package com.myoa.engineering.crawl.shopping.configuration.datasource;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.processor.configuration.properties.DatasourceProperties;
|
||||
import com.myoa.engineering.crawl.ppomppu.processor.configuration.properties.DatasourceProperties.DataSourcePropertiesUnit;
|
||||
import com.myoa.engineering.crawl.ppomppu.processor.configuration.properties.HibernateProperties;
|
||||
import com.myoa.engineering.crawl.ppomppu.processor.configuration.properties.HikariProperties;
|
||||
import com.myoa.engineering.crawl.ppomppu.processor.domain.BaseScanDomain;
|
||||
import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository.BaseScanRepository;
|
||||
import com.myoa.engineering.crawl.shopping.configuration.datasource.properties.DatasourceProperties;
|
||||
import com.myoa.engineering.crawl.shopping.configuration.datasource.properties.HibernateProperties;
|
||||
import com.myoa.engineering.crawl.shopping.configuration.datasource.properties.HikariProperties;
|
||||
import com.myoa.engineering.crawl.shopping.domain.entity.BaseScanDomain;
|
||||
import com.myoa.engineering.crawl.shopping.infra.repository.BaseScanRepository;
|
||||
import com.zaxxer.hikari.HikariConfig;
|
||||
import com.zaxxer.hikari.HikariDataSource;
|
||||
import java.util.Enumeration;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.Properties;
|
||||
import javax.persistence.EntityManagerFactory;
|
||||
import javax.sql.DataSource;
|
||||
import jakarta.persistence.EntityManagerFactory;
|
||||
import lombok.NonNull;
|
||||
import org.hibernate.boot.model.naming.CamelCaseToUnderscoresNamingStrategy;
|
||||
import org.hibernate.boot.model.naming.ImplicitNamingStrategyJpaCompliantImpl;
|
||||
import org.hibernate.cfg.AvailableSettings;
|
||||
import org.springframework.beans.factory.annotation.Qualifier;
|
||||
import org.springframework.boot.orm.jpa.EntityManagerFactoryBuilder;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.data.jpa.repository.config.EnableJpaAuditing;
|
||||
import org.springframework.data.jpa.repository.config.EnableJpaRepositories;
|
||||
import org.springframework.orm.jpa.JpaTransactionManager;
|
||||
import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean;
|
||||
import org.springframework.transaction.PlatformTransactionManager;
|
||||
|
||||
@Configuration
|
||||
@EnableJpaRepositories(basePackageClasses = BaseScanRepository.class,
|
||||
entityManagerFactoryRef = "ppomppuNotifierProcessorEntityManagerFactory",
|
||||
transactionManagerRef = "ppomppuNotifierProcessorTransactionManager"
|
||||
)
|
||||
public class PpomppuDatasourceConfiguration {
|
||||
import javax.sql.DataSource;
|
||||
import java.util.Enumeration;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.Properties;
|
||||
|
||||
private static final String DATA_SOURCE_UNIT_NAME = "ppn_mysql";
|
||||
@Configuration
|
||||
@EnableJpaAuditing
|
||||
@EnableJpaRepositories(basePackageClasses = BaseScanRepository.class,
|
||||
entityManagerFactoryRef = "shoppingCrawlerEntityManagerFactory",
|
||||
transactionManagerRef = "shoppingCrawlerTransactionManager"
|
||||
)
|
||||
public class ShoppingCrawlerDatasourceConfiguration {
|
||||
|
||||
private static final String DATA_SOURCE_UNIT_NAME = "crawler-shopping";
|
||||
|
||||
private final DatasourceProperties dataSourceProeprties;
|
||||
private final HikariProperties hikariProperties;
|
||||
private final HibernateProperties hibernateProperties;
|
||||
|
||||
public PpomppuDatasourceConfiguration(DatasourceProperties dataSourceProeprties,
|
||||
HikariProperties hikariProperties,
|
||||
HibernateProperties hibernateProperties) {
|
||||
public ShoppingCrawlerDatasourceConfiguration(DatasourceProperties dataSourceProeprties,
|
||||
HikariProperties hikariProperties,
|
||||
HibernateProperties hibernateProperties) {
|
||||
this.dataSourceProeprties = dataSourceProeprties;
|
||||
this.hikariProperties = hikariProperties;
|
||||
this.hibernateProperties = hibernateProperties;
|
||||
}
|
||||
|
||||
@Bean(name = "ppomppuNotifierProcessorDataSource")
|
||||
@Bean(name = "shoppingCrawlerDataSource")
|
||||
public DataSource dataSource() {
|
||||
DataSourcePropertiesUnit dataSourcePropertiesUnit = dataSourceProeprties.find(DATA_SOURCE_UNIT_NAME);
|
||||
DatasourceProperties.DataSourcePropertiesUnit dataSourcePropertiesUnit = dataSourceProeprties.find(DATA_SOURCE_UNIT_NAME);
|
||||
|
||||
final HikariConfig hikariConfig = new HikariConfig();
|
||||
hikariConfig.setJdbcUrl(dataSourcePropertiesUnit.toCompletedJdbcUrl());
|
||||
|
@ -69,19 +73,19 @@ public class PpomppuDatasourceConfiguration {
|
|||
return dataSource;
|
||||
}
|
||||
|
||||
@Bean("ppomppuNotifierProcessorEntityManagerFactory")
|
||||
@Bean("shoppingCrawlerEntityManagerFactory")
|
||||
public LocalContainerEntityManagerFactoryBean entityManagerFactory(
|
||||
EntityManagerFactoryBuilder builder,
|
||||
@Qualifier("ppomppuNotifierProcessorDataSource") DataSource dataSource) {
|
||||
EntityManagerFactoryBuilder builder,
|
||||
@Qualifier("shoppingCrawlerDataSource") DataSource dataSource) {
|
||||
return builder.dataSource(dataSource)
|
||||
.packages(BaseScanDomain.class)
|
||||
.properties(getPropsMap(DATA_SOURCE_UNIT_NAME))
|
||||
.build();
|
||||
}
|
||||
|
||||
@Bean("ppomppuNotifierProcessorTransactionManager")
|
||||
@Bean("shoppingCrawlerTransactionManager")
|
||||
public PlatformTransactionManager transactionManager(
|
||||
@Qualifier("ppomppuNotifierProcessorEntityManagerFactory") EntityManagerFactory entityManagerFactory) {
|
||||
@Qualifier("shoppingCrawlerEntityManagerFactory") EntityManagerFactory entityManagerFactory) {
|
||||
return new JpaTransactionManager(entityManagerFactory);
|
||||
}
|
||||
|
||||
|
@ -91,12 +95,9 @@ public class PpomppuDatasourceConfiguration {
|
|||
properties.put(AvailableSettings.FORMAT_SQL, hibernateProperties.getFormatSql());
|
||||
properties.put(AvailableSettings.SHOW_SQL, hibernateProperties.getShowSql());
|
||||
properties.put(AvailableSettings.HBM2DDL_AUTO, hibernateProperties.getHbm2ddlAuto());
|
||||
properties.put(AvailableSettings.CONNECTION_PROVIDER_DISABLES_AUTOCOMMIT,
|
||||
hibernateProperties.getDisableAutoCommit());
|
||||
properties.put(AvailableSettings.IMPLICIT_NAMING_STRATEGY,
|
||||
"org.springframework.boot.orm.jpa.hibernate.SpringImplicitNamingStrategy");
|
||||
properties.put(AvailableSettings.PHYSICAL_NAMING_STRATEGY,
|
||||
"org.springframework.boot.orm.jpa.hibernate.SpringPhysicalNamingStrategy");
|
||||
properties.put(AvailableSettings.CONNECTION_PROVIDER_DISABLES_AUTOCOMMIT, hibernateProperties.getDisableAutoCommit());
|
||||
properties.put(AvailableSettings.IMPLICIT_NAMING_STRATEGY, ImplicitNamingStrategyJpaCompliantImpl.class.getName());
|
||||
properties.put(AvailableSettings.PHYSICAL_NAMING_STRATEGY, CamelCaseToUnderscoresNamingStrategy.class.getName());
|
||||
properties.put(AvailableSettings.GENERATE_STATISTICS, "false");
|
||||
properties.put(AvailableSettings.GLOBALLY_QUOTED_IDENTIFIERS, "true");
|
||||
properties.put(AvailableSettings.GLOBALLY_QUOTED_IDENTIFIERS_SKIP_COLUMN_DEFINITIONS, "true");
|
||||
|
@ -104,7 +105,7 @@ public class PpomppuDatasourceConfiguration {
|
|||
properties.put(AvailableSettings.ORDER_INSERTS, "true");
|
||||
properties.put(AvailableSettings.ORDER_UPDATES, "true");
|
||||
properties.put(AvailableSettings.BATCH_VERSIONED_DATA, "true");
|
||||
properties.put(AvailableSettings.USE_NEW_ID_GENERATOR_MAPPINGS, "false");
|
||||
// properties.put(AvailableSettings.JPA_ID_GENERATOR_GLOBAL_SCOPE_COMPLIANCE, "false");
|
||||
return properties;
|
||||
}
|
||||
|
|
@ -1,29 +1,26 @@
|
|||
package com.myoa.engineering.crawl.ppomppu.processor.configuration.properties;
|
||||
package com.myoa.engineering.crawl.shopping.configuration.datasource.properties;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.support.util.ObjectUtil;
|
||||
import java.util.List;
|
||||
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
import com.myoa.engineering.crawl.shopping.support.util.ObjectUtil;
|
||||
import lombok.Data;
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Component
|
||||
@Setter
|
||||
@Getter
|
||||
@ConfigurationProperties(prefix = "infra.database")
|
||||
@Data
|
||||
@ConfigurationProperties(prefix = "datasource")
|
||||
public class DatasourceProperties {
|
||||
|
||||
private List<DataSourcePropertiesUnit> units;
|
||||
|
||||
@Getter
|
||||
@Setter
|
||||
@Data
|
||||
public static class DataSourcePropertiesUnit {
|
||||
|
||||
private String unitName;
|
||||
private String schemaName;
|
||||
private String connectionParameters;
|
||||
private String datasourceUrl;
|
||||
private String dbConnectionUrl;
|
||||
private Boolean isSimpleConnectionUrl;
|
||||
private String username;
|
||||
private String password;
|
||||
|
@ -31,9 +28,9 @@ public class DatasourceProperties {
|
|||
|
||||
public String toCompletedJdbcUrl() {
|
||||
if (ObjectUtil.isEmpty(isSimpleConnectionUrl) || isSimpleConnectionUrl == false) {
|
||||
return String.format("%s/%s?%s", datasourceUrl, schemaName, connectionParameters);
|
||||
return String.format("%s/%s?%s", dbConnectionUrl, schemaName, connectionParameters);
|
||||
}
|
||||
return datasourceUrl;
|
||||
return dbConnectionUrl;
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -42,7 +39,7 @@ public class DatasourceProperties {
|
|||
.filter(e -> e.getUnitName().equals(unitName))
|
||||
.findFirst()
|
||||
.orElseThrow(
|
||||
() -> new IllegalArgumentException(this.getClass().getName() + ": unitName Not found. " + unitName));
|
||||
() -> new IllegalArgumentException(this.getClass().getName() + ": unitName Not found. " + unitName));
|
||||
}
|
||||
|
||||
}
|
|
@ -1,14 +1,12 @@
|
|||
package com.myoa.engineering.crawl.ppomppu.processor.configuration.properties;
|
||||
package com.myoa.engineering.crawl.shopping.configuration.datasource.properties;
|
||||
|
||||
import java.util.List;
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
|
||||
import org.hibernate.dialect.MySQL8Dialect;
|
||||
import org.hibernate.dialect.MySQLDialect;
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Component
|
||||
@Setter
|
||||
@Getter
|
|
@ -1,4 +1,4 @@
|
|||
package com.myoa.engineering.crawl.ppomppu.processor.configuration.properties;
|
||||
package com.myoa.engineering.crawl.shopping.configuration.datasource.properties;
|
||||
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
|
@ -1,4 +1,4 @@
|
|||
package com.myoa.engineering.crawl.ppomppu.sender.configuration.properties;
|
||||
package com.myoa.engineering.crawl.shopping.configuration.slack.properties;
|
||||
|
||||
import java.util.List;
|
||||
import lombok.Data;
|
||||
|
@ -7,18 +7,16 @@ import lombok.Setter;
|
|||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
@Getter
|
||||
@Setter
|
||||
@Data
|
||||
@Component
|
||||
@ConfigurationProperties("infra.slack.bot")
|
||||
@ConfigurationProperties(prefix = "slack.bot")
|
||||
public class SlackSecretProperties {
|
||||
|
||||
private List<SlackSecretPropertiesUnit> units;
|
||||
|
||||
@Data
|
||||
public static class SlackSecretPropertiesUnit {
|
||||
|
||||
private String botName;
|
||||
private String botUnitName;
|
||||
private String username;
|
||||
private String iconEmoji;
|
||||
private String channel;
|
||||
|
@ -27,7 +25,7 @@ public class SlackSecretProperties {
|
|||
|
||||
public SlackSecretPropertiesUnit find(String botUnitName) {
|
||||
return units.stream()
|
||||
.filter(e -> e.getBotName().equals(botUnitName))
|
||||
.filter(e -> e.getBotUnitName().equals(botUnitName))
|
||||
.findFirst()
|
||||
.orElseThrow(() -> new IllegalArgumentException("not found bot unit name : " + botUnitName));
|
||||
}
|
|
@ -0,0 +1,22 @@
|
|||
package com.myoa.engineering.crawl.shopping.controller;
|
||||
|
||||
import com.myoa.engineering.crawl.shopping.crawlhandler.PpomppuCrawlDomesticHandler;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
@RestController
|
||||
@RequestMapping("/api/v1/exploit")
|
||||
public class TestAPIController {
|
||||
|
||||
private final PpomppuCrawlDomesticHandler ppomppuCrawlDomesticHandler;
|
||||
|
||||
public TestAPIController(PpomppuCrawlDomesticHandler ppomppuCrawlDomesticHandler) {
|
||||
this.ppomppuCrawlDomesticHandler = ppomppuCrawlDomesticHandler;
|
||||
}
|
||||
|
||||
@GetMapping("/triggers")
|
||||
public void triggerExploit() {
|
||||
ppomppuCrawlDomesticHandler.handle();
|
||||
}
|
||||
}
|
|
@ -0,0 +1,10 @@
|
|||
package com.myoa.engineering.crawl.shopping.crawlhandler;
|
||||
|
||||
import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget;
|
||||
|
||||
public interface CrawlHandler {
|
||||
|
||||
CrawlTarget getCrawlTarget();
|
||||
|
||||
void handle();
|
||||
}
|
|
@ -0,0 +1,18 @@
|
|||
package com.myoa.engineering.crawl.shopping.crawlhandler;
|
||||
|
||||
import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
@Slf4j
|
||||
@Component
|
||||
public class FmkoreaCrawlHandler implements CrawlHandler {
|
||||
@Override
|
||||
public CrawlTarget getCrawlTarget() {
|
||||
return CrawlTarget.FMKOREA;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void handle() {
|
||||
}
|
||||
}
|
|
@ -0,0 +1,58 @@
|
|||
package com.myoa.engineering.crawl.shopping.crawlhandler;
|
||||
|
||||
import com.myoa.engineering.crawl.shopping.crawlhandler.parser.PpomppuArticleParserV2;
|
||||
import com.myoa.engineering.crawl.shopping.domain.entity.v2.Article;
|
||||
import com.myoa.engineering.crawl.shopping.infra.client.ppomppu.PpomppuBoardClientV2;
|
||||
import com.myoa.engineering.crawl.shopping.service.ArticleCommandService;
|
||||
import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
@Component
|
||||
public class PpomppuCrawlDomesticHandler implements CrawlHandler {
|
||||
|
||||
private final PpomppuBoardClientV2 ppomppuBoardClient;
|
||||
private final PpomppuArticleParserV2 ppomppuArticleParserV2;
|
||||
private final ArticleCommandService articleCommandService;
|
||||
|
||||
public PpomppuCrawlDomesticHandler(PpomppuBoardClientV2 ppomppuBoardClient,
|
||||
PpomppuArticleParserV2 ppomppuArticleParserV2,
|
||||
ArticleCommandService articleCommandService) {
|
||||
this.ppomppuBoardClient = ppomppuBoardClient;
|
||||
this.ppomppuArticleParserV2 = ppomppuArticleParserV2;
|
||||
this.articleCommandService = articleCommandService;
|
||||
}
|
||||
|
||||
@Override
|
||||
public CrawlTarget getCrawlTarget() {
|
||||
return CrawlTarget.PPOMPPU_DOMESTIC;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void handle() {
|
||||
|
||||
String boardHtmlPage1 = ppomppuBoardClient.getBoardHtml("/zboard/zboard.php", generateRequestParams(1));
|
||||
List<Article> parsedPage1 = ppomppuArticleParserV2.parse(boardHtmlPage1);
|
||||
|
||||
String boardHtmlPage2 = ppomppuBoardClient.getBoardHtml("/zboard/zboard.php", generateRequestParams(2));
|
||||
List<Article> parsedPage2 = ppomppuArticleParserV2.parse(boardHtmlPage2);
|
||||
|
||||
List<Article> merged = Stream.of(parsedPage1, parsedPage2)
|
||||
.flatMap(List::stream)
|
||||
.map(e -> e.updateCrawlTarget(getCrawlTarget()))
|
||||
.toList();
|
||||
|
||||
articleCommandService.upsert(merged);
|
||||
}
|
||||
|
||||
private Map<String, String> generateRequestParams(int pageId) {
|
||||
Map<String, String> params = new HashMap<>();
|
||||
params.put("id", "ppomppu");
|
||||
params.put("page", String.valueOf(pageId));
|
||||
return params;
|
||||
}
|
||||
}
|
|
@ -0,0 +1,115 @@
|
|||
package com.myoa.engineering.crawl.shopping.crawlhandler.parser;
|
||||
|
||||
import com.myoa.engineering.crawl.shopping.domain.entity.v2.Article;
|
||||
import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName;
|
||||
import com.myoa.engineering.crawl.shopping.util.DateTimeUtils;
|
||||
import com.myoa.engineering.crawl.shopping.util.NumberUtils;
|
||||
import io.micrometer.core.instrument.util.StringUtils;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.jsoup.Jsoup;
|
||||
import org.jsoup.nodes.Document;
|
||||
import org.jsoup.nodes.Element;
|
||||
import org.jsoup.select.Elements;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.time.ZoneId;
|
||||
import java.time.ZonedDateTime;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.util.List;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
@Slf4j
|
||||
@Component
|
||||
public final class PpomppuArticleParserV2 {
|
||||
|
||||
private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yy.MM.dd HH:mm:ss")
|
||||
.withZone(ZoneId.of("Asia/Seoul"));
|
||||
|
||||
private PpomppuArticleParserV2() {
|
||||
}
|
||||
|
||||
public List<Article> parse(String html) {
|
||||
Elements trElements = converHtmlToTrElements(html);
|
||||
return trElements.stream()
|
||||
.filter(this::isRealArticle)
|
||||
.map(this::parse)
|
||||
.toList();
|
||||
|
||||
}
|
||||
|
||||
private Elements converHtmlToTrElements(String data) {
|
||||
Document document = Jsoup.parse(data);
|
||||
Elements trList = document.getElementById("revolution_main_table").getElementsByTag("tr");
|
||||
return trList;
|
||||
}
|
||||
|
||||
private boolean isRealArticle(Element tr) {
|
||||
Elements tdList = tr.getElementsByTag("td");
|
||||
if (tdList.size() != 6) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!hasOnlyNumeric(tdList.get(0))) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
private Pattern pattern_numeric = Pattern.compile("\\d+");
|
||||
|
||||
private boolean hasOnlyNumeric(Element td) {
|
||||
return pattern_numeric.matcher(td.text()).matches();
|
||||
}
|
||||
|
||||
private Article parse(Element tr) {
|
||||
Elements tdList = tr.getElementsByTag("td");
|
||||
Long articleId = Long.parseLong(tdList.get(0).text());
|
||||
|
||||
String title = tdList.get(2).text();
|
||||
String articleUrl = parseArticleUrl(tdList.get(2).getElementsByTag("a").attr("href"));
|
||||
String boardName = parseBoardName(title);
|
||||
Integer recommended = parseRecommended(tdList.get(4));
|
||||
Integer hit = NumberUtils.parseInt(tdList.get(5).text(), 0);
|
||||
ZonedDateTime registeredAt = DateTimeUtils.parse(tdList.get(3).text());
|
||||
|
||||
return Article.builder()
|
||||
.articleId(articleId)
|
||||
.title(title)
|
||||
.boardName(boardName)
|
||||
.articleUrl(articleUrl)
|
||||
.recommended(recommended)
|
||||
.hit(hit)
|
||||
.registeredAt(registeredAt)
|
||||
.build();
|
||||
}
|
||||
|
||||
public Integer parseRecommended(Element td) {
|
||||
final String voteString = td.text();
|
||||
if (StringUtils.isEmpty(voteString)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
final int voteUp = Integer.parseInt(td.text().split(" - ")[0]);
|
||||
final int voteDown = Integer.parseInt(td.text().split(" - ")[1]);
|
||||
int recommended = voteUp - voteDown;
|
||||
return recommended;
|
||||
}
|
||||
|
||||
public static String parseArticleUrl(String data) {
|
||||
return PpomppuBoardName.ofViewPageUrl(data);
|
||||
}
|
||||
|
||||
Pattern patternBoardName = Pattern.compile("\\[(.+?)\\]");
|
||||
|
||||
public String parseBoardName(String fullTitle) {
|
||||
Matcher matcher = patternBoardName.matcher(fullTitle);
|
||||
String lastMatched = null;
|
||||
while (matcher.find()) {
|
||||
lastMatched = matcher.group(1);
|
||||
}
|
||||
return lastMatched;
|
||||
}
|
||||
|
||||
}
|
|
@ -1,17 +1,18 @@
|
|||
package com.myoa.engineering.crawl.ppomppu.processor.domain;
|
||||
package com.myoa.engineering.crawl.shopping.domain.entity;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.time.Instant;
|
||||
import javax.persistence.Column;
|
||||
import lombok.Getter;
|
||||
import org.springframework.data.annotation.CreatedDate;
|
||||
import org.springframework.data.annotation.LastModifiedDate;
|
||||
|
||||
/**
|
||||
* Auditable
|
||||
*
|
||||
* @author Shin Woo-jin (woozu.shin@kakaoent.com)
|
||||
* @since 2021-09-08
|
||||
*/
|
||||
import jakarta.persistence.*;
|
||||
import org.springframework.data.jpa.domain.support.AuditingEntityListener;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.time.Instant;
|
||||
|
||||
@Getter
|
||||
@MappedSuperclass
|
||||
@EntityListeners(AuditingEntityListener.class)
|
||||
public abstract class Auditable implements Serializable {
|
||||
private static final long serialVersionUID = -7105030870015828551L;
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
package com.myoa.engineering.crawl.ppomppu.processor.domain;
|
||||
package com.myoa.engineering.crawl.shopping.domain.entity;
|
||||
|
||||
/**
|
||||
* BaseScanDomain
|
|
@ -1,19 +1,16 @@
|
|||
package com.myoa.engineering.crawl.ppomppu.processor.domain;
|
||||
package com.myoa.engineering.crawl.shopping.domain.entity.v1;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
|
||||
import java.time.Instant;
|
||||
import javax.persistence.Column;
|
||||
import javax.persistence.Entity;
|
||||
import javax.persistence.EnumType;
|
||||
import javax.persistence.Enumerated;
|
||||
import javax.persistence.GeneratedValue;
|
||||
import javax.persistence.GenerationType;
|
||||
import javax.persistence.Id;
|
||||
import javax.persistence.Table;
|
||||
import com.myoa.engineering.crawl.shopping.domain.entity.Auditable;
|
||||
import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName;
|
||||
import jakarta.persistence.*;
|
||||
import lombok.Builder;
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.ToString;
|
||||
|
||||
import java.time.ZonedDateTime;
|
||||
|
||||
@ToString
|
||||
@Getter
|
||||
@NoArgsConstructor
|
||||
@Entity
|
||||
|
@ -44,11 +41,11 @@ public class PpomppuArticle extends Auditable {
|
|||
private Integer recommended;
|
||||
|
||||
@Column
|
||||
private Instant registeredAt;
|
||||
private ZonedDateTime registeredAt;
|
||||
|
||||
@Builder
|
||||
public PpomppuArticle(Long id, Long articleId, PpomppuBoardName boardName, String articleUrl,
|
||||
String title, Integer recommended, Integer hit, Instant registeredAt) {
|
||||
String title, Integer recommended, Integer hit, ZonedDateTime registeredAt) {
|
||||
this.id = id;
|
||||
this.articleId = articleId;
|
||||
this.boardName = boardName;
|
||||
|
@ -59,8 +56,4 @@ public class PpomppuArticle extends Auditable {
|
|||
this.registeredAt = registeredAt;
|
||||
}
|
||||
|
||||
public PpomppuArticle updateBoardName(PpomppuBoardName boardName) {
|
||||
this.boardName = boardName;
|
||||
return this;
|
||||
}
|
||||
}
|
|
@ -1,19 +1,14 @@
|
|||
package com.myoa.engineering.crawl.ppomppu.processor.domain;
|
||||
package com.myoa.engineering.crawl.shopping.domain.entity.v1;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
|
||||
import java.time.Instant;
|
||||
import javax.persistence.Column;
|
||||
import javax.persistence.Entity;
|
||||
import javax.persistence.EnumType;
|
||||
import javax.persistence.Enumerated;
|
||||
import javax.persistence.GeneratedValue;
|
||||
import javax.persistence.GenerationType;
|
||||
import javax.persistence.Id;
|
||||
import javax.persistence.Table;
|
||||
import com.myoa.engineering.crawl.shopping.domain.entity.Auditable;
|
||||
import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName;
|
||||
import jakarta.persistence.*;
|
||||
import lombok.Builder;
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
import java.time.Instant;
|
||||
|
||||
@Getter
|
||||
@NoArgsConstructor
|
||||
@Entity
|
|
@ -1,15 +1,12 @@
|
|||
package com.myoa.engineering.crawl.ppomppu.processor.domain;
|
||||
package com.myoa.engineering.crawl.shopping.domain.entity.v1;
|
||||
|
||||
import java.time.Instant;
|
||||
import javax.persistence.Column;
|
||||
import javax.persistence.Entity;
|
||||
import javax.persistence.GeneratedValue;
|
||||
import javax.persistence.GenerationType;
|
||||
import javax.persistence.Id;
|
||||
import javax.persistence.Table;
|
||||
import com.myoa.engineering.crawl.shopping.domain.entity.Auditable;
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
import jakarta.persistence.*;
|
||||
import java.time.Instant;
|
||||
|
||||
@Getter
|
||||
@NoArgsConstructor
|
||||
@Entity
|
|
@ -0,0 +1,27 @@
|
|||
package com.myoa.engineering.crawl.shopping.domain.entity.v1;
|
||||
|
||||
import com.myoa.engineering.crawl.shopping.domain.entity.Auditable;
|
||||
import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName;
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
import jakarta.persistence.*;
|
||||
|
||||
@Getter
|
||||
@NoArgsConstructor
|
||||
@Entity
|
||||
@Table(name = "subscribed_board")
|
||||
public class SubscribedBoard extends Auditable {
|
||||
|
||||
@Id
|
||||
@GeneratedValue(strategy = GenerationType.IDENTITY)
|
||||
private Long id;
|
||||
|
||||
@Column
|
||||
private Long userId;
|
||||
|
||||
@Column
|
||||
@Enumerated(EnumType.STRING)
|
||||
private PpomppuBoardName boardName;
|
||||
|
||||
}
|
|
@ -1,20 +1,17 @@
|
|||
package com.myoa.engineering.crawl.ppomppu.processor.domain;
|
||||
package com.myoa.engineering.crawl.shopping.domain.entity.v1;
|
||||
|
||||
import java.time.Instant;
|
||||
import javax.persistence.Column;
|
||||
import javax.persistence.Entity;
|
||||
import javax.persistence.GeneratedValue;
|
||||
import javax.persistence.GenerationType;
|
||||
import javax.persistence.Id;
|
||||
import javax.persistence.Table;
|
||||
import com.myoa.engineering.crawl.shopping.domain.entity.Auditable;
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
import jakarta.persistence.*;
|
||||
import java.time.Instant;
|
||||
|
||||
@Getter
|
||||
@NoArgsConstructor
|
||||
@Entity
|
||||
@Table(name = "subscribed_user")
|
||||
public class SubscribedUser extends Auditable{
|
||||
public class SubscribedUser extends Auditable {
|
||||
|
||||
@Id
|
||||
@GeneratedValue(strategy = GenerationType.IDENTITY)
|
|
@ -0,0 +1,26 @@
|
|||
package com.myoa.engineering.crawl.shopping.domain.entity.v2;
|
||||
|
||||
import com.myoa.engineering.crawl.shopping.domain.entity.Auditable;
|
||||
import jakarta.persistence.*;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Getter
|
||||
@Builder
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
@Entity
|
||||
@Table
|
||||
public class AppUser extends Auditable {
|
||||
@Id
|
||||
@GeneratedValue(strategy = GenerationType.IDENTITY)
|
||||
private Long id;
|
||||
|
||||
@Column
|
||||
private String name;
|
||||
|
||||
@Column
|
||||
private Boolean enabled;
|
||||
}
|
|
@ -0,0 +1,63 @@
|
|||
package com.myoa.engineering.crawl.shopping.domain.entity.v2;
|
||||
|
||||
import com.myoa.engineering.crawl.shopping.domain.entity.Auditable;
|
||||
import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget;
|
||||
import jakarta.persistence.*;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
import java.time.ZonedDateTime;
|
||||
|
||||
@Getter
|
||||
@Builder
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
@Entity
|
||||
@Table
|
||||
public class Article extends Auditable {
|
||||
|
||||
@Id
|
||||
@GeneratedValue(strategy = GenerationType.IDENTITY)
|
||||
private Long id;
|
||||
|
||||
@Column
|
||||
private Long articleId;
|
||||
|
||||
@Column
|
||||
@Enumerated(EnumType.STRING)
|
||||
private CrawlTarget crawlTarget;
|
||||
|
||||
@Column
|
||||
private String boardName;
|
||||
|
||||
@Column
|
||||
private String articleUrl;
|
||||
|
||||
@Column
|
||||
private String title;
|
||||
|
||||
@Column
|
||||
private Integer hit;
|
||||
|
||||
@Column
|
||||
private Integer recommended;
|
||||
|
||||
@Column
|
||||
private ZonedDateTime registeredAt;
|
||||
|
||||
public Article update(Article article) {
|
||||
this.boardName = article.getBoardName();
|
||||
this.articleUrl = article.getArticleUrl();
|
||||
this.title = article.getTitle();
|
||||
this.hit = article.getHit();
|
||||
this.recommended = article.getRecommended();
|
||||
return this;
|
||||
}
|
||||
|
||||
public Article updateCrawlTarget(CrawlTarget crawlTarget) {
|
||||
this.crawlTarget = crawlTarget;
|
||||
return this;
|
||||
}
|
||||
}
|
|
@ -0,0 +1,30 @@
|
|||
package com.myoa.engineering.crawl.shopping.domain.entity.v2;
|
||||
|
||||
import com.myoa.engineering.crawl.shopping.domain.entity.Auditable;
|
||||
import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget;
|
||||
import jakarta.persistence.*;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Getter
|
||||
@Builder
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
@Entity
|
||||
@Table
|
||||
public class SubscribedKeyword extends Auditable {
|
||||
|
||||
@Id
|
||||
@GeneratedValue(strategy = GenerationType.IDENTITY)
|
||||
private Long id;
|
||||
|
||||
@Column
|
||||
private String keyword;
|
||||
|
||||
@Column
|
||||
@Enumerated(EnumType.STRING)
|
||||
private CrawlTarget crawlTarget;
|
||||
|
||||
}
|
|
@ -0,0 +1,27 @@
|
|||
package com.myoa.engineering.crawl.shopping.domain.model.v2;
|
||||
|
||||
import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
import java.time.ZonedDateTime;
|
||||
|
||||
@Getter
|
||||
@Builder
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class ArticleModel {
|
||||
|
||||
private Long id;
|
||||
private Long articleId;
|
||||
private CrawlTarget crawlTarget;
|
||||
private String boardName;
|
||||
private String articleUrl;
|
||||
private String title;
|
||||
private Integer hit;
|
||||
private Integer recommended;
|
||||
private ZonedDateTime registeredAt;
|
||||
|
||||
}
|
|
@ -1,12 +1,13 @@
|
|||
package com.myoa.engineering.crawl.ppomppu.processor.dto;
|
||||
package com.myoa.engineering.crawl.shopping.dto;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
|
||||
import java.io.Serializable;
|
||||
import java.time.Instant;
|
||||
import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName;
|
||||
import lombok.Builder;
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.time.Instant;
|
||||
|
||||
/**
|
||||
* FeedParsedResult
|
||||
*
|
|
@ -1,4 +1,4 @@
|
|||
package com.myoa.engineering.crawl.ppomppu.processor.dto;
|
||||
package com.myoa.engineering.crawl.shopping.dto;
|
||||
|
||||
/**
|
||||
* PpomppuArticle
|
|
@ -1,38 +1,38 @@
|
|||
package com.myoa.engineering.crawl.ppomppu.processor.dto;
|
||||
package com.myoa.engineering.crawl.shopping.dto;
|
||||
|
||||
import com.myoa.engineering.crawl.shopping.domain.entity.v1.PpomppuArticle;
|
||||
import com.myoa.engineering.crawl.shopping.support.dto.SimpleMessageDTO;
|
||||
|
||||
import java.time.Instant;
|
||||
import java.time.ZoneId;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.util.List;
|
||||
import java.util.function.Function;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.dto.SimpleMessageDTO;
|
||||
|
||||
/**
|
||||
* PpomppuArticleTransformer
|
||||
*
|
||||
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
|
||||
* @since 2021-11-21
|
||||
*
|
||||
*/
|
||||
public final class PpomppuArticleTransformer {
|
||||
|
||||
private PpomppuArticleTransformer() {}
|
||||
private PpomppuArticleTransformer() {
|
||||
}
|
||||
|
||||
private static final String MESSAGE_FORMAT_V1 = "%s)) `%s` <%s:LINK>";
|
||||
|
||||
private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")
|
||||
.withZone(ZoneId.of("Asia/Seoul"));
|
||||
|
||||
/*
|
||||
public static final Function<PpomppuArticle, SimpleMessageDTO> TRANSFORM_TO_MESSAGE_DTO = entity ->
|
||||
SimpleMessageDTO.builder()
|
||||
.requestedAt(Instant.now())
|
||||
.publishedAt(entity.getRegisteredAt())
|
||||
.title(String.format(MESSAGE_FORMAT_V1, entity.getBoardName().getMenuName(), entity.getTitle()))
|
||||
.body(entity.getArticleUrl())
|
||||
.build();
|
||||
|
||||
SimpleMessageDTO.builder()
|
||||
.requestedAt(Instant.now())
|
||||
.publishedAt(entity.getRegisteredAt())
|
||||
.title(String.format(MESSAGE_FORMAT_V1, entity.getBoardName().getMenuName(), entity.getTitle()))
|
||||
.body(entity.getArticleUrl())
|
||||
.build();
|
||||
*/
|
||||
// https://stackoverflow.com/questions/24882927/using-streams-to-convert-a-list-of-objects-into-a-string-obtained-from-the-tostr
|
||||
public static SimpleMessageDTO transform(List<PpomppuArticle> articles) {
|
||||
Instant requestedAt = Instant.now();
|
|
@ -1,4 +1,4 @@
|
|||
package com.myoa.engineering.crawl.ppomppu.processor.dto.constant;
|
||||
package com.myoa.engineering.crawl.shopping.dto.constant;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Getter;
|
|
@ -1,4 +1,4 @@
|
|||
package com.myoa.engineering.crawl.ppomppu.sender.dto;
|
||||
package com.myoa.engineering.crawl.shopping.dto.slack;
|
||||
|
||||
import java.io.Serializable;
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
package com.myoa.engineering.crawl.ppomppu.sender.dto;
|
||||
package com.myoa.engineering.crawl.shopping.dto.slack;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
|
|
@ -0,0 +1,12 @@
|
|||
package com.myoa.engineering.crawl.shopping.event;
|
||||
|
||||
import org.springframework.context.ApplicationEvent;
|
||||
|
||||
public class ArticleUpsertEvent extends ApplicationEvent {
|
||||
|
||||
public ArticleUpsertEvent(Object source) {
|
||||
super(source);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
package com.myoa.engineering.crawl.shopping.event;
|
||||
|
||||
import com.myoa.engineering.crawl.shopping.domain.model.v2.ArticleModel;
|
||||
import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget;
|
||||
import lombok.Getter;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Getter
|
||||
@Deprecated
|
||||
public class ArticleUpsertEventCommand {
|
||||
private final List<ArticleModel> articles;
|
||||
private final CrawlTarget crawlTarget;
|
||||
|
||||
public ArticleUpsertEventCommand(List<ArticleModel> articles, CrawlTarget crawlTarget) {
|
||||
this.articles = articles;
|
||||
this.crawlTarget = crawlTarget;
|
||||
}
|
||||
}
|
|
@ -0,0 +1,23 @@
|
|||
package com.myoa.engineering.crawl.shopping.event.handler;
|
||||
|
||||
import com.myoa.engineering.crawl.shopping.event.ArticleUpsertEvent;
|
||||
import com.myoa.engineering.crawl.shopping.service.SubscribedKeywordQueryService;
|
||||
import org.ahocorasick.trie.Trie;
|
||||
import org.springframework.context.event.EventListener;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
@Component
|
||||
public class ArticleUpsertEventListener {
|
||||
|
||||
private final SubscribedKeywordQueryService subscribedKeywordQueryService;
|
||||
|
||||
public ArticleUpsertEventListener(SubscribedKeywordQueryService subscribedKeywordQueryService) {
|
||||
this.subscribedKeywordQueryService = subscribedKeywordQueryService;
|
||||
}
|
||||
|
||||
@EventListener
|
||||
public void handleArticleUpsertEvent(ArticleUpsertEvent event) {
|
||||
|
||||
System.out.println("event = " + event);
|
||||
}
|
||||
}
|
|
@ -1,22 +1,7 @@
|
|||
package com.myoa.engineering.crawl.ppomppu.processor.infrastructure.client;
|
||||
|
||||
import org.springframework.core.ParameterizedTypeReference;
|
||||
import org.springframework.http.HttpHeaders;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.web.reactive.function.client.WebClient;
|
||||
import org.springframework.web.reactive.function.client.WebClientRequestException;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.processor.dto.constant.WebClientPropertiesUnitName;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.dto.SimpleMessageDTO;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebClientFilterFactory;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebFluxExchangeStragiesFactory;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.webclient.properties.WebClientProperties;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.webclient.properties.WebClientProperties.WebClientPropertiesUnit;
|
||||
package com.myoa.engineering.crawl.shopping.infra.client;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import reactor.core.publisher.Mono;
|
||||
import reactor.core.scheduler.Schedulers;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/**
|
||||
* PpomppuNotifierSenderAPIClient
|
||||
|
@ -27,7 +12,7 @@ import reactor.core.scheduler.Schedulers;
|
|||
@Slf4j
|
||||
@Component
|
||||
public class MessageSenderAPIClient {
|
||||
|
||||
/*
|
||||
private final WebClient webClient;
|
||||
|
||||
public MessageSenderAPIClient(WebClientProperties webClientProperties) {
|
||||
|
@ -53,4 +38,6 @@ public class MessageSenderAPIClient {
|
|||
return Mono.empty();
|
||||
});
|
||||
}
|
||||
|
||||
*/
|
||||
}
|
|
@ -0,0 +1,12 @@
|
|||
package com.myoa.engineering.crawl.shopping.infra.client.fmkorea;
|
||||
|
||||
import org.springframework.cloud.openfeign.FeignClient;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
|
||||
@FeignClient(value = "fmkorea-board-client", url = "https://fmkorea.com")
|
||||
public interface FmkoreaBoardClient {
|
||||
|
||||
@GetMapping("{boardLink}")
|
||||
String getBoardHtml(@PathVariable("boardLink") String boardLink);
|
||||
}
|
|
@ -1,16 +1,7 @@
|
|||
package com.myoa.engineering.crawl.ppomppu.processor.infrastructure.client;
|
||||
package com.myoa.engineering.crawl.shopping.infra.client.ppomppu;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebClientFilterFactory;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebFluxExchangeStragiesFactory;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import org.springframework.core.ParameterizedTypeReference;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.web.reactive.function.client.WebClient;
|
||||
import org.springframework.web.reactive.function.client.WebClientRequestException;
|
||||
import reactor.core.publisher.Mono;
|
||||
import reactor.core.scheduler.Schedulers;
|
||||
|
||||
/**
|
||||
* PpomppuBoardFeedRetriever
|
||||
|
@ -20,11 +11,11 @@ import reactor.core.scheduler.Schedulers;
|
|||
*/
|
||||
@Slf4j
|
||||
@Component
|
||||
public class PpomppuBoardFeedRetriever {
|
||||
|
||||
public class PpomppuBoardClient {
|
||||
/*
|
||||
private final WebClient webClient;
|
||||
|
||||
public PpomppuBoardFeedRetriever(WebClient.Builder webClientBuilder) {
|
||||
public PpomppuBoardClient(WebClient.Builder webClientBuilder) {
|
||||
this.webClient = webClientBuilder.baseUrl(PpomppuBoardName.PPOMPPU_URL)
|
||||
.exchangeStrategies(WebFluxExchangeStragiesFactory.ofTextHtml())
|
||||
.filter(WebClientFilterFactory.logRequest())
|
||||
|
@ -43,5 +34,5 @@ public class PpomppuBoardFeedRetriever {
|
|||
});
|
||||
// .doOnNext(e -> log.info("[getHtml] {}", e));
|
||||
}
|
||||
|
||||
*/
|
||||
}
|
|
@ -0,0 +1,17 @@
|
|||
package com.myoa.engineering.crawl.shopping.infra.client.ppomppu;
|
||||
|
||||
import org.springframework.cloud.openfeign.FeignClient;
|
||||
import org.springframework.cloud.openfeign.SpringQueryMap;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
@FeignClient(value = "ppomppu-board-client", url = "https://www.ppomppu.co.kr")
|
||||
public interface PpomppuBoardClientV2 {
|
||||
|
||||
@GetMapping("{boardLink}")
|
||||
String getBoardHtml(@PathVariable("boardLink") String boardLink,
|
||||
@SpringQueryMap Map<String, String> params);
|
||||
}
|
|
@ -1,22 +1,25 @@
|
|||
package com.myoa.engineering.crawl.ppomppu.sender.infrastructure.client;
|
||||
package com.myoa.engineering.crawl.shopping.infra.client.slack;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.sender.configuration.properties.SlackSecretProperties;
|
||||
import com.myoa.engineering.crawl.ppomppu.sender.configuration.properties.SlackSecretProperties.SlackSecretPropertiesUnit;
|
||||
import com.myoa.engineering.crawl.ppomppu.sender.dto.SlackMessageDTO;
|
||||
import com.myoa.engineering.crawl.shopping.configuration.slack.properties.SlackSecretProperties;
|
||||
import com.myoa.engineering.crawl.shopping.dto.slack.SlackMessageDTO;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
@Slf4j
|
||||
@Component
|
||||
public class MongeShoppingBotSlackMessageSender extends SlackMessageSender {
|
||||
public class MongeShoppingBotSlackMessageSender {
|
||||
|
||||
private static final String SLACK_SECRET_UNIT_NAME = "monge_shopping_bot";
|
||||
private static final String SLACK_SECRET_UNIT_NAME = "shopping-crawler";
|
||||
|
||||
private final SlackSecretPropertiesUnit slackProperties;
|
||||
private final SlackSecretProperties.SlackSecretPropertiesUnit slackProperties;
|
||||
private final SlackAPIClient slackAPIClient;
|
||||
private final String token;
|
||||
|
||||
public MongeShoppingBotSlackMessageSender(SlackSecretProperties slackSecretProperties) {
|
||||
super(slackSecretProperties.find(SLACK_SECRET_UNIT_NAME).getToken());
|
||||
public MongeShoppingBotSlackMessageSender(SlackAPIClient slackAPIClient,
|
||||
SlackSecretProperties slackSecretProperties) {
|
||||
this.slackAPIClient = slackAPIClient;
|
||||
this.slackProperties = slackSecretProperties.find(SLACK_SECRET_UNIT_NAME);
|
||||
this.token = slackProperties.getToken();
|
||||
}
|
||||
|
||||
public SlackMessageDTO ofMessageTemplate() {
|
|
@ -0,0 +1,11 @@
|
|||
package com.myoa.engineering.crawl.shopping.infra.client.slack;
|
||||
|
||||
import org.springframework.cloud.openfeign.FeignClient;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
|
||||
@FeignClient(value = "slack-api-client", url = "https://slack.com")
|
||||
public interface SlackAPIClient {
|
||||
|
||||
@PostMapping("/api/v1/messages/sendMessage/messengers/slack")
|
||||
String sendMessage();
|
||||
}
|
|
@ -1,17 +1,6 @@
|
|||
package com.myoa.engineering.crawl.ppomppu.sender.infrastructure.client;
|
||||
|
||||
import org.springframework.http.HttpHeaders;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.web.reactive.function.client.WebClient;
|
||||
import org.springframework.web.reactive.function.client.WebClientRequestException;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.sender.dto.SlackMessageDTO;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebClientFilterFactory;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebFluxExchangeStragiesFactory;
|
||||
package com.myoa.engineering.crawl.shopping.infra.client.slack;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import reactor.core.publisher.Mono;
|
||||
import reactor.core.scheduler.Schedulers;
|
||||
|
||||
/**
|
||||
* SlackMessageSender
|
||||
|
@ -20,8 +9,9 @@ import reactor.core.scheduler.Schedulers;
|
|||
* @since 2021-09-08
|
||||
*/
|
||||
@Slf4j
|
||||
public class SlackMessageSender implements MessageSender<SlackMessageDTO> {
|
||||
public class SlackMessageSender { /* implements MessageSender<SlackMessageDTO> {*/
|
||||
|
||||
/*
|
||||
private static final String SLACK_API_URL = "https://slack.com/api";
|
||||
|
||||
private final WebClient webClient;
|
||||
|
@ -50,4 +40,6 @@ public class SlackMessageSender implements MessageSender<SlackMessageDTO> {
|
|||
});
|
||||
}
|
||||
|
||||
*/
|
||||
|
||||
}
|
|
@ -0,0 +1,4 @@
|
|||
package com.myoa.engineering.crawl.shopping.infra.repository;
|
||||
|
||||
public interface BaseScanRepository {
|
||||
}
|
|
@ -1,6 +1,6 @@
|
|||
package com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository;
|
||||
package com.myoa.engineering.crawl.shopping.infra.repository.v1;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
|
||||
import com.myoa.engineering.crawl.shopping.domain.entity.v1.PpomppuArticle;
|
||||
import org.springframework.data.jpa.repository.JpaRepository;
|
||||
import org.springframework.stereotype.Repository;
|
||||
|
|
@ -1,11 +1,12 @@
|
|||
package com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository;
|
||||
package com.myoa.engineering.crawl.shopping.infra.repository.v1;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuBoardFeedStatus;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
|
||||
import java.util.Optional;
|
||||
import com.myoa.engineering.crawl.shopping.domain.entity.v1.PpomppuBoardFeedStatus;
|
||||
import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName;
|
||||
import org.springframework.data.jpa.repository.JpaRepository;
|
||||
import org.springframework.stereotype.Repository;
|
||||
|
||||
import java.util.Optional;
|
||||
|
||||
@Repository
|
||||
public interface PpomppuBoardFeedStatusRepository extends JpaRepository<PpomppuBoardFeedStatus, Long> {
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
package com.myoa.engineering.crawl.shopping.infra.repository.v2;
|
||||
|
||||
import com.myoa.engineering.crawl.shopping.domain.entity.v2.AppUser;
|
||||
import org.springframework.data.jpa.repository.JpaRepository;
|
||||
import org.springframework.stereotype.Repository;
|
||||
|
||||
@Repository
|
||||
public interface AppUserRepository extends JpaRepository<AppUser, Long> {
|
||||
}
|
|
@ -0,0 +1,16 @@
|
|||
package com.myoa.engineering.crawl.shopping.infra.repository.v2;
|
||||
|
||||
import com.myoa.engineering.crawl.shopping.domain.entity.v2.Article;
|
||||
import org.springframework.data.jpa.repository.JpaRepository;
|
||||
import org.springframework.stereotype.Repository;
|
||||
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
@Repository
|
||||
public interface ArticleRepository extends JpaRepository<Article, Long> {
|
||||
|
||||
Optional<Article> findByArticleId(Long articleId);
|
||||
List<Article> findByArticleIdIn(Collection<Long> articleId);
|
||||
}
|
|
@ -0,0 +1,14 @@
|
|||
package com.myoa.engineering.crawl.shopping.infra.repository.v2;
|
||||
|
||||
import com.myoa.engineering.crawl.shopping.domain.entity.v2.SubscribedKeyword;
|
||||
import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget;
|
||||
import org.springframework.data.jpa.repository.JpaRepository;
|
||||
import org.springframework.stereotype.Repository;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Repository
|
||||
public interface SubscribedKeywordRepository extends JpaRepository<SubscribedKeyword, Long> {
|
||||
|
||||
List<SubscribedKeyword> findByCrawlTarget(CrawlTarget crawlTarget);
|
||||
}
|
|
@ -0,0 +1,27 @@
|
|||
package com.myoa.engineering.crawl.shopping.scheduler;
|
||||
|
||||
import com.myoa.engineering.crawl.shopping.crawlhandler.CrawlHandler;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.scheduling.annotation.EnableScheduling;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Slf4j
|
||||
@Component
|
||||
@EnableScheduling
|
||||
public class ParseEventEmitter {
|
||||
private final List<CrawlHandler> crawlHandlers;
|
||||
|
||||
|
||||
public ParseEventEmitter(List<CrawlHandler> crawlHandlers) {
|
||||
this.crawlHandlers = crawlHandlers;
|
||||
}
|
||||
|
||||
// @Scheduled(cron = "0 0/5 * * * ?")
|
||||
public void emit() {
|
||||
log.info("[emitDomesticBoard] trigger fired!");
|
||||
crawlHandlers.forEach(CrawlHandler::handle);
|
||||
|
||||
}
|
||||
}
|
|
@ -0,0 +1,81 @@
|
|||
package com.myoa.engineering.crawl.shopping.service;
|
||||
|
||||
import com.myoa.engineering.crawl.shopping.domain.entity.v2.Article;
|
||||
import com.myoa.engineering.crawl.shopping.domain.model.v2.ArticleModel;
|
||||
import com.myoa.engineering.crawl.shopping.event.ArticleUpsertEvent;
|
||||
import com.myoa.engineering.crawl.shopping.infra.repository.v2.ArticleRepository;
|
||||
import jakarta.transaction.Transactional;
|
||||
import org.springframework.context.ApplicationEventPublisher;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.function.Function;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Service
|
||||
public class ArticleCommandService {
|
||||
|
||||
private final ArticleRepository articleRepository;
|
||||
private final ApplicationEventPublisher applicationEventPublisher;
|
||||
|
||||
public ArticleCommandService(ArticleRepository articleRepository,
|
||||
ApplicationEventPublisher applicationEventPublisher) {
|
||||
this.articleRepository = articleRepository;
|
||||
this.applicationEventPublisher = applicationEventPublisher;
|
||||
}
|
||||
|
||||
|
||||
@Transactional
|
||||
public void upsert(List<Article> articles) {
|
||||
Map<Long, Article> articleMap = articles.stream()
|
||||
.collect(Collectors.toMap(Article::getArticleId, e -> e));
|
||||
|
||||
List<Article> saved = articleRepository.findByArticleIdIn(articleMap.keySet());
|
||||
List<Article> updated = saved.stream()
|
||||
.filter(e -> articleMap.containsKey(e.getArticleId()))
|
||||
.map(e -> e.update(articleMap.get(e.getArticleId())))
|
||||
.peek(e -> articleMap.remove(e.getArticleId()))
|
||||
.sorted(Comparator.comparing(Article::getArticleId))
|
||||
.toList();
|
||||
|
||||
List<Article> newArticles = articleMap.values()
|
||||
.stream()
|
||||
.sorted(Comparator.comparing(Article::getArticleId))
|
||||
.toList();
|
||||
|
||||
articleRepository.saveAll(updated);
|
||||
articleRepository.saveAll(newArticles);
|
||||
|
||||
publish(newArticles);
|
||||
}
|
||||
|
||||
private void publish(List<Article> articles) {
|
||||
List<ArticleModel> articleModels =
|
||||
articles.stream()
|
||||
.map(transformer)
|
||||
.toList();
|
||||
applicationEventPublisher.publishEvent(new ArticleUpsertEvent(articleModels));
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public void upsert(Article article) {
|
||||
Article saved = articleRepository.findByArticleId(article.getArticleId())
|
||||
.orElse(article);
|
||||
articleRepository.save(saved);
|
||||
}
|
||||
|
||||
public static Function<Article, ArticleModel> transformer =
|
||||
article -> ArticleModel.builder()
|
||||
.id(article.getId())
|
||||
.articleId(article.getArticleId())
|
||||
.crawlTarget(article.getCrawlTarget())
|
||||
.boardName(article.getBoardName())
|
||||
.articleUrl(article.getArticleUrl())
|
||||
.title(article.getTitle())
|
||||
.hit(article.getHit())
|
||||
.recommended(article.getRecommended())
|
||||
.registeredAt(article.getRegisteredAt())
|
||||
.build();
|
||||
}
|
|
@ -1,15 +1,7 @@
|
|||
package com.myoa.engineering.crawl.ppomppu.processor.service;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
|
||||
import com.myoa.engineering.crawl.ppomppu.processor.dto.PpomppuArticleTransformer;
|
||||
import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.client.MessageSenderAPIClient;
|
||||
package com.myoa.engineering.crawl.shopping.service;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import reactor.core.publisher.Mono;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
/**
|
||||
* MessageSenderService
|
||||
|
@ -21,18 +13,20 @@ import reactor.core.publisher.Mono;
|
|||
@Service
|
||||
public class MessageSenderService {
|
||||
|
||||
/*
|
||||
private final MessageSenderAPIClient messageSenderAPIClient;
|
||||
|
||||
public MessageSenderService(MessageSenderAPIClient messageSenderAPIClient) {
|
||||
this.messageSenderAPIClient = messageSenderAPIClient;
|
||||
}
|
||||
|
||||
public Mono<String> sendMessageToSlack(PpomppuArticle article) {
|
||||
public String sendMessageToSlack(PpomppuArticle article) {
|
||||
return messageSenderAPIClient.sendMessageToSlack(PpomppuArticleTransformer.TRANSFORM_TO_MESSAGE_DTO.apply(article));
|
||||
}
|
||||
|
||||
public Mono<String> sendMessageToSlack(List<PpomppuArticle> articles) {
|
||||
public String sendMessageToSlack(List<PpomppuArticle> articles) {
|
||||
return messageSenderAPIClient.sendMessageToSlack(PpomppuArticleTransformer.transform(articles));
|
||||
}
|
||||
*/
|
||||
|
||||
}
|
|
@ -1,20 +1,18 @@
|
|||
package com.myoa.engineering.crawl.ppomppu.processor.service;
|
||||
package com.myoa.engineering.crawl.shopping.service;
|
||||
|
||||
import com.myoa.engineering.crawl.shopping.domain.entity.v1.PpomppuArticle;
|
||||
import com.myoa.engineering.crawl.shopping.domain.entity.v1.PpomppuBoardFeedStatus;
|
||||
import com.myoa.engineering.crawl.shopping.infra.repository.v1.PpomppuArticleRepository;
|
||||
import com.myoa.engineering.crawl.shopping.infra.repository.v1.PpomppuBoardFeedStatusRepository;
|
||||
import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
|
||||
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuBoardFeedStatus;
|
||||
import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository.PpomppuArticleRepository;
|
||||
import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository.PpomppuBoardFeedStatusRepository;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
public class PpomppuArticleService {
|
|
@ -1,16 +1,12 @@
|
|||
package com.myoa.engineering.crawl.ppomppu.processor.service;
|
||||
package com.myoa.engineering.crawl.shopping.service;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
|
||||
import com.myoa.engineering.crawl.ppomppu.processor.dto.PpomppuArticleParser;
|
||||
import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.client.PpomppuBoardFeedRetriever;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
|
||||
import java.util.List;
|
||||
import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.jsoup.Jsoup;
|
||||
import org.jsoup.nodes.Element;
|
||||
import org.springframework.stereotype.Component;
|
||||
import reactor.core.publisher.Flux;
|
||||
import reactor.core.publisher.Mono;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* PpomppuFeedService
|
||||
|
@ -21,7 +17,7 @@ import reactor.core.publisher.Mono;
|
|||
@Slf4j
|
||||
@Component
|
||||
public class PpomppuFeedService {
|
||||
|
||||
/*
|
||||
private final PpomppuBoardFeedRetriever ppomppuBoardFeedRetriever;
|
||||
|
||||
public PpomppuFeedService(PpomppuBoardFeedRetriever ppomppuBoardFeedRetriever) {
|
||||
|
@ -56,4 +52,6 @@ public class PpomppuFeedService {
|
|||
private PpomppuArticle convertFromElement(Element element) {
|
||||
return PpomppuArticleParser.toArticle(element.getElementsByTag("td"));
|
||||
}
|
||||
|
||||
*/
|
||||
}
|
|
@ -0,0 +1,26 @@
|
|||
package com.myoa.engineering.crawl.shopping.service;
|
||||
|
||||
import com.myoa.engineering.crawl.shopping.domain.entity.v2.SubscribedKeyword;
|
||||
import com.myoa.engineering.crawl.shopping.infra.repository.v2.SubscribedKeywordRepository;
|
||||
import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Service
|
||||
public class SubscribedKeywordQueryService {
|
||||
|
||||
private final SubscribedKeywordRepository subscribedKeywordRepository;
|
||||
|
||||
public SubscribedKeywordQueryService(SubscribedKeywordRepository subscribedKeywordRepository) {
|
||||
this.subscribedKeywordRepository = subscribedKeywordRepository;
|
||||
}
|
||||
|
||||
public List<SubscribedKeyword> findAll() {
|
||||
return subscribedKeywordRepository.findAll();
|
||||
}
|
||||
|
||||
public List<SubscribedKeyword> findByCrawlTarget(CrawlTarget crawlTarget) {
|
||||
return subscribedKeywordRepository.findByCrawlTarget(crawlTarget);
|
||||
}
|
||||
}
|
|
@ -0,0 +1,27 @@
|
|||
package com.myoa.engineering.crawl.shopping.util;
|
||||
|
||||
import java.time.*;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
|
||||
public final class DateTimeUtils {
|
||||
|
||||
private static final DateTimeFormatter FORMATTER_HHMMss = DateTimeFormatter.ofPattern("HH:mm:ss");
|
||||
private static final ZoneId ZONE_ASIA_SEOUL = ZoneId.of("Asia/Seoul");
|
||||
|
||||
private DateTimeUtils() {
|
||||
}
|
||||
|
||||
public static ZonedDateTime parse(String HHMMss) {
|
||||
try {
|
||||
LocalTime time = LocalTime.parse(HHMMss, FORMATTER_HHMMss);
|
||||
LocalDateTime dateTime = LocalDateTime.of(LocalDate.now(), time);
|
||||
if (dateTime.isAfter(LocalDateTime.now())) {
|
||||
dateTime = dateTime.minusDays(1);
|
||||
}
|
||||
return dateTime.atZone(ZONE_ASIA_SEOUL);
|
||||
} catch (Exception e) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
|
@ -0,0 +1,32 @@
|
|||
package com.myoa.engineering.crawl.shopping.util;
|
||||
|
||||
public final class NumberUtils {
|
||||
|
||||
private NumberUtils() {
|
||||
}
|
||||
|
||||
public static Integer parseInt(String value) {
|
||||
return parseInt(value, null);
|
||||
}
|
||||
|
||||
public static Integer parseInt(String value, Integer defaultValue) {
|
||||
try {
|
||||
return Integer.parseInt(value);
|
||||
} catch (NumberFormatException e) {
|
||||
return defaultValue;
|
||||
}
|
||||
}
|
||||
|
||||
public static Long parseLong(String value) {
|
||||
return parseLong(value, null);
|
||||
}
|
||||
|
||||
public static Long parseLong(String value, Long defaultValue) {
|
||||
try {
|
||||
return Long.parseLong(value);
|
||||
} catch (NumberFormatException e) {
|
||||
return defaultValue;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue