Compare commits
31 Commits
3f912d8c84
...
feature/PP
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6d315e2a9f | ||
|
|
8eb431a812 | ||
|
|
520a651a70 | ||
|
|
7b230fdb74 | ||
|
|
b204b70b79 | ||
|
|
0524a18ee5 | ||
|
|
541490d9ac | ||
|
|
9adcecb04f | ||
|
|
ed96cbab8f | ||
| 26520fba79 | |||
|
|
a0c3962e0d | ||
|
|
c97c8dc01f | ||
|
|
1505227037 | ||
|
|
bbf4affc16 | ||
|
|
24a848dc9f | ||
|
|
5b4b44f093 | ||
| 0afa52aa53 | |||
|
|
27dd2893bd | ||
| 4aa0a6e50b | |||
|
|
b9c96d4447 | ||
|
|
b22b1675e9 | ||
| 22ac349d26 | |||
| b03fe85ad5 | |||
|
|
86fa1cbe09 | ||
|
|
ab4ab339f6 | ||
| 398a94ce3f | |||
|
|
cf7425faae | ||
|
|
08e1f99ab0 | ||
| 408b3de1c7 | |||
|
|
b7fe17ab2c | ||
|
|
98c34bb468 |
2
.gitignore
vendored
2
.gitignore
vendored
@@ -35,3 +35,5 @@ out/
|
||||
|
||||
### VS Code ###
|
||||
.vscode/
|
||||
|
||||
temppassword.yml
|
||||
46
PpomppuNotifier_DB.sql
Normal file
46
PpomppuNotifier_DB.sql
Normal file
@@ -0,0 +1,46 @@
|
||||
create table "ppomppu_article"
|
||||
(
|
||||
"id" bigint generated by default as identity,
|
||||
"article_id" bigint,
|
||||
"article_url" varchar(255),
|
||||
"board_name" integer,
|
||||
"hit" integer,
|
||||
"recommended" integer,
|
||||
"registered_at" timestamp,
|
||||
"title" varchar(255),
|
||||
primary key ("id")
|
||||
)
|
||||
|
||||
create table "ppomppu_board_feed_status"
|
||||
(
|
||||
"id" bigint generated by default as identity,
|
||||
"board_name" integer,
|
||||
"latest_parsed_article_id" bigint,
|
||||
"updated_at" timestamp,
|
||||
primary key ("id")
|
||||
)
|
||||
|
||||
create table "published_history"
|
||||
(
|
||||
"id" bigint generated by default as identity,
|
||||
"board_name_list" varchar(255),
|
||||
"published_at" timestamp,
|
||||
"user_id" bigint,
|
||||
primary key ("id")
|
||||
)
|
||||
|
||||
create table "subscribed_board"
|
||||
(
|
||||
"id" bigint generated by default as identity,
|
||||
"board_name" integer,
|
||||
"user_id" bigint,
|
||||
primary key ("id")
|
||||
)
|
||||
|
||||
create table "subscribed_user"
|
||||
(
|
||||
"id" bigint generated by default as identity,
|
||||
"registered_at" timestamp,
|
||||
"user_id" bigint,
|
||||
primary key ("id")
|
||||
)
|
||||
54
PpomppuNotifier_ERD.puml
Normal file
54
PpomppuNotifier_ERD.puml
Normal file
@@ -0,0 +1,54 @@
|
||||
@startuml
|
||||
'https://plantuml.com/sequence-diagram
|
||||
|
||||
class SubscribedUser {
|
||||
- id
|
||||
+ user_id
|
||||
+ registered_at
|
||||
+ created_at
|
||||
+ modified_at
|
||||
}
|
||||
|
||||
class SubscribedBoard {
|
||||
- id
|
||||
- user_id
|
||||
+ board_name
|
||||
+ created_at
|
||||
+ modified_at
|
||||
}
|
||||
|
||||
class PublishedHisotry {
|
||||
- id
|
||||
+ user_id
|
||||
+ board_name_list
|
||||
+ published_at
|
||||
+ created_at
|
||||
+ modified_at
|
||||
}
|
||||
|
||||
class PpomppuArticle {
|
||||
- id
|
||||
+ article_id
|
||||
+ board_name
|
||||
+ article_url
|
||||
+ title
|
||||
+ recommended
|
||||
+ hit
|
||||
+ registered_at
|
||||
+ created_at
|
||||
+ modified_at
|
||||
}
|
||||
|
||||
class PpomppuBoardFeedStatus {
|
||||
- id
|
||||
+ board_name
|
||||
+ latest_parsed_article_id
|
||||
+ updated_at
|
||||
+ created_at
|
||||
+ modified_at
|
||||
}
|
||||
|
||||
SubscribedUser --o{ SubscribedBoard
|
||||
SubscribedUser --o{ PublishedHisotry
|
||||
|
||||
@enduml
|
||||
16
build.gradle
16
build.gradle
@@ -6,7 +6,7 @@ plugins {
|
||||
}
|
||||
|
||||
group = 'com.myoa.engineering.crawl.ppomppu'
|
||||
version = '0.0.1-SNAPSHOT'
|
||||
version = '1.1.1'
|
||||
sourceCompatibility = '11'
|
||||
|
||||
configurations {
|
||||
@@ -20,6 +20,9 @@ repositories {
|
||||
}
|
||||
|
||||
allprojects {
|
||||
group = 'com.myoa.engineering.crawl.ppomppu'
|
||||
version = '1.1.1'
|
||||
|
||||
apply plugin: 'java'
|
||||
apply plugin: 'idea'
|
||||
apply plugin: 'org.springframework.boot'
|
||||
@@ -32,6 +35,17 @@ allprojects {
|
||||
}
|
||||
}
|
||||
|
||||
ext {
|
||||
set('springCloudVersion', "2020.0.4")
|
||||
}
|
||||
|
||||
dependencyManagement {
|
||||
imports {
|
||||
mavenBom "org.springframework.cloud:spring-cloud-dependencies:${springCloudVersion}"
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/* task initSourceFolders {
|
||||
sourceSets*.java.srcDirs*.each {
|
||||
if( !it.exists() ) {
|
||||
|
||||
3
copy.bat
Normal file
3
copy.bat
Normal file
@@ -0,0 +1,3 @@
|
||||
xcopy /y .\processor\build\libs\*.jar .\
|
||||
xcopy /y .\receiver\build\libs\*.jar .\
|
||||
xcopy /y .\sender\build\libs\*.jar .\
|
||||
30
processor/build.gradle
Normal file
30
processor/build.gradle
Normal file
@@ -0,0 +1,30 @@
|
||||
|
||||
dependencies {
|
||||
developmentOnly 'org.springframework.boot:spring-boot-devtools'
|
||||
runtimeOnly 'com.h2database:h2'
|
||||
runtimeOnly 'mysql:mysql-connector-java'
|
||||
compileOnly 'org.projectlombok:lombok'
|
||||
|
||||
implementation project(':support')
|
||||
// https://projectreactor.io/docs/core/release/reference/#debug-activate
|
||||
implementation 'org.springframework.boot:spring-boot-starter-webflux'
|
||||
implementation 'org.springframework.boot:spring-boot-starter-data-jpa'
|
||||
implementation 'org.springframework.boot:spring-boot-starter-jdbc'
|
||||
implementation 'org.springframework.cloud:spring-cloud-starter-config'
|
||||
implementation 'org.springframework.boot:spring-boot-starter-actuator'
|
||||
implementation 'com.rometools:rome:1.16.0'
|
||||
implementation 'org.jsoup:jsoup:1.14.2'
|
||||
implementation 'com.h2database:h2:1.4.200'
|
||||
|
||||
annotationProcessor 'org.springframework.boot:spring-boot-configuration-processor'
|
||||
annotationProcessor 'org.projectlombok:lombok'
|
||||
|
||||
testImplementation 'org.springframework.boot:spring-boot-starter-test'
|
||||
}
|
||||
|
||||
test {
|
||||
useJUnitPlatform()
|
||||
testLogging {
|
||||
events "passed", "skipped", "failed"
|
||||
}
|
||||
}
|
||||
@@ -1,10 +1,23 @@
|
||||
package com.myoa.engineering.crawl.ppomppu.processor;
|
||||
|
||||
import org.springframework.boot.SpringApplication;
|
||||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||
import org.springframework.context.annotation.Import;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.support.webclient.PpomppuNotifierWebClientConfiguration;
|
||||
|
||||
/**
|
||||
* ProcessorApplication
|
||||
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
|
||||
* @since 2021-08-20
|
||||
*
|
||||
*/
|
||||
@Import({ PpomppuNotifierWebClientConfiguration.class })
|
||||
@SpringBootApplication
|
||||
public class ProcessorApplication {
|
||||
|
||||
public static void main(String[] args) {
|
||||
SpringApplication.run(ProcessorApplication.class, args);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
package com.myoa.engineering.crawl.ppomppu.processor.configuration;
|
||||
|
||||
import java.sql.SQLException;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.h2.tools.Server;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.context.annotation.Profile;
|
||||
import org.springframework.context.event.ContextClosedEvent;
|
||||
import org.springframework.context.event.ContextRefreshedEvent;
|
||||
import org.springframework.context.event.EventListener;
|
||||
|
||||
@Slf4j
|
||||
@Profile({"datasource-local", "datasource-development"})
|
||||
@Configuration
|
||||
public class H2ConsoleConfiguration {
|
||||
|
||||
private Server webServer;
|
||||
|
||||
@Value("${spring.h2.console.port}")
|
||||
private String port;
|
||||
|
||||
@EventListener(ContextRefreshedEvent.class)
|
||||
public void start() throws SQLException {
|
||||
log.info("starting h2 console");
|
||||
this.webServer = Server.createWebServer("-webPort", port, "-tcpAllowOthers").start();
|
||||
}
|
||||
|
||||
@EventListener(ContextClosedEvent.class)
|
||||
public void stop() {
|
||||
log.info("stopping h2 console");
|
||||
this.webServer.stop(); ;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,125 @@
|
||||
package com.myoa.engineering.crawl.ppomppu.processor.configuration;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.processor.configuration.properties.DatasourceProperties;
|
||||
import com.myoa.engineering.crawl.ppomppu.processor.configuration.properties.DatasourceProperties.DataSourcePropertiesUnit;
|
||||
import com.myoa.engineering.crawl.ppomppu.processor.configuration.properties.HibernateProperties;
|
||||
import com.myoa.engineering.crawl.ppomppu.processor.configuration.properties.HikariProperties;
|
||||
import com.myoa.engineering.crawl.ppomppu.processor.domain.BaseScanDomain;
|
||||
import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository.BaseScanRepository;
|
||||
import com.zaxxer.hikari.HikariConfig;
|
||||
import com.zaxxer.hikari.HikariDataSource;
|
||||
import java.util.Enumeration;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.Properties;
|
||||
import javax.persistence.EntityManagerFactory;
|
||||
import javax.sql.DataSource;
|
||||
import lombok.NonNull;
|
||||
import org.hibernate.cfg.AvailableSettings;
|
||||
import org.springframework.beans.factory.annotation.Qualifier;
|
||||
import org.springframework.boot.orm.jpa.EntityManagerFactoryBuilder;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.data.jpa.repository.config.EnableJpaRepositories;
|
||||
import org.springframework.orm.jpa.JpaTransactionManager;
|
||||
import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean;
|
||||
import org.springframework.transaction.PlatformTransactionManager;
|
||||
|
||||
@Configuration
|
||||
@EnableJpaRepositories(basePackageClasses = BaseScanRepository.class,
|
||||
entityManagerFactoryRef = "ppomppuNotifierProcessorEntityManagerFactory",
|
||||
transactionManagerRef = "ppomppuNotifierProcessorTransactionManager"
|
||||
)
|
||||
public class PpomppuDatasourceConfiguration {
|
||||
|
||||
private static final String DATA_SOURCE_UNIT_NAME = "ppn_mysql";
|
||||
|
||||
private final DatasourceProperties dataSourceProeprties;
|
||||
private final HikariProperties hikariProperties;
|
||||
private final HibernateProperties hibernateProperties;
|
||||
|
||||
public PpomppuDatasourceConfiguration(DatasourceProperties dataSourceProeprties,
|
||||
HikariProperties hikariProperties,
|
||||
HibernateProperties hibernateProperties) {
|
||||
this.dataSourceProeprties = dataSourceProeprties;
|
||||
this.hikariProperties = hikariProperties;
|
||||
this.hibernateProperties = hibernateProperties;
|
||||
}
|
||||
|
||||
@Bean(name = "ppomppuNotifierProcessorDataSource")
|
||||
public DataSource dataSource() {
|
||||
DataSourcePropertiesUnit dataSourcePropertiesUnit = dataSourceProeprties.find(DATA_SOURCE_UNIT_NAME);
|
||||
|
||||
final HikariConfig hikariConfig = new HikariConfig();
|
||||
hikariConfig.setJdbcUrl(dataSourcePropertiesUnit.toCompletedJdbcUrl());
|
||||
hikariConfig.setDriverClassName(dataSourcePropertiesUnit.getDriverClassName());
|
||||
hikariConfig.setUsername(dataSourcePropertiesUnit.getUsername());
|
||||
hikariConfig.setPassword(dataSourcePropertiesUnit.getPassword());
|
||||
hikariConfig.setAutoCommit(hikariProperties.getAutoCommit());
|
||||
hikariConfig.setMaximumPoolSize(hikariProperties.getMaximumPoolSize());
|
||||
hikariConfig.setMinimumIdle(hikariProperties.getMinimumIdle());
|
||||
if (hikariProperties.getMaximumPoolSize() > hikariProperties.getMinimumIdle()) {
|
||||
hikariConfig.setIdleTimeout(hikariProperties.getIdleTimeout());
|
||||
}
|
||||
hikariConfig.setValidationTimeout(hikariProperties.getValidationTimeout());
|
||||
hikariConfig.setConnectionTimeout(hikariProperties.getConnectionTimeout());
|
||||
hikariConfig.setMaxLifetime(hikariProperties.getMaxLifetime());
|
||||
|
||||
final DataSource dataSource = new HikariDataSource(hikariConfig);
|
||||
return dataSource;
|
||||
}
|
||||
|
||||
@Bean("ppomppuNotifierProcessorEntityManagerFactory")
|
||||
public LocalContainerEntityManagerFactoryBean entityManagerFactory(
|
||||
EntityManagerFactoryBuilder builder,
|
||||
@Qualifier("ppomppuNotifierProcessorDataSource") DataSource dataSource) {
|
||||
return builder.dataSource(dataSource)
|
||||
.packages(BaseScanDomain.class)
|
||||
.properties(getPropsMap(DATA_SOURCE_UNIT_NAME))
|
||||
.build();
|
||||
}
|
||||
|
||||
@Bean("ppomppuNotifierProcessorTransactionManager")
|
||||
public PlatformTransactionManager transactionManager(
|
||||
@Qualifier("ppomppuNotifierProcessorEntityManagerFactory") EntityManagerFactory entityManagerFactory) {
|
||||
return new JpaTransactionManager(entityManagerFactory);
|
||||
}
|
||||
|
||||
public static Properties getProps(@NonNull HibernateProperties.HibernatePropertiesUnit hibernateProperties) {
|
||||
Properties properties = new Properties();
|
||||
properties.put(AvailableSettings.DIALECT, hibernateProperties.getDialect());
|
||||
properties.put(AvailableSettings.FORMAT_SQL, hibernateProperties.getFormatSql());
|
||||
properties.put(AvailableSettings.SHOW_SQL, hibernateProperties.getShowSql());
|
||||
properties.put(AvailableSettings.HBM2DDL_AUTO, hibernateProperties.getHbm2ddlAuto());
|
||||
properties.put(AvailableSettings.CONNECTION_PROVIDER_DISABLES_AUTOCOMMIT,
|
||||
hibernateProperties.getDisableAutoCommit());
|
||||
properties.put(AvailableSettings.IMPLICIT_NAMING_STRATEGY,
|
||||
"org.springframework.boot.orm.jpa.hibernate.SpringImplicitNamingStrategy");
|
||||
properties.put(AvailableSettings.PHYSICAL_NAMING_STRATEGY,
|
||||
"org.springframework.boot.orm.jpa.hibernate.SpringPhysicalNamingStrategy");
|
||||
properties.put(AvailableSettings.GENERATE_STATISTICS, "false");
|
||||
properties.put(AvailableSettings.GLOBALLY_QUOTED_IDENTIFIERS, "true");
|
||||
properties.put(AvailableSettings.GLOBALLY_QUOTED_IDENTIFIERS_SKIP_COLUMN_DEFINITIONS, "true");
|
||||
properties.put(AvailableSettings.STATEMENT_BATCH_SIZE, "20");
|
||||
properties.put(AvailableSettings.ORDER_INSERTS, "true");
|
||||
properties.put(AvailableSettings.ORDER_UPDATES, "true");
|
||||
properties.put(AvailableSettings.BATCH_VERSIONED_DATA, "true");
|
||||
properties.put(AvailableSettings.USE_NEW_ID_GENERATOR_MAPPINGS, "false");
|
||||
return properties;
|
||||
}
|
||||
|
||||
public Map<String, String> getPropsMap(@NonNull String unitName) {
|
||||
return convertPropertiestoMaps(getProps(hibernateProperties.find(unitName)));
|
||||
}
|
||||
|
||||
public Map<String, String> convertPropertiestoMaps(Properties properties) {
|
||||
Map<String, String> propertiesMap = new HashMap<>();
|
||||
|
||||
for (Enumeration<?> e = properties.propertyNames(); e.hasMoreElements(); ) {
|
||||
String key = (String) e.nextElement();
|
||||
propertiesMap.put(key, properties.getProperty(key));
|
||||
}
|
||||
return propertiesMap;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
package com.myoa.engineering.crawl.ppomppu.processor.configuration.properties;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.support.util.ObjectUtil;
|
||||
import java.util.List;
|
||||
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
@Component
|
||||
@Setter
|
||||
@Getter
|
||||
@ConfigurationProperties(prefix = "infra.database")
|
||||
public class DatasourceProperties {
|
||||
|
||||
private List<DataSourcePropertiesUnit> units;
|
||||
|
||||
@Getter
|
||||
@Setter
|
||||
public static class DataSourcePropertiesUnit {
|
||||
|
||||
private String unitName;
|
||||
private String schemaName;
|
||||
private String connectionParameters;
|
||||
private String datasourceUrl;
|
||||
private Boolean isSimpleConnectionUrl;
|
||||
private String username;
|
||||
private String password;
|
||||
private String driverClassName;
|
||||
|
||||
public String toCompletedJdbcUrl() {
|
||||
if (ObjectUtil.isEmpty(isSimpleConnectionUrl) || isSimpleConnectionUrl == false) {
|
||||
return String.format("%s/%s?%s", datasourceUrl, schemaName, connectionParameters);
|
||||
}
|
||||
return datasourceUrl;
|
||||
}
|
||||
}
|
||||
|
||||
public DataSourcePropertiesUnit find(String unitName) {
|
||||
return units.stream()
|
||||
.filter(e -> e.getUnitName().equals(unitName))
|
||||
.findFirst()
|
||||
.orElseThrow(
|
||||
() -> new IllegalArgumentException(this.getClass().getName() + ": unitName Not found. " + unitName));
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
package com.myoa.engineering.crawl.ppomppu.processor.configuration.properties;
|
||||
|
||||
import java.util.List;
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
|
||||
import org.hibernate.dialect.MySQL8Dialect;
|
||||
import org.hibernate.dialect.MySQLDialect;
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
@Component
|
||||
@Setter
|
||||
@Getter
|
||||
@ConfigurationProperties(prefix = "hibernate")
|
||||
public class HibernateProperties {
|
||||
|
||||
private List<HibernatePropertiesUnit> units;
|
||||
|
||||
@Getter
|
||||
@Setter
|
||||
public static class HibernatePropertiesUnit {
|
||||
|
||||
private String unitName;
|
||||
private String dialect;
|
||||
private String formatSql;
|
||||
private String showSql;
|
||||
private String hbm2ddlAuto;
|
||||
private String disableAutoCommit;
|
||||
|
||||
}
|
||||
|
||||
public HibernatePropertiesUnit find(String unitName) {
|
||||
return units.stream()
|
||||
.filter(x -> x.getUnitName().equals(unitName))
|
||||
.findFirst()
|
||||
.orElseThrow(
|
||||
() -> new IllegalArgumentException(this.getClass().getName() + ": unitName Not found. " + unitName));
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
package com.myoa.engineering.crawl.ppomppu.processor.configuration.properties;
|
||||
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
@Component
|
||||
@Setter
|
||||
@Getter
|
||||
@ConfigurationProperties(prefix = "spring.datasource.hikari")
|
||||
public class HikariProperties {
|
||||
|
||||
private Integer minimumIdle;
|
||||
private Integer maximumPoolSize;
|
||||
private Integer idleTimeout;
|
||||
private Integer validationTimeout;
|
||||
private Integer connectionTimeout;
|
||||
private Integer maxLifetime;
|
||||
private Boolean autoCommit;
|
||||
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
package com.myoa.engineering.crawl.ppomppu.processor.controller;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
|
||||
import com.myoa.engineering.crawl.ppomppu.processor.dto.FeedParsedResult;
|
||||
import com.myoa.engineering.crawl.ppomppu.processor.service.MessageSenderService;
|
||||
import com.myoa.engineering.crawl.ppomppu.processor.service.PpomppuArticleService;
|
||||
import com.myoa.engineering.crawl.ppomppu.processor.service.PpomppuFeedService;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.dto.APIResponse;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import reactor.core.publisher.Mono;
|
||||
|
||||
/**
|
||||
* CrawlAPIController
|
||||
*
|
||||
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
|
||||
* @since 2021-09-05
|
||||
*/
|
||||
@Slf4j
|
||||
@RestController
|
||||
@RequestMapping("/api/v1/crawl")
|
||||
public class CrawlAPIController {
|
||||
|
||||
private final PpomppuFeedService ppomppuRSSFeedService;
|
||||
private final PpomppuArticleService ppomppuArticleService;
|
||||
private final MessageSenderService messageSenderService;
|
||||
|
||||
public CrawlAPIController(PpomppuFeedService ppomppuRSSFeedService,
|
||||
PpomppuArticleService ppomppuArticleService,
|
||||
MessageSenderService messageSenderService) {
|
||||
this.ppomppuRSSFeedService = ppomppuRSSFeedService;
|
||||
this.ppomppuArticleService = ppomppuArticleService;
|
||||
this.messageSenderService = messageSenderService;
|
||||
}
|
||||
|
||||
@PostMapping("/boards/{boardName}")
|
||||
public Mono<APIResponse<FeedParsedResult>> crawlBoard(@PathVariable("boardName") PpomppuBoardName boardName) {
|
||||
log.info("got request... {}", boardName);
|
||||
FeedParsedResult result = FeedParsedResult.of(boardName);
|
||||
|
||||
Mono<String> publishedMessages =
|
||||
ppomppuRSSFeedService.getArticles(boardName)
|
||||
.map(e -> ppomppuArticleService.filterOnlyNewArticles(boardName, e))
|
||||
.map(e -> ppomppuArticleService.save(boardName, e))
|
||||
.filter(e -> !e.isEmpty())
|
||||
.flatMap(e -> messageSenderService.sendBlockMessageToSlack(boardName, e));
|
||||
|
||||
return publishedMessages.then(Mono.just(APIResponse.success(result.done())));
|
||||
}
|
||||
|
||||
@PostMapping("/exploit/boards/{boardName}")
|
||||
public Mono<APIResponse<String>> crawlBoardDryRun(
|
||||
@PathVariable("boardName") PpomppuBoardName boardName) {
|
||||
log.info("got request... {}", boardName);
|
||||
Mono<String> publishedMessages =
|
||||
ppomppuRSSFeedService.getArticles(boardName)
|
||||
.flatMap(e -> messageSenderService.sendBlockMessageToSlack(boardName, e));
|
||||
|
||||
return publishedMessages.map(APIResponse::success);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
package com.myoa.engineering.crawl.ppomppu.processor.domain;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.time.Instant;
|
||||
import javax.persistence.Column;
|
||||
import org.springframework.data.annotation.CreatedDate;
|
||||
import org.springframework.data.annotation.LastModifiedDate;
|
||||
|
||||
/**
|
||||
* Auditable
|
||||
*
|
||||
* @author Shin Woo-jin (woozu.shin@kakaoent.com)
|
||||
* @since 2021-09-08
|
||||
*/
|
||||
public abstract class Auditable implements Serializable {
|
||||
private static final long serialVersionUID = -7105030870015828551L;
|
||||
|
||||
@Column
|
||||
@CreatedDate
|
||||
private Instant createdAt;
|
||||
|
||||
@Column
|
||||
@LastModifiedDate
|
||||
private Instant modifiedAt;
|
||||
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
package com.myoa.engineering.crawl.ppomppu.processor.domain;
|
||||
|
||||
/**
|
||||
* BaseScanDomain
|
||||
*
|
||||
* @author Shin Woo-jin (woozu.shin@kakaoent.com)
|
||||
* @since 2021-09-08
|
||||
*/
|
||||
public interface BaseScanDomain {
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
package com.myoa.engineering.crawl.ppomppu.processor.domain;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
|
||||
import java.time.Instant;
|
||||
import javax.persistence.Column;
|
||||
import javax.persistence.Entity;
|
||||
import javax.persistence.EnumType;
|
||||
import javax.persistence.Enumerated;
|
||||
import javax.persistence.GeneratedValue;
|
||||
import javax.persistence.GenerationType;
|
||||
import javax.persistence.Id;
|
||||
import javax.persistence.Table;
|
||||
import lombok.Builder;
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Getter
|
||||
@NoArgsConstructor
|
||||
@Entity
|
||||
@Table(name = "ppomppu_article")
|
||||
public class PpomppuArticle extends Auditable {
|
||||
|
||||
@Id
|
||||
@GeneratedValue(strategy = GenerationType.IDENTITY)
|
||||
private Long id;
|
||||
|
||||
@Column
|
||||
private Long articleId;
|
||||
|
||||
@Column
|
||||
@Enumerated(EnumType.STRING)
|
||||
private PpomppuBoardName boardName;
|
||||
|
||||
@Column
|
||||
private String articleUrl;
|
||||
|
||||
@Column
|
||||
private String thumbnailUrl;
|
||||
|
||||
@Column
|
||||
private String title;
|
||||
|
||||
@Column
|
||||
private Integer hit;
|
||||
|
||||
@Column
|
||||
private Integer recommended;
|
||||
|
||||
@Column
|
||||
private Instant registeredAt;
|
||||
|
||||
@Builder
|
||||
public PpomppuArticle(Long id, Long articleId, PpomppuBoardName boardName, String articleUrl,
|
||||
String thumbnailUrl, String title, Integer recommended, Integer hit,
|
||||
Instant registeredAt) {
|
||||
this.id = id;
|
||||
this.articleId = articleId;
|
||||
this.boardName = boardName;
|
||||
this.articleUrl = articleUrl;
|
||||
this.thumbnailUrl = thumbnailUrl;
|
||||
this.title = title;
|
||||
this.recommended = recommended;
|
||||
this.hit = hit;
|
||||
this.registeredAt = registeredAt;
|
||||
}
|
||||
|
||||
public PpomppuArticle updateBoardName(PpomppuBoardName boardName) {
|
||||
this.boardName = boardName;
|
||||
return this;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
package com.myoa.engineering.crawl.ppomppu.processor.domain;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
|
||||
import java.time.Instant;
|
||||
import javax.persistence.Column;
|
||||
import javax.persistence.Entity;
|
||||
import javax.persistence.EnumType;
|
||||
import javax.persistence.Enumerated;
|
||||
import javax.persistence.GeneratedValue;
|
||||
import javax.persistence.GenerationType;
|
||||
import javax.persistence.Id;
|
||||
import javax.persistence.Table;
|
||||
import lombok.Builder;
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Getter
|
||||
@NoArgsConstructor
|
||||
@Entity
|
||||
@Table(name = "ppomppu_board_feed_status")
|
||||
public class PpomppuBoardFeedStatus extends Auditable {
|
||||
|
||||
@Id
|
||||
@GeneratedValue(strategy = GenerationType.IDENTITY)
|
||||
private Long id;
|
||||
|
||||
@Column
|
||||
private Long latestParsedArticleId;
|
||||
|
||||
@Column
|
||||
@Enumerated(EnumType.STRING)
|
||||
private PpomppuBoardName boardName;
|
||||
|
||||
@Column
|
||||
private Instant updatedAt;
|
||||
|
||||
public static PpomppuBoardFeedStatus of(PpomppuBoardName boardName, Long latestArticleId) {
|
||||
return PpomppuBoardFeedStatus.builder()
|
||||
.boardName(boardName)
|
||||
.latestParsedArticleId(latestArticleId)
|
||||
.updatedAt(Instant.now())
|
||||
.build();
|
||||
}
|
||||
|
||||
public void updateArticleId(Long latestArticleId) {
|
||||
this.updatedAt = Instant.now();
|
||||
this.latestParsedArticleId = latestArticleId;
|
||||
}
|
||||
|
||||
@Builder
|
||||
public PpomppuBoardFeedStatus(Long id, Long latestParsedArticleId, PpomppuBoardName boardName, Instant updatedAt) {
|
||||
this.id = id;
|
||||
this.latestParsedArticleId = latestParsedArticleId;
|
||||
this.boardName = boardName;
|
||||
this.updatedAt = updatedAt;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
package com.myoa.engineering.crawl.ppomppu.processor.domain;
|
||||
|
||||
import java.time.Instant;
|
||||
import javax.persistence.Column;
|
||||
import javax.persistence.Entity;
|
||||
import javax.persistence.GeneratedValue;
|
||||
import javax.persistence.GenerationType;
|
||||
import javax.persistence.Id;
|
||||
import javax.persistence.Table;
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Getter
|
||||
@NoArgsConstructor
|
||||
@Entity
|
||||
@Table(name = "published_history")
|
||||
public class PublishedHistory extends Auditable {
|
||||
|
||||
@Id
|
||||
@GeneratedValue(strategy = GenerationType.IDENTITY)
|
||||
private Long id;
|
||||
|
||||
@Column
|
||||
private Long userId;
|
||||
|
||||
@Column
|
||||
private String boardNameList;
|
||||
|
||||
@Column
|
||||
private Instant publishedAt;
|
||||
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
package com.myoa.engineering.crawl.ppomppu.processor.domain;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
|
||||
import javax.persistence.Column;
|
||||
import javax.persistence.Entity;
|
||||
import javax.persistence.EnumType;
|
||||
import javax.persistence.Enumerated;
|
||||
import javax.persistence.GeneratedValue;
|
||||
import javax.persistence.GenerationType;
|
||||
import javax.persistence.Id;
|
||||
import javax.persistence.Table;
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Getter
|
||||
@NoArgsConstructor
|
||||
@Entity
|
||||
@Table(name = "subscribed_board")
|
||||
public class SubscribedBoard extends Auditable {
|
||||
|
||||
@Id
|
||||
@GeneratedValue(strategy = GenerationType.IDENTITY)
|
||||
private Long id;
|
||||
|
||||
@Column
|
||||
private Long userId;
|
||||
|
||||
@Column
|
||||
@Enumerated(EnumType.STRING)
|
||||
private PpomppuBoardName boardName;
|
||||
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
package com.myoa.engineering.crawl.ppomppu.processor.domain;
|
||||
|
||||
import java.time.Instant;
|
||||
import javax.persistence.Column;
|
||||
import javax.persistence.Entity;
|
||||
import javax.persistence.GeneratedValue;
|
||||
import javax.persistence.GenerationType;
|
||||
import javax.persistence.Id;
|
||||
import javax.persistence.Table;
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Getter
|
||||
@NoArgsConstructor
|
||||
@Entity
|
||||
@Table(name = "subscribed_user")
|
||||
public class SubscribedUser extends Auditable{
|
||||
|
||||
@Id
|
||||
@GeneratedValue(strategy = GenerationType.IDENTITY)
|
||||
private Long id;
|
||||
|
||||
@Column
|
||||
private Long userId;
|
||||
|
||||
@Column
|
||||
private Instant registeredAt;
|
||||
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
package com.myoa.engineering.crawl.ppomppu.processor.dto;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
|
||||
import java.io.Serializable;
|
||||
import java.time.Instant;
|
||||
import lombok.Builder;
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
/**
|
||||
* FeedParsedResult
|
||||
*
|
||||
* @author Shin Woo-jin (woozu.shin@kakaoent.com)
|
||||
* @since 2021-09-08
|
||||
*/
|
||||
@Getter
|
||||
@NoArgsConstructor
|
||||
public class FeedParsedResult implements Serializable {
|
||||
|
||||
private static final long serialVersionUID = -3771310078623481348L;
|
||||
|
||||
private PpomppuBoardName boardName;
|
||||
private Instant requestedAt;
|
||||
private Instant processedAt;
|
||||
|
||||
@Builder
|
||||
public FeedParsedResult(PpomppuBoardName boardName, Instant requestedAt, Instant processedAt) {
|
||||
this.boardName = boardName;
|
||||
this.requestedAt = requestedAt;
|
||||
this.processedAt = processedAt;
|
||||
}
|
||||
|
||||
public static FeedParsedResult of(PpomppuBoardName boardName) {
|
||||
return FeedParsedResult.builder()
|
||||
.boardName(boardName)
|
||||
.requestedAt(Instant.now())
|
||||
.build();
|
||||
}
|
||||
|
||||
public FeedParsedResult done() {
|
||||
this.processedAt = Instant.now();
|
||||
return this;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
package com.myoa.engineering.crawl.ppomppu.processor.dto;
|
||||
|
||||
import java.time.Instant;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.util.DateUtil;
|
||||
|
||||
import lombok.Builder;
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
/**
|
||||
* PpomppuArticleParseDTO
|
||||
*
|
||||
* @author Shin Woo-jin (woozu.shin@kakaoent.com)
|
||||
* @since 2021-09-08
|
||||
*/
|
||||
@Getter
|
||||
@NoArgsConstructor
|
||||
public class PpomppuArticleParseDTO {
|
||||
|
||||
private String id;
|
||||
private String articleId;
|
||||
private String boardName;
|
||||
private String articleUrl;
|
||||
private String thumbnailUrl;
|
||||
private String title;
|
||||
private String hit;
|
||||
private Integer recommended;
|
||||
private String registeredAt;
|
||||
|
||||
@Builder
|
||||
public PpomppuArticleParseDTO(String id, String articleId, String boardName, String articleUrl,
|
||||
String thumbnailUrl, String title, String hit, Integer recommended,
|
||||
String registeredAt) {
|
||||
this.id = id;
|
||||
this.articleId = articleId;
|
||||
this.boardName = boardName;
|
||||
this.articleUrl = articleUrl;
|
||||
this.thumbnailUrl = thumbnailUrl;
|
||||
this.title = title;
|
||||
this.hit = hit;
|
||||
this.recommended = recommended;
|
||||
this.registeredAt = registeredAt;
|
||||
}
|
||||
|
||||
public boolean isInValidated() {
|
||||
return articleId == null || articleId.isEmpty()
|
||||
|| hit == null || hit.isEmpty();
|
||||
}
|
||||
|
||||
public PpomppuArticle convert() {
|
||||
if (isInValidated()) {
|
||||
throw new IllegalArgumentException("PpomppuArticleParseDTO was invalidated");
|
||||
}
|
||||
return PpomppuArticle.builder()
|
||||
.articleId(Long.parseLong(articleId))
|
||||
.title(title)
|
||||
.articleUrl(articleUrl)
|
||||
.thumbnailUrl(thumbnailUrl)
|
||||
.recommended(recommended)
|
||||
.hit(Integer.parseInt(hit))
|
||||
.registeredAt(DateUtil.DATE_TIME_FORMATTER.parse(registeredAt, Instant::from))
|
||||
.build();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
package com.myoa.engineering.crawl.ppomppu.processor.dto;
|
||||
|
||||
import org.jsoup.nodes.Element;
|
||||
import org.jsoup.select.Elements;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
|
||||
|
||||
/**
|
||||
* PpomppuArticleTransformer
|
||||
*
|
||||
* @author Shin Woo-jin (woozu.shin@kakaoent.com)
|
||||
* @since 2021-09-08
|
||||
*/
|
||||
public final class PpomppuArticleParser {
|
||||
|
||||
private PpomppuArticleParser() {}
|
||||
|
||||
public static PpomppuArticle toArticle(Elements articleElement) {
|
||||
final String articleIdString = PpomppuArticleParser.parseArticleId(articleElement.get(0));
|
||||
final String title = PpomppuArticleParser.parseTitle(articleElement.get(2));
|
||||
final String articleUrl = PpomppuArticleParser.parseArticleUrl(articleElement.get(2));
|
||||
final String thumbnailUrl = PpomppuArticleParser.parseThumbnailUrl(articleElement.get(3));
|
||||
final Integer recommended = PpomppuArticleParser.parseRecommended(articleElement.get(6));
|
||||
final String hitString = PpomppuArticleParser.parseHit(articleElement.get(7));
|
||||
final String registeredAtString = PpomppuArticleParser.parseRegisteredAt(articleElement.get(5));
|
||||
|
||||
return PpomppuArticleParseDTO.builder()
|
||||
.articleId(articleIdString)
|
||||
.title(title)
|
||||
.articleUrl(articleUrl)
|
||||
.thumbnailUrl(thumbnailUrl)
|
||||
.recommended(recommended)
|
||||
.hit(hitString)
|
||||
.registeredAt(registeredAtString)
|
||||
.build()
|
||||
.convert();
|
||||
}
|
||||
|
||||
public static String parseArticleId(Element td) {
|
||||
return td.text().trim();
|
||||
}
|
||||
|
||||
public static String parseTitle(Element td) {
|
||||
return td.getElementsByTag("a").text();
|
||||
}
|
||||
|
||||
public static String parseArticleUrl(Element td) {
|
||||
return PpomppuBoardName.ofViewPageUrl(td.getElementsByTag("a").attr("href"));
|
||||
}
|
||||
|
||||
public static String parseThumbnailUrl(Element td) {
|
||||
return "https:" + td.getElementsByTag("img").get(0).attr("src");
|
||||
}
|
||||
|
||||
public static Integer parseRecommended(Element td) {
|
||||
final String voteString = td.text();
|
||||
final int recommended;
|
||||
|
||||
if (voteString.isEmpty()) {
|
||||
recommended = 0;
|
||||
} else {
|
||||
final int voteUp = Integer.parseInt(td.text().split(" - ")[0]);
|
||||
final int voteDown = Integer.parseInt(td.text().split(" - ")[1]);
|
||||
recommended = voteUp - voteDown;
|
||||
}
|
||||
return recommended;
|
||||
}
|
||||
|
||||
public static String parseHit(Element td) {
|
||||
return td.text();
|
||||
}
|
||||
|
||||
public static String parseRegisteredAt(Element td) {
|
||||
return td.attr("title");
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
package com.myoa.engineering.crawl.ppomppu.processor.dto;
|
||||
|
||||
import java.time.Instant;
|
||||
import java.util.List;
|
||||
import java.util.function.Function;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.dto.BlockMessageDTO;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.dto.SimpleMessageDTO;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.util.DateUtil;
|
||||
|
||||
/**
|
||||
* PpomppuArticleTransformer
|
||||
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
|
||||
* @since 2021-11-21
|
||||
*
|
||||
*/
|
||||
public final class PpomppuArticleTransformer {
|
||||
|
||||
private PpomppuArticleTransformer() {}
|
||||
|
||||
private static final String MESSAGE_FORMAT_V1 = "%s)) <%s|LINK> `%s` ";
|
||||
private static final String MESSAGE_FORMAT_V2 = "%s *<%s|LINK>*\n%s";
|
||||
private static final String TITLE_FORMAT_V1 = "_*:hearts: %s | %s*_";
|
||||
|
||||
public static final Function<PpomppuArticle, SimpleMessageDTO> TRANSFORM_TO_MESSAGE_DTO = article ->
|
||||
SimpleMessageDTO.builder()
|
||||
.requestedAt(Instant.now())
|
||||
.publishedAt(article.getRegisteredAt())
|
||||
.title(String.format(MESSAGE_FORMAT_V1,
|
||||
article.getBoardName().getMenuName(), article.getArticleUrl(),
|
||||
article.getTitle()))
|
||||
.body(article.getArticleUrl())
|
||||
.build();
|
||||
|
||||
// https://stackoverflow.com/questions/24882927/using-streams-to-convert-a-list-of-objects-into-a-string-obtained-from-the-tostr
|
||||
public static SimpleMessageDTO transformToSimpleMessage(List<PpomppuArticle> articles) {
|
||||
Instant requestedAt = Instant.now();
|
||||
String body = articles.stream()
|
||||
.map(PpomppuArticleTransformer::convertToInlineMessage)
|
||||
.collect(Collectors.joining("\n\n"));
|
||||
return SimpleMessageDTO.builder()
|
||||
.requestedAt(requestedAt)
|
||||
.title(DateUtil.DATE_TIME_FORMATTER.format(requestedAt))
|
||||
.body(body)
|
||||
.build();
|
||||
}
|
||||
|
||||
public static BlockMessageDTO transformToBlockMessage(PpomppuBoardName boardName, List<PpomppuArticle> articles) {
|
||||
Instant requestedAt = Instant.now();
|
||||
List<BlockMessageDTO.Block> body = articles.stream()
|
||||
.map(e -> BlockMessageDTO.createBlock(convertToInlineMessage(e),
|
||||
e.getThumbnailUrl()))
|
||||
.collect(Collectors.toList());
|
||||
return BlockMessageDTO.builder()
|
||||
.requestedAt(requestedAt)
|
||||
.title(String.format(TITLE_FORMAT_V1,
|
||||
boardName.getMenuName(),
|
||||
DateUtil.DATE_TIME_FORMATTER.format(requestedAt)))
|
||||
.blocks(body)
|
||||
.build();
|
||||
}
|
||||
|
||||
public static String convertToInlineMessage(PpomppuArticle article) {
|
||||
return String.format(MESSAGE_FORMAT_V2,
|
||||
article.getBoardName().getMenuName(), article.getArticleUrl(), article.getTitle());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
package com.myoa.engineering.crawl.ppomppu.processor.dto.constant;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Getter;
|
||||
|
||||
/**
|
||||
* WebClientPropertiesUnitName
|
||||
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
|
||||
* @since 2021-11-18
|
||||
*
|
||||
*/
|
||||
@Getter
|
||||
@AllArgsConstructor
|
||||
public enum WebClientPropertiesUnitName {
|
||||
PPOMPPU_NOTIFIER_SENDER_API("ppn-sender-api"),
|
||||
;
|
||||
|
||||
private String unitName;
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
package com.myoa.engineering.crawl.ppomppu.processor.infrastructure.client;
|
||||
|
||||
import org.springframework.core.ParameterizedTypeReference;
|
||||
import org.springframework.http.HttpHeaders;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.web.reactive.function.client.WebClient;
|
||||
import org.springframework.web.reactive.function.client.WebClientRequestException;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.processor.dto.constant.WebClientPropertiesUnitName;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.dto.BlockMessageDTO;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.dto.SimpleMessageDTO;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebClientFilterFactory;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebFluxExchangeStragiesFactory;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.webclient.properties.WebClientProperties;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.webclient.properties.WebClientProperties.WebClientPropertiesUnit;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import reactor.core.publisher.Mono;
|
||||
import reactor.core.scheduler.Schedulers;
|
||||
|
||||
/**
|
||||
* PpomppuNotifierSenderAPIClient
|
||||
*
|
||||
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
|
||||
* @since 2021-11-17
|
||||
*/
|
||||
@Slf4j
|
||||
@Component
|
||||
public class MessageSenderAPIClient {
|
||||
|
||||
private final WebClient webClient;
|
||||
|
||||
public MessageSenderAPIClient(WebClientProperties webClientProperties) {
|
||||
WebClientPropertiesUnit webClientPropertiesUnit =
|
||||
webClientProperties.find(WebClientPropertiesUnitName.PPOMPPU_NOTIFIER_SENDER_API.getUnitName());
|
||||
this.webClient = WebClient.builder()
|
||||
.baseUrl(webClientPropertiesUnit.getBaseUrl())
|
||||
.exchangeStrategies(WebFluxExchangeStragiesFactory.ofDefault())
|
||||
.defaultHeader(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_VALUE)
|
||||
// .filter(WebClientFilterFactory.logRequest())
|
||||
// .filter(WebClientFilterFactory.logResponse())
|
||||
.build();
|
||||
}
|
||||
|
||||
public Mono<String> sendSimpleMessageToSlack(SimpleMessageDTO dto) {
|
||||
return webClient.post()
|
||||
.uri("/api/v1/messages/sendSimpleMessage/messengers/slack")
|
||||
.bodyValue(dto)
|
||||
.exchangeToMono(e -> e.bodyToMono(new ParameterizedTypeReference<String>() {}))
|
||||
.publishOn(Schedulers.boundedElastic())
|
||||
.onErrorResume(WebClientRequestException.class, t -> {
|
||||
log.info("Exception occured, ignoring. : {}", t.getClass().getSimpleName());
|
||||
return Mono.empty();
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
public Mono<String> sendBlockMessageToSlack(BlockMessageDTO dto) {
|
||||
return webClient.post()
|
||||
.uri("/api/v1/messages/sendBlockMessage/messengers/slack")
|
||||
.bodyValue(dto)
|
||||
.exchangeToMono(e -> e.bodyToMono(new ParameterizedTypeReference<String>() {}))
|
||||
.publishOn(Schedulers.boundedElastic())
|
||||
.onErrorResume(WebClientRequestException.class, t -> {
|
||||
log.info("Exception occured, ignoring. : {}", t.getClass().getSimpleName());
|
||||
return Mono.empty();
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
package com.myoa.engineering.crawl.ppomppu.processor.infrastructure.client;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebClientFilterFactory;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebFluxExchangeStragiesFactory;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import org.springframework.core.ParameterizedTypeReference;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.web.reactive.function.client.WebClient;
|
||||
import org.springframework.web.reactive.function.client.WebClientRequestException;
|
||||
import reactor.core.publisher.Mono;
|
||||
import reactor.core.scheduler.Schedulers;
|
||||
|
||||
/**
|
||||
* PpomppuBoardFeedRetriever
|
||||
*
|
||||
* @author Shin Woo-jin (woozu.shin@kakaoent.com)
|
||||
* @since 2021-09-08
|
||||
*/
|
||||
@Slf4j
|
||||
@Component
|
||||
public class PpomppuBoardFeedRetriever {
|
||||
|
||||
private final WebClient webClient;
|
||||
|
||||
public PpomppuBoardFeedRetriever(WebClient.Builder webClientBuilder) {
|
||||
this.webClient = webClientBuilder.baseUrl(PpomppuBoardName.PPOMPPU_URL)
|
||||
.exchangeStrategies(WebFluxExchangeStragiesFactory.ofTextHtml())
|
||||
.filter(WebClientFilterFactory.logRequest())
|
||||
.filter(WebClientFilterFactory.logResponse())
|
||||
.build();
|
||||
}
|
||||
|
||||
public Mono<String> getHtml(String uri) {
|
||||
return webClient.get()
|
||||
.uri(uri)
|
||||
.exchangeToMono(e -> e.bodyToMono(String.class))
|
||||
.publishOn(Schedulers.boundedElastic())
|
||||
.onErrorResume(WebClientRequestException.class, t -> {
|
||||
log.info("Exception occured, ignoring. : {}", t.getClass().getSimpleName());
|
||||
return Mono.empty();
|
||||
});
|
||||
// .doOnNext(e -> log.info("[getHtml] {}", e));
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
package com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository;
|
||||
|
||||
public interface BaseScanRepository {
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
package com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
|
||||
import org.springframework.data.jpa.repository.JpaRepository;
|
||||
import org.springframework.stereotype.Repository;
|
||||
|
||||
@Repository
|
||||
public interface PpomppuArticleRepository extends JpaRepository<PpomppuArticle, Long> {
|
||||
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
package com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuBoardFeedStatus;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
|
||||
import java.util.Optional;
|
||||
import org.springframework.data.jpa.repository.JpaRepository;
|
||||
import org.springframework.stereotype.Repository;
|
||||
|
||||
@Repository
|
||||
public interface PpomppuBoardFeedStatusRepository extends JpaRepository<PpomppuBoardFeedStatus, Long> {
|
||||
|
||||
Optional<PpomppuBoardFeedStatus> findByBoardName(PpomppuBoardName boardName);
|
||||
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
package com.myoa.engineering.crawl.ppomppu.processor.service;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
|
||||
import com.myoa.engineering.crawl.ppomppu.processor.dto.PpomppuArticleTransformer;
|
||||
import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.client.MessageSenderAPIClient;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import reactor.core.publisher.Mono;
|
||||
|
||||
/**
|
||||
* MessageSenderService
|
||||
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
|
||||
* @since 2021-11-21
|
||||
*
|
||||
*/
|
||||
@Slf4j
|
||||
@Service
|
||||
public class MessageSenderService {
|
||||
|
||||
private final MessageSenderAPIClient messageSenderAPIClient;
|
||||
|
||||
public MessageSenderService(MessageSenderAPIClient messageSenderAPIClient) {
|
||||
this.messageSenderAPIClient = messageSenderAPIClient;
|
||||
}
|
||||
|
||||
public Mono<String> sendSimpleMessageToSlack(PpomppuArticle article) {
|
||||
return messageSenderAPIClient.sendSimpleMessageToSlack(PpomppuArticleTransformer.TRANSFORM_TO_MESSAGE_DTO.apply(article));
|
||||
}
|
||||
|
||||
public Mono<String> sendSimpleMessageToSlack(List<PpomppuArticle> articles) {
|
||||
return messageSenderAPIClient.sendSimpleMessageToSlack(PpomppuArticleTransformer.transformToSimpleMessage(articles));
|
||||
}
|
||||
|
||||
public Mono<String> sendBlockMessageToSlack(PpomppuBoardName boardName, List<PpomppuArticle> articles) {
|
||||
return messageSenderAPIClient.sendBlockMessageToSlack(
|
||||
PpomppuArticleTransformer.transformToBlockMessage(boardName, articles));
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
package com.myoa.engineering.crawl.ppomppu.processor.service;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
|
||||
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuBoardFeedStatus;
|
||||
import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository.PpomppuArticleRepository;
|
||||
import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository.PpomppuBoardFeedStatusRepository;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
public class PpomppuArticleService {
|
||||
|
||||
private final PpomppuArticleRepository ppomppuArticleRepository;
|
||||
|
||||
private final PpomppuBoardFeedStatusRepository ppomppuBoardFeedStatusRepository;
|
||||
|
||||
public PpomppuArticleService(PpomppuArticleRepository ppomppuArticleRepository,
|
||||
PpomppuBoardFeedStatusRepository ppomppuBoardFeedStatusRepository) {
|
||||
this.ppomppuArticleRepository = ppomppuArticleRepository;
|
||||
this.ppomppuBoardFeedStatusRepository = ppomppuBoardFeedStatusRepository;
|
||||
}
|
||||
|
||||
@Transactional(readOnly = true)
|
||||
public List<PpomppuArticle> filterOnlyNewArticles(PpomppuBoardName boardName, List<PpomppuArticle> articles) {
|
||||
Optional<PpomppuBoardFeedStatus> boardFeedStatus = ppomppuBoardFeedStatusRepository.findByBoardName(boardName);
|
||||
Long latestArticleId = boardFeedStatus.map(PpomppuBoardFeedStatus::getLatestParsedArticleId)
|
||||
.orElse(0L);
|
||||
|
||||
log.info("latestArticleId : {}", latestArticleId);
|
||||
return articles.stream()
|
||||
.filter(e -> e.getArticleId().compareTo(latestArticleId) > 0)
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public List<PpomppuArticle> save(PpomppuBoardName boardName, List<PpomppuArticle> articles) {
|
||||
Long latestArticleId = articles.stream()
|
||||
.map(PpomppuArticle::getArticleId)
|
||||
.max(Long::compareTo)
|
||||
.orElse(0L);
|
||||
|
||||
// save PpomppuBoardFeedStatus
|
||||
Optional<PpomppuBoardFeedStatus> boardFeedStatus = ppomppuBoardFeedStatusRepository.findByBoardName(boardName);
|
||||
log.info("[save] boardName: {}, isPresent?: {}, latestArticleId: {}",
|
||||
boardName, boardFeedStatus.isPresent(), latestArticleId);
|
||||
log.info("[save] articles count: {}, article ids: {}",
|
||||
articles.size(), articles.stream().map(PpomppuArticle::getArticleId).toArray());
|
||||
boardFeedStatus.ifPresentOrElse(e -> {
|
||||
if (latestArticleId.longValue() > 0L) {
|
||||
e.updateArticleId(latestArticleId);
|
||||
ppomppuBoardFeedStatusRepository.save(e);
|
||||
}
|
||||
},
|
||||
() -> ppomppuBoardFeedStatusRepository.save(PpomppuBoardFeedStatus.of(boardName,
|
||||
latestArticleId)));
|
||||
|
||||
// save real articles.
|
||||
return ppomppuArticleRepository.saveAll(articles);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
package com.myoa.engineering.crawl.ppomppu.processor.service;
|
||||
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
|
||||
import org.jsoup.Jsoup;
|
||||
import org.jsoup.nodes.Element;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
|
||||
import com.myoa.engineering.crawl.ppomppu.processor.dto.PpomppuArticleParser;
|
||||
import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.client.PpomppuBoardFeedRetriever;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import reactor.core.publisher.Flux;
|
||||
import reactor.core.publisher.Mono;
|
||||
|
||||
/**
|
||||
* PpomppuFeedService
|
||||
*
|
||||
* @author Shin Woo-jin (woozu.shin@kakaoent.com)
|
||||
* @since 2021-09-08
|
||||
*/
|
||||
@Slf4j
|
||||
@Component
|
||||
public class PpomppuFeedService {
|
||||
|
||||
private final PpomppuBoardFeedRetriever ppomppuBoardFeedRetriever;
|
||||
|
||||
public PpomppuFeedService(PpomppuBoardFeedRetriever ppomppuBoardFeedRetriever) {
|
||||
this.ppomppuBoardFeedRetriever = ppomppuBoardFeedRetriever;
|
||||
}
|
||||
|
||||
public Mono<List<PpomppuArticle>> getArticles(PpomppuBoardName boardName) {
|
||||
final Mono<String> html = ppomppuBoardFeedRetriever.getHtml(boardName.getResourcePath());
|
||||
final Mono<Element> tbody = extractTbodyFromHtml(html);
|
||||
// .doOnNext(e -> log.info("pre tbody - {}", e.html()));
|
||||
return extractArticlesFromTbody(tbody).map(this::convertFromElement)
|
||||
.onErrorContinue((t, e) -> log.error("Error occured : {}, value: {}",
|
||||
e, t.getLocalizedMessage()))
|
||||
.map(e -> e.updateBoardName(boardName))
|
||||
.sort(Comparator.comparing(PpomppuArticle::getArticleId))
|
||||
// .doOnNext(e -> log.info("parsed Result: {}", e))
|
||||
.collectList();
|
||||
}
|
||||
|
||||
private Mono<Element> extractTbodyFromHtml(Mono<String> html) {
|
||||
return html.map(Jsoup::parse)
|
||||
.mapNotNull(e -> e.getElementById("revolution_main_table"))
|
||||
.map(e -> e.getElementsByTag("tbody"))
|
||||
// .doOnNext(e -> log.info("tbody - {}", e.html()))
|
||||
.map(e -> e.stream()
|
||||
.findFirst()
|
||||
.orElseThrow(() -> new IndexOutOfBoundsException("no tbody")));
|
||||
}
|
||||
|
||||
private Flux<Element> extractArticlesFromTbody(Mono<Element> tbody) {
|
||||
return Flux.concat(tbody.flatMapMany(e -> Flux.fromArray(e.select("tr.list0").toArray(Element[]::new))),
|
||||
tbody.flatMapMany(e -> Flux.fromArray(e.select("tr.list1").toArray(Element[]::new))));
|
||||
}
|
||||
|
||||
private PpomppuArticle convertFromElement(Element element) {
|
||||
return PpomppuArticleParser.toArticle(element.getElementsByTag("td"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
/*
|
||||
* Copyright (c) 2019 LINE Corporation. All rights reserved.
|
||||
* LINE Corporation PROPRIETARY/CONFIDENTIAL. Use is subject to license terms.
|
||||
*/
|
||||
|
||||
package com.myoa.engineering.crawl.ppomppu.processor.util;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.Collection;
|
||||
|
||||
/**
|
||||
* NumberUtils
|
||||
*
|
||||
* @author Shin Woo-jin (lp12254@linecorp.com)
|
||||
* @since 2019-10-28
|
||||
*/
|
||||
public final class ObjectUtil {
|
||||
|
||||
private ObjectUtil() {
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if given object is null.
|
||||
* <code>
|
||||
* e == object == > false e == null == > true
|
||||
* </code>
|
||||
*
|
||||
* @param e Target object
|
||||
* @param <E> Unfixed specific type. If you want restrict specific interface, Copy and extend qualifier.
|
||||
* @return Is null given object?
|
||||
*/
|
||||
public static <E> boolean isNullObject(final E e) {
|
||||
return e == null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if given object is not null.
|
||||
* <code>
|
||||
* e == object == > false e == null == > true
|
||||
* </code>
|
||||
*
|
||||
* @param e Target object
|
||||
* @param <E> Unfixed specific type. If you want restrict specific interface, Copy and extend qualifier.
|
||||
* @return Is not null given object?
|
||||
*/
|
||||
public static <E> boolean isNotEmpty(final E e) {
|
||||
return !isNullObject(e);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if there are any null object in given objects.
|
||||
* <code>
|
||||
* args == object = > false args == object, object = > false args == null, null, object = > true args
|
||||
* == null = > true args == null, null = > true
|
||||
* </code>
|
||||
*
|
||||
* @param args Want to check objects that have null.
|
||||
* @return Is there objects array has null?
|
||||
*/
|
||||
public static boolean hasNullObject(Object... args) {
|
||||
return Arrays.stream(args).anyMatch(ObjectUtil::isNullObject);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check given objects are not empty.
|
||||
* <code>
|
||||
* args == object = > true args == object, object = > true args == null, null, object = > false args
|
||||
* == null = > false args == null, null = > false
|
||||
* </code>
|
||||
*
|
||||
* @param args Want to check objects that have null.
|
||||
* @return Is there objects array has null?
|
||||
*/
|
||||
public static boolean hasAllObject(Object... args) {
|
||||
return Arrays.stream(args).noneMatch(ObjectUtil::isNullObject);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if there are all null object in given objects.
|
||||
* <code>
|
||||
* args == object = > false args == object, object = > false args == null, null, object = > false args
|
||||
* == null = > true args == null, null = > true
|
||||
* </code>
|
||||
*
|
||||
* @param args Want to check objects that have null.
|
||||
* @return Is there null all of given objects?
|
||||
*/
|
||||
public static boolean hasAllNullObjects(final Object... args) {
|
||||
return Arrays.stream(args).allMatch(ObjectUtil::isNullObject);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if given collection object is null or empty collecton.
|
||||
* <code>
|
||||
* e == null = > true e == emptyCollection = > true e == hasElement = > false
|
||||
* </code>
|
||||
*
|
||||
* @param e e is must be Collection object
|
||||
* @param <E> E is must be extended Collection Class
|
||||
* @return boolean. given collection is null or empty?
|
||||
*/
|
||||
public static <E extends Collection<?>> boolean isNullOrEmptyCollection(final E e) {
|
||||
return e == null || e.isEmpty();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get collection's size. Even it pointed null
|
||||
*
|
||||
* @param e e is must be Collection object
|
||||
* @param <E> E is must be extended Collection Class
|
||||
* @return integer value. given collection's size.
|
||||
*/
|
||||
public static <E extends Collection<?>> int getCollectionSize(final E e) {
|
||||
if (isNullOrEmptyCollection(e)) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
return e.size();
|
||||
}
|
||||
|
||||
}
|
||||
12
processor/src/main/resources/application-development.yml
Normal file
12
processor/src/main/resources/application-development.yml
Normal file
@@ -0,0 +1,12 @@
|
||||
spring:
|
||||
config:
|
||||
activate:
|
||||
on-profile: development
|
||||
import:
|
||||
- "configserver:http://192.168.0.100:11080"
|
||||
|
||||
|
||||
server:
|
||||
port: 20081
|
||||
|
||||
# import: optional:configserver:http://localhost:11080 # can be start up even config server was not found.
|
||||
11
processor/src/main/resources/application-local.yml
Normal file
11
processor/src/main/resources/application-local.yml
Normal file
@@ -0,0 +1,11 @@
|
||||
spring:
|
||||
config:
|
||||
activate:
|
||||
on-profile: local
|
||||
import:
|
||||
- "configserver:http://localhost:20085"
|
||||
|
||||
server:
|
||||
port: 20081
|
||||
|
||||
# import: optional:configserver:http://localhost:11080 # can be start up even config server was not found.
|
||||
6
processor/src/main/resources/application-production.yml
Normal file
6
processor/src/main/resources/application-production.yml
Normal file
@@ -0,0 +1,6 @@
|
||||
spring:
|
||||
config:
|
||||
activate:
|
||||
on-profile: production
|
||||
import:
|
||||
- "configserver:http://ppn-config-server:20080"
|
||||
@@ -0,0 +1,25 @@
|
||||
spring:
|
||||
application:
|
||||
name: ppn-processor
|
||||
main:
|
||||
allow-bean-definition-overriding: true
|
||||
profiles:
|
||||
active: ${SPRING_ACTIVE_PROFILE:local}
|
||||
group:
|
||||
local: "local,datasource-local,webclient-local"
|
||||
development: "development,datasource-development,webclient-development"
|
||||
production: "production,datasource-production,webclient-production"
|
||||
freemarker:
|
||||
enabled: false
|
||||
|
||||
server:
|
||||
port: 20080
|
||||
error:
|
||||
whitelabel:
|
||||
enabled: false
|
||||
|
||||
management:
|
||||
endpoints:
|
||||
web:
|
||||
exposure:
|
||||
include: refresh,health
|
||||
|
||||
15
processor/src/main/resources/logback-spring.xml
Normal file
15
processor/src/main/resources/logback-spring.xml
Normal file
@@ -0,0 +1,15 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<configuration>
|
||||
<springProperty name="DEFAULT_LEVEL_CONFIG" source="log.defaultLevel" />
|
||||
<springProfile name="local">
|
||||
<include resource="logback/logback-development.xml" />
|
||||
<logger name="org.apache.kafka" level="INFO" />
|
||||
</springProfile>
|
||||
<springProfile name="development">
|
||||
<include resource="logback/logback-development.xml" />
|
||||
<logger name="org.apache.kafka" level="INFO" />
|
||||
</springProfile>
|
||||
<springProfile name="production">
|
||||
<include resource="logback/logback-production.xml" />
|
||||
</springProfile>
|
||||
</configuration>
|
||||
@@ -0,0 +1,23 @@
|
||||
<included>
|
||||
<property name="FILE_LOG_PATTERN"
|
||||
value="%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{45}:%L - %msg%n" />
|
||||
<property name="LOG_FILE_BASE" value="lcp-benefit-benefit-api" />
|
||||
<appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
|
||||
<file>${DIRECTORY}/${LOG_FILE_BASE}_log</file>
|
||||
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
|
||||
<fileNamePattern>${DIRECTORY}/${LOG_FILE_BASE}_log.%d{yyyyMMdd}.%i</fileNamePattern>
|
||||
<timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
|
||||
<maxFileSize>1000MB</maxFileSize>
|
||||
</timeBasedFileNamingAndTriggeringPolicy>
|
||||
<maxHistory>60</maxHistory>
|
||||
</rollingPolicy>
|
||||
<encoder>
|
||||
<pattern>${FILE_LOG_PATTERN}</pattern>
|
||||
<immediateFlush>${IMMEDIATE_FLUSH}</immediateFlush>
|
||||
</encoder>
|
||||
</appender>
|
||||
<appender name="ASYNC_FILE" class="ch.qos.logback.classic.AsyncAppender">
|
||||
<queueSize>1024</queueSize>
|
||||
<appender-ref ref="FILE" />
|
||||
</appender>
|
||||
</included>
|
||||
19
processor/src/main/resources/logback/logback-development.xml
Normal file
19
processor/src/main/resources/logback/logback-development.xml
Normal file
@@ -0,0 +1,19 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<included>
|
||||
<!-- =========== property BETA ========= -->
|
||||
<property name="DEFAULT_LEVEL" value="${DEFAULT_LEVEL_CONFIG:-INFO}"/>
|
||||
<!--file-->
|
||||
<property name="DIRECTORY" value="/home1/www/logs/supervisor"/>
|
||||
<property name="IMMEDIATE_FLUSH" value="true"/>
|
||||
<!--nelo2-->
|
||||
<property name="NELO2_LEVEL" value="WARN"/>
|
||||
<!-- =========== include appender =========== -->
|
||||
<include resource="org/springframework/boot/logging/logback/defaults.xml"/>
|
||||
<include resource="org/springframework/boot/logging/logback/console-appender.xml"/>
|
||||
<include resource="logback/component/logback-nelo2.xml"/>
|
||||
<include resource="logback/component/logback-datachain.xml"/>
|
||||
<!-- =========== root logger ============== -->
|
||||
<root level="${DEFAULT_LEVEL}">
|
||||
<appender-ref ref="CONSOLE"/>
|
||||
</root>
|
||||
</included>
|
||||
19
processor/src/main/resources/logback/logback-production.xml
Normal file
19
processor/src/main/resources/logback/logback-production.xml
Normal file
@@ -0,0 +1,19 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<included>
|
||||
<!-- =========== property RELEASE ========= -->
|
||||
<property name="DEFAULT_LEVEL" value="${DEFAULT_LEVEL_CONFIG:-INFO}"/>
|
||||
<!--file-->
|
||||
<property name="DIRECTORY" value="/home1/www/logs/supervisor"/>
|
||||
<property name="IMMEDIATE_FLUSH" value="true"/>
|
||||
<!--nelo2-->
|
||||
<property name="NELO2_LEVEL" value="WARN"/>
|
||||
<!-- =========== include appender =========== -->
|
||||
<include resource="org/springframework/boot/logging/logback/defaults.xml"/>
|
||||
<include resource="org/springframework/boot/logging/logback/console-appender.xml"/>
|
||||
<include resource="logback/component/logback-nelo2.xml"/>
|
||||
<include resource="logback/component/logback-datachain.xml"/>
|
||||
<!-- =========== root logger ============== -->
|
||||
<root level="${DEFAULT_LEVEL}">
|
||||
<appender-ref ref="CONSOLE"/>
|
||||
</root>
|
||||
</included>
|
||||
12
processor/src/test/resources/logback-development.xml
Normal file
12
processor/src/test/resources/logback-development.xml
Normal file
@@ -0,0 +1,12 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<included>
|
||||
<!-- =========== property BETA ========= -->
|
||||
<property name="DEFAULT_LEVEL" value="${DEFAULT_LEVEL_CONFIG:-INFO}"/>
|
||||
<!-- =========== include appender =========== -->
|
||||
<include resource="org/springframework/boot/logging/logback/defaults.xml"/>
|
||||
<include resource="org/springframework/boot/logging/logback/console-appender.xml"/>
|
||||
<!-- =========== root logger ============== -->
|
||||
<root level="${DEFAULT_LEVEL}">
|
||||
<appender-ref ref="CONSOLE"/>
|
||||
</root>
|
||||
</included>
|
||||
6
processor/src/test/resources/logback-spring.xml
Normal file
6
processor/src/test/resources/logback-spring.xml
Normal file
@@ -0,0 +1,6 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<configuratiown>
|
||||
<springProperty name="DEFAULT_LEVEL_CONFIG" source="log.defaultLevel"/>
|
||||
|
||||
<include resource="logback-development.xml"/>
|
||||
</configuratiown>
|
||||
@@ -7,6 +7,9 @@ dependencies {
|
||||
implementation project(':support')
|
||||
// https://projectreactor.io/docs/core/release/reference/#debug-activate
|
||||
implementation 'org.springframework.boot:spring-boot-starter-webflux'
|
||||
implementation 'org.springframework.boot:spring-boot-configuration-processor'
|
||||
implementation 'org.springframework.cloud:spring-cloud-starter-config'
|
||||
implementation 'org.springframework.boot:spring-boot-starter-actuator'
|
||||
implementation 'org.telegram:telegrambots:5.3.0'
|
||||
|
||||
annotationProcessor 'org.springframework.boot:spring-boot-configuration-processor'
|
||||
|
||||
@@ -2,6 +2,11 @@ package com.myoa.engineering.crawl.ppomppu.receiver;
|
||||
|
||||
import org.springframework.boot.SpringApplication;
|
||||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||
import org.springframework.boot.context.properties.EnableConfigurationProperties;
|
||||
import org.springframework.context.annotation.Import;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.receiver.configuration.properties.TelegramBotProperties;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.webclient.PpomppuNotifierWebClientConfiguration;
|
||||
|
||||
/**
|
||||
* ReceiverApplication
|
||||
@@ -9,7 +14,9 @@ import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||
* @since 2021-08-20
|
||||
*
|
||||
*/
|
||||
@Import({ PpomppuNotifierWebClientConfiguration.class})
|
||||
@SpringBootApplication
|
||||
// @EnableConfigurationProperties({ TelegramBotProperties.class })
|
||||
public class ReceiverApplication {
|
||||
|
||||
public static void main(String[] args) {
|
||||
|
||||
@@ -8,8 +8,10 @@ import org.telegram.telegrambots.meta.TelegramBotsApi;
|
||||
import org.telegram.telegrambots.meta.exceptions.TelegramApiException;
|
||||
import org.telegram.telegrambots.updatesreceivers.DefaultBotSession;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.receiver.handler.message.MessageHandler;
|
||||
import com.myoa.engineering.crawl.ppomppu.receiver.configuration.properties.TelegramBotProperties;
|
||||
import com.myoa.engineering.crawl.ppomppu.receiver.configuration.properties.TelegramBotProperties.TelegramBotPropertiesUnit;
|
||||
import com.myoa.engineering.crawl.ppomppu.receiver.dispatch.MessageDispatcher;
|
||||
import com.myoa.engineering.crawl.ppomppu.receiver.handler.message.MessageHandler;
|
||||
|
||||
/**
|
||||
* TelegramBotConfiguration
|
||||
@@ -20,9 +22,7 @@ import com.myoa.engineering.crawl.ppomppu.receiver.dispatch.MessageDispatcher;
|
||||
@Configuration
|
||||
public class TelegramBotConfiguration {
|
||||
|
||||
private static final String BOT_TOKEN = ""; // TODO extract to property
|
||||
private static final String BOT_NAME = ""; // TODO extract to property
|
||||
|
||||
private static final String BOT_PROPERTIES_UNIT_NAME = "ppomppu_notify_bot";
|
||||
@Bean
|
||||
public TelegramBotsApi telegramBotsApi(MessageDispatcher messageDispatcher) throws TelegramApiException {
|
||||
TelegramBotsApi api = new TelegramBotsApi(DefaultBotSession.class);
|
||||
@@ -31,7 +31,9 @@ public class TelegramBotConfiguration {
|
||||
}
|
||||
|
||||
@Bean
|
||||
public MessageDispatcher messageDispatcher(List<MessageHandler> messageHandlers) {
|
||||
return new MessageDispatcher(messageHandlers, BOT_TOKEN, BOT_NAME);
|
||||
public MessageDispatcher messageDispatcher(List<MessageHandler> messageHandlers,
|
||||
TelegramBotProperties telegramBotProperties) {
|
||||
TelegramBotPropertiesUnit propertiesUnit = telegramBotProperties.find(BOT_PROPERTIES_UNIT_NAME);
|
||||
return new MessageDispatcher(messageHandlers, propertiesUnit.getName(), propertiesUnit.getToken());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
package com.myoa.engineering.crawl.ppomppu.receiver.configuration.properties;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
import org.springframework.boot.context.properties.ConstructorBinding;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.Setter;
|
||||
|
||||
/**
|
||||
* TelegramBotProperties
|
||||
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
|
||||
* @since 2021-09-05
|
||||
*
|
||||
*/
|
||||
@Component
|
||||
@Setter
|
||||
@Getter
|
||||
@ConfigurationProperties(prefix = "infra.telegram.bot")
|
||||
public class TelegramBotProperties {
|
||||
|
||||
private List<TelegramBotPropertiesUnit> units = new ArrayList<>();
|
||||
|
||||
@Data
|
||||
public static class TelegramBotPropertiesUnit {
|
||||
private String unitName;
|
||||
private String name;
|
||||
private String token;
|
||||
}
|
||||
|
||||
public TelegramBotPropertiesUnit find(String unitName) {
|
||||
return units.stream()
|
||||
.filter(e -> e.getUnitName().equals(unitName))
|
||||
.findFirst()
|
||||
.orElseThrow(
|
||||
() -> new IllegalArgumentException(this.getClass().getName() + ": unitName Not found. " + unitName));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
package com.myoa.engineering.crawl.ppomppu.receiver.controller.v1;
|
||||
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.receiver.service.ProcessorAPIService;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
|
||||
|
||||
import reactor.core.publisher.Mono;
|
||||
|
||||
/**
|
||||
* EventAPIController
|
||||
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
|
||||
* @since 2021-09-05
|
||||
*
|
||||
*/
|
||||
@RestController
|
||||
@RequestMapping("/api/v1/")
|
||||
public class EventAPIController {
|
||||
|
||||
private final ProcessorAPIService processorAPIService;
|
||||
|
||||
public EventAPIController(ProcessorAPIService processorAPIService) {
|
||||
this.processorAPIService = processorAPIService;
|
||||
}
|
||||
|
||||
@PostMapping("/exploit/parsePpomppuRSS/{boardName}")
|
||||
public Mono<String> parsePpomppuRSS(@PathVariable("boardName") PpomppuBoardName boardName) {
|
||||
return processorAPIService.emitParseEvent(boardName);
|
||||
}
|
||||
}
|
||||
@@ -14,13 +14,13 @@ import lombok.extern.slf4j.Slf4j;
|
||||
public class MessageDispatcher extends TelegramLongPollingBot {
|
||||
|
||||
private final List<MessageHandler> messageHandlers;
|
||||
private final String botToken;
|
||||
private final String botName;
|
||||
private final String botToken;
|
||||
|
||||
public MessageDispatcher(List<MessageHandler> messageHandlers, String botToken, String botName) {
|
||||
public MessageDispatcher(List<MessageHandler> messageHandlers, String botName, String botToken) {
|
||||
this.messageHandlers = messageHandlers;
|
||||
this.botToken = botToken;
|
||||
this.botName = botName;
|
||||
this.botToken = botToken;
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -33,7 +33,6 @@ public class MessageDispatcher extends TelegramLongPollingBot {
|
||||
|
||||
Message message = update.getMessage();
|
||||
MessageHandler handler = getMessageHandler(message);
|
||||
log.info(message.getText());
|
||||
handler.handle(message);
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
package com.myoa.engineering.crawl.ppomppu.receiver.dto;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
|
||||
import java.io.Serializable;
|
||||
import java.time.Instant;
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
/**
|
||||
* FeedParsedResult
|
||||
*
|
||||
* @author Shin Woo-jin (woozu.shin@kakaoent.com)
|
||||
* @since 2021-09-08
|
||||
*/
|
||||
@Getter
|
||||
@NoArgsConstructor
|
||||
public class FeedParsedResult implements Serializable {
|
||||
|
||||
private static final long serialVersionUID = -3771310078623481348L;
|
||||
|
||||
private PpomppuBoardName boardName;
|
||||
private Instant requestedAt;
|
||||
private Instant processedAt;
|
||||
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
package com.myoa.engineering.crawl.ppomppu.receiver.dto.constant;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Getter;
|
||||
|
||||
/**
|
||||
* WebClientPropertiesUnitName
|
||||
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
|
||||
* @since 2021-11-18
|
||||
*
|
||||
*/
|
||||
@Getter
|
||||
@AllArgsConstructor
|
||||
public enum WebClientPropertiesUnitName {
|
||||
PPOMPPU_NOTIFIER_PROCESSOR_API("ppn-processor-api"),
|
||||
;
|
||||
|
||||
private String unitName;
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
package com.myoa.engineering.crawl.ppomppu.receiver.handler.message;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.support.util.ObjectUtil;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.telegram.telegrambots.meta.api.objects.Message;
|
||||
|
||||
@Slf4j
|
||||
@Component
|
||||
public class HelloWorldMessageHandler implements MessageHandler {
|
||||
|
||||
@Override
|
||||
public boolean isApplicable(Message message) {
|
||||
return ObjectUtil.isEmpty(message);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void handle(Message message) {
|
||||
// skip empty event message.
|
||||
}
|
||||
}
|
||||
@@ -1,17 +1,18 @@
|
||||
package com.myoa.engineering.crawl.ppomppu.receiver.handler.message;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.support.util.ObjectUtil;
|
||||
import org.telegram.telegrambots.meta.api.objects.Message;
|
||||
|
||||
/**
|
||||
* TextMessageHandler
|
||||
*
|
||||
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
|
||||
* @since 2021-08-21
|
||||
*
|
||||
*/
|
||||
public interface TextMessageHandler extends MessageHandler {
|
||||
|
||||
@Override
|
||||
default boolean isApplicable(Message message) {
|
||||
return message.isUserMessage() && message.hasText();
|
||||
return ObjectUtil.isNotEmpty(message) && message.isUserMessage() && message.hasText();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -25,7 +25,6 @@ public class CommonTextHandler implements TextMessageHandler {
|
||||
@Override
|
||||
public void handle(Message message) {
|
||||
log.info("CommonTextHandler : {}", message.getText());
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -1,22 +1,44 @@
|
||||
package com.myoa.engineering.crawl.ppomppu.receiver.infrastructure.client;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.core.ParameterizedTypeReference;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.web.reactive.function.client.WebClient;
|
||||
import org.springframework.web.reactive.function.client.WebClientRequestException;
|
||||
|
||||
import java.util.List;
|
||||
import com.myoa.engineering.crawl.ppomppu.receiver.dto.constant.WebClientPropertiesUnitName;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebClientFilterFactory;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.webclient.properties.WebClientProperties;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.webclient.properties.WebClientProperties.WebClientPropertiesUnit;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import reactor.core.publisher.Mono;
|
||||
import reactor.core.scheduler.Schedulers;
|
||||
|
||||
@Slf4j
|
||||
@Component
|
||||
public class ProcessorAPIWebClient {
|
||||
|
||||
|
||||
@Value("${webclient.some}")
|
||||
private String baseUrl;
|
||||
|
||||
private final WebClient webClient;
|
||||
|
||||
public ProcessorAPIWebClient(WebClient.Builder webClientBuilder) {
|
||||
this.webClient = webClientBuilder.baseUrl("soundhoundfound-processor:20080")
|
||||
.build();
|
||||
public ProcessorAPIWebClient(WebClientProperties webClientProperties) {
|
||||
WebClientPropertiesUnit webClientPropertiesUnit =
|
||||
webClientProperties.find(WebClientPropertiesUnitName.PPOMPPU_NOTIFIER_PROCESSOR_API.getUnitName());
|
||||
this.webClient = WebClient.builder()
|
||||
.baseUrl(webClientPropertiesUnit.getBaseUrl())
|
||||
.filter(WebClientFilterFactory.logRequest())
|
||||
.filter(WebClientFilterFactory.logResponse())
|
||||
.build();
|
||||
}
|
||||
|
||||
public Mono<String> emitParseEvent(PpomppuBoardName boardName) {
|
||||
return webClient.post()
|
||||
.uri("/api/v1/crawl/boards/{boardName}", boardName)
|
||||
.exchangeToMono(e -> e.bodyToMono(new ParameterizedTypeReference<String>() {}))
|
||||
.publishOn(Schedulers.boundedElastic())
|
||||
.onErrorResume(WebClientRequestException.class, t -> {
|
||||
log.info("Exception occured, ignoring. : {}", t.getClass().getSimpleName());
|
||||
return Mono.empty();
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
package com.myoa.engineering.crawl.ppomppu.receiver.scheduler;
|
||||
|
||||
import java.util.Arrays;
|
||||
|
||||
import org.springframework.scheduling.annotation.EnableScheduling;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.receiver.service.ProcessorAPIService;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
/**
|
||||
* ParseEventEmitter
|
||||
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
|
||||
* @since 2021-09-05
|
||||
*
|
||||
*/
|
||||
@Slf4j
|
||||
@Component
|
||||
@EnableScheduling
|
||||
public class ParseEventEmitter {
|
||||
|
||||
private final ProcessorAPIService processorAPIService;
|
||||
|
||||
public ParseEventEmitter(ProcessorAPIService processorAPIService) {
|
||||
this.processorAPIService = processorAPIService;
|
||||
}
|
||||
|
||||
@Scheduled(fixedRate = 600 * 1000L)
|
||||
public void emitBoards() {
|
||||
log.info("[emitDomesticBoard] trigger fired!");
|
||||
Arrays.stream(PpomppuBoardName.values())
|
||||
.filter(PpomppuBoardName::isCrawlWithDefaultTimer)
|
||||
.forEach(boardName -> processorAPIService.emitParseEvent(boardName).block());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
package com.myoa.engineering.crawl.ppomppu.receiver.service;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.receiver.infrastructure.client.ProcessorAPIWebClient;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import reactor.core.publisher.Mono;
|
||||
|
||||
/**
|
||||
* ProcessorAPIService
|
||||
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
|
||||
* @since 2021-09-05
|
||||
*
|
||||
*/
|
||||
@Slf4j
|
||||
@Service
|
||||
public class ProcessorAPIService {
|
||||
|
||||
private final ProcessorAPIWebClient processorAPIWebClient;
|
||||
|
||||
public ProcessorAPIService(ProcessorAPIWebClient processorAPIWebClient) {
|
||||
this.processorAPIWebClient = processorAPIWebClient;
|
||||
}
|
||||
|
||||
public Mono<String> emitParseEvent(PpomppuBoardName boardName) {
|
||||
return processorAPIWebClient.emitParseEvent(boardName);
|
||||
}
|
||||
}
|
||||
@@ -3,4 +3,4 @@ spring:
|
||||
activate:
|
||||
on-profile: development
|
||||
import:
|
||||
- classpath:/development/webclient.yml
|
||||
- "configserver:http://192.168.0.100:11080"
|
||||
6
receiver/src/main/resources/application-local.yml
Normal file
6
receiver/src/main/resources/application-local.yml
Normal file
@@ -0,0 +1,6 @@
|
||||
spring:
|
||||
config:
|
||||
activate:
|
||||
on-profile: local
|
||||
import:
|
||||
- "configserver:http://localhost:20085"
|
||||
@@ -3,4 +3,4 @@ spring:
|
||||
activate:
|
||||
on-profile: production
|
||||
import:
|
||||
- classpath:/production/webclient.yml
|
||||
- "configserver:http://ppn-config-server:20080"
|
||||
@@ -1,10 +1,25 @@
|
||||
spring:
|
||||
application:
|
||||
name: ppn-receiver
|
||||
main:
|
||||
allow-bean-definition-overriding: true
|
||||
profiles:
|
||||
active: development
|
||||
active: ${SPRING_ACTIVE_PROFILE:local}
|
||||
group:
|
||||
local: "local,webclient-local"
|
||||
development: "development,webclient-development"
|
||||
production: "production,webclient-production"
|
||||
freemarker:
|
||||
enabled: false
|
||||
|
||||
webclient:
|
||||
some: not-test
|
||||
server:
|
||||
port: 20080
|
||||
error:
|
||||
whitelabel:
|
||||
enabled: false
|
||||
|
||||
management:
|
||||
endpoints:
|
||||
web:
|
||||
exposure:
|
||||
include: refresh,health
|
||||
|
||||
15
receiver/src/main/resources/logback-spring.xml
Normal file
15
receiver/src/main/resources/logback-spring.xml
Normal file
@@ -0,0 +1,15 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<configuration>
|
||||
<springProperty name="DEFAULT_LEVEL_CONFIG" source="log.defaultLevel" />
|
||||
<springProfile name="local">
|
||||
<include resource="logback/logback-development.xml" />
|
||||
<logger name="org.apache.kafka" level="INFO" />
|
||||
</springProfile>
|
||||
<springProfile name="development">
|
||||
<include resource="logback/logback-development.xml" />
|
||||
<logger name="org.apache.kafka" level="INFO" />
|
||||
</springProfile>
|
||||
<springProfile name="production">
|
||||
<include resource="logback/logback-production.xml" />
|
||||
</springProfile>
|
||||
</configuration>
|
||||
@@ -0,0 +1,23 @@
|
||||
<included>
|
||||
<property name="FILE_LOG_PATTERN"
|
||||
value="%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{45}:%L - %msg%n" />
|
||||
<property name="LOG_FILE_BASE" value="lcp-benefit-benefit-api" />
|
||||
<appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
|
||||
<file>${DIRECTORY}/${LOG_FILE_BASE}_log</file>
|
||||
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
|
||||
<fileNamePattern>${DIRECTORY}/${LOG_FILE_BASE}_log.%d{yyyyMMdd}.%i</fileNamePattern>
|
||||
<timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
|
||||
<maxFileSize>1000MB</maxFileSize>
|
||||
</timeBasedFileNamingAndTriggeringPolicy>
|
||||
<maxHistory>60</maxHistory>
|
||||
</rollingPolicy>
|
||||
<encoder>
|
||||
<pattern>${FILE_LOG_PATTERN}</pattern>
|
||||
<immediateFlush>${IMMEDIATE_FLUSH}</immediateFlush>
|
||||
</encoder>
|
||||
</appender>
|
||||
<appender name="ASYNC_FILE" class="ch.qos.logback.classic.AsyncAppender">
|
||||
<queueSize>1024</queueSize>
|
||||
<appender-ref ref="FILE" />
|
||||
</appender>
|
||||
</included>
|
||||
19
receiver/src/main/resources/logback/logback-development.xml
Normal file
19
receiver/src/main/resources/logback/logback-development.xml
Normal file
@@ -0,0 +1,19 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<included>
|
||||
<!-- =========== property BETA ========= -->
|
||||
<property name="DEFAULT_LEVEL" value="${DEFAULT_LEVEL_CONFIG:-INFO}"/>
|
||||
<!--file-->
|
||||
<property name="DIRECTORY" value="/home1/www/logs/supervisor"/>
|
||||
<property name="IMMEDIATE_FLUSH" value="true"/>
|
||||
<!--nelo2-->
|
||||
<property name="NELO2_LEVEL" value="WARN"/>
|
||||
<!-- =========== include appender =========== -->
|
||||
<include resource="org/springframework/boot/logging/logback/defaults.xml"/>
|
||||
<include resource="org/springframework/boot/logging/logback/console-appender.xml"/>
|
||||
<include resource="logback/component/logback-nelo2.xml"/>
|
||||
<include resource="logback/component/logback-datachain.xml"/>
|
||||
<!-- =========== root logger ============== -->
|
||||
<root level="${DEFAULT_LEVEL}">
|
||||
<appender-ref ref="CONSOLE"/>
|
||||
</root>
|
||||
</included>
|
||||
19
receiver/src/main/resources/logback/logback-production.xml
Normal file
19
receiver/src/main/resources/logback/logback-production.xml
Normal file
@@ -0,0 +1,19 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<included>
|
||||
<!-- =========== property RELEASE ========= -->
|
||||
<property name="DEFAULT_LEVEL" value="${DEFAULT_LEVEL_CONFIG:-INFO}"/>
|
||||
<!--file-->
|
||||
<property name="DIRECTORY" value="/home1/www/logs/supervisor"/>
|
||||
<property name="IMMEDIATE_FLUSH" value="true"/>
|
||||
<!--nelo2-->
|
||||
<property name="NELO2_LEVEL" value="WARN"/>
|
||||
<!-- =========== include appender =========== -->
|
||||
<include resource="org/springframework/boot/logging/logback/defaults.xml"/>
|
||||
<include resource="org/springframework/boot/logging/logback/console-appender.xml"/>
|
||||
<include resource="logback/component/logback-nelo2.xml"/>
|
||||
<include resource="logback/component/logback-datachain.xml"/>
|
||||
<!-- =========== root logger ============== -->
|
||||
<root level="${DEFAULT_LEVEL}">
|
||||
<appender-ref ref="CONSOLE"/>
|
||||
</root>
|
||||
</included>
|
||||
23
sender/build.gradle
Normal file
23
sender/build.gradle
Normal file
@@ -0,0 +1,23 @@
|
||||
dependencies {
|
||||
developmentOnly 'org.springframework.boot:spring-boot-devtools'
|
||||
runtimeOnly 'mysql:mysql-connector-java'
|
||||
compileOnly 'org.projectlombok:lombok'
|
||||
|
||||
implementation project(':support')
|
||||
// https://projectreactor.io/docs/core/release/reference/#debug-activate
|
||||
implementation 'org.springframework.boot:spring-boot-starter-webflux'
|
||||
implementation 'org.springframework.boot:spring-boot-starter-actuator'
|
||||
implementation 'org.springframework.cloud:spring-cloud-starter-config'
|
||||
|
||||
annotationProcessor 'org.springframework.boot:spring-boot-configuration-processor'
|
||||
annotationProcessor 'org.projectlombok:lombok'
|
||||
|
||||
testImplementation 'org.springframework.boot:spring-boot-starter-test'
|
||||
}
|
||||
|
||||
test {
|
||||
useJUnitPlatform()
|
||||
testLogging {
|
||||
events "passed", "skipped", "failed"
|
||||
}
|
||||
}
|
||||
@@ -1,10 +1,22 @@
|
||||
package com.myoa.engineering.crawl.ppomppu.sender;
|
||||
|
||||
import org.springframework.boot.SpringApplication;
|
||||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||
import org.springframework.context.annotation.Import;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.support.webclient.PpomppuNotifierWebClientConfiguration;
|
||||
|
||||
/**
|
||||
* SenderApplication
|
||||
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
|
||||
* @since 2021-08-20
|
||||
*
|
||||
*/
|
||||
@Import({ PpomppuNotifierWebClientConfiguration.class })
|
||||
@SpringBootApplication
|
||||
public class SenderApplication {
|
||||
|
||||
public static void main(String[] args) {
|
||||
SpringApplication.run(SenderApplication.class, args);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
package com.myoa.engineering.crawl.ppomppu.sender.configuration.properties;
|
||||
|
||||
import java.util.List;
|
||||
import lombok.Data;
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
@Getter
|
||||
@Setter
|
||||
@Component
|
||||
@ConfigurationProperties("infra.slack.bot")
|
||||
public class SlackSecretProperties {
|
||||
|
||||
private List<SlackSecretPropertiesUnit> units;
|
||||
|
||||
@Data
|
||||
public static class SlackSecretPropertiesUnit {
|
||||
|
||||
private String botName;
|
||||
private String username;
|
||||
private String iconEmoji;
|
||||
private String channel;
|
||||
private String token;
|
||||
}
|
||||
|
||||
public SlackSecretPropertiesUnit find(String botUnitName) {
|
||||
return units.stream()
|
||||
.filter(e -> e.getBotName().equals(botUnitName))
|
||||
.findFirst()
|
||||
.orElseThrow(() -> new IllegalArgumentException("not found bot unit name : " + botUnitName));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
package com.myoa.engineering.crawl.ppomppu.sender.controller;
|
||||
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.sender.dto.SlackBaseMessageBlock;
|
||||
import com.myoa.engineering.crawl.ppomppu.sender.dto.SlackMessageDTO;
|
||||
import com.myoa.engineering.crawl.ppomppu.sender.infrastructure.client.MongeShoppingBotSlackMessageSender;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.dto.APIResponse;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.dto.BlockMessageDTO;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.dto.SimpleMessageDTO;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.util.ObjectMapperFactory;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import reactor.core.publisher.Mono;
|
||||
|
||||
/**
|
||||
* MessageSenderAPIController
|
||||
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
|
||||
* @since 2021-11-21
|
||||
*
|
||||
*/
|
||||
@Slf4j
|
||||
@RestController
|
||||
@RequestMapping("/api/v1")
|
||||
public class MessageSenderAPIController {
|
||||
|
||||
private final MongeShoppingBotSlackMessageSender sender;
|
||||
|
||||
public MessageSenderAPIController(MongeShoppingBotSlackMessageSender sender) {
|
||||
this.sender = sender;
|
||||
}
|
||||
|
||||
@PostMapping("/messages/sendSimpleMessage/messengers/slack")
|
||||
public Mono<APIResponse<SimpleMessageDTO>> sendSimpleMessageToSlack(@RequestBody SimpleMessageDTO dto) {
|
||||
return sender.sendMessage(sender.ofMessage(dto.getBody()))
|
||||
.then(Mono.just(APIResponse.success(dto)));
|
||||
}
|
||||
|
||||
@PostMapping("/messages/sendBlockMessage/messengers/slack")
|
||||
public Mono<APIResponse<BlockMessageDTO>> sendBlockMessageToSlack(@RequestBody BlockMessageDTO dto) {
|
||||
if (dto.getBlocks().isEmpty()) {
|
||||
return Mono.just(APIResponse.fail(dto, "empty blocks"));
|
||||
}
|
||||
|
||||
|
||||
return sender.sendMessage(buildSlackMessageDTO(dto))
|
||||
// .doOnNext(e -> log.info("[sendBlockMessageToSlack] slackMessageDTO: {}",
|
||||
// ObjectMapperFactory.writeAsString(buildSlackMessageDTO(dto))))
|
||||
.then(Mono.just(APIResponse.success(dto)));
|
||||
}
|
||||
|
||||
private SlackMessageDTO buildSlackMessageDTO(BlockMessageDTO dto) {
|
||||
SlackMessageDTO slackMessageDTO = sender.ofBlockMessageBased();
|
||||
slackMessageDTO.addSectionBlock(dto.getTitle());
|
||||
dto.getBlocks().forEach(slackMessageDTO::addSectionBlock);
|
||||
slackMessageDTO.addBlock(SlackBaseMessageBlock.ofDivider());
|
||||
return slackMessageDTO;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
package com.myoa.engineering.crawl.ppomppu.sender.controller;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.sender.infrastructure.client.MongeShoppingBotSlackMessageSender;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import reactor.core.publisher.Mono;
|
||||
|
||||
/**
|
||||
* TestAPIController
|
||||
*
|
||||
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
|
||||
* @since 2021-11-15
|
||||
*/
|
||||
@Slf4j
|
||||
@RestController
|
||||
@RequestMapping("/api/v1")
|
||||
public class TestAPIController {
|
||||
|
||||
private final MongeShoppingBotSlackMessageSender sender;
|
||||
|
||||
public TestAPIController(MongeShoppingBotSlackMessageSender sender) {
|
||||
this.sender = sender;
|
||||
}
|
||||
|
||||
@GetMapping("/test")
|
||||
public Mono<String> test() {
|
||||
log.info("received!!!");
|
||||
return sender.sendMessage(sender.ofMessage("testtesttest!!!"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
package com.myoa.engineering.crawl.ppomppu.sender.dto;
|
||||
|
||||
import java.io.Serializable;
|
||||
|
||||
/**
|
||||
* MessageDTO
|
||||
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
|
||||
* @since 2021-11-14
|
||||
*
|
||||
*/
|
||||
public interface MessageDTO extends Serializable {
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
package com.myoa.engineering.crawl.ppomppu.sender.dto;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||
|
||||
import lombok.Builder;
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
/**
|
||||
* MessageBlock
|
||||
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
|
||||
* @since 2021-11-30
|
||||
*
|
||||
*/
|
||||
@Getter
|
||||
@NoArgsConstructor
|
||||
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||
public class SlackBaseMessageBlock implements SlackMessageBlock {
|
||||
private static final long serialVersionUID = 1597984001727808419L;
|
||||
|
||||
private SlackMessageBlockType type;
|
||||
private String text;
|
||||
|
||||
@Builder
|
||||
private SlackBaseMessageBlock(SlackMessageBlockType type, String text) {
|
||||
this.type = type;
|
||||
this.text = text;
|
||||
}
|
||||
|
||||
public static SlackBaseMessageBlock ofMarkDown(String message) {
|
||||
return SlackBaseMessageBlock.builder()
|
||||
.type(SlackMessageBlockType.MARKDOWN)
|
||||
.text(message)
|
||||
.build();
|
||||
}
|
||||
|
||||
public static SlackBaseMessageBlock ofDivider() {
|
||||
return SlackBaseMessageBlock.builder()
|
||||
.type(SlackMessageBlockType.DIVIDER)
|
||||
.build();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getType() {
|
||||
return type.getType();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
package com.myoa.engineering.crawl.ppomppu.sender.dto;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
|
||||
import lombok.Builder;
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
/**
|
||||
* SlackImageMessageBlock
|
||||
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
|
||||
* @since 2021-11-30
|
||||
*
|
||||
*/
|
||||
@Getter
|
||||
@NoArgsConstructor
|
||||
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||
public class SlackImageMessageBlock implements SlackMessageBlock {
|
||||
private static final long serialVersionUID = 1597984001727808419L;
|
||||
|
||||
private SlackMessageBlockType type;
|
||||
|
||||
@JsonProperty(value = "image_url", required = true)
|
||||
private String imageUrl;
|
||||
|
||||
@JsonProperty(value = "alt_text", required = true)
|
||||
private String altText;
|
||||
|
||||
@Builder
|
||||
private SlackImageMessageBlock(SlackMessageBlockType type, String imageUrl, String altText) {
|
||||
this.type = type;
|
||||
this.imageUrl = imageUrl;
|
||||
this.altText = altText;
|
||||
}
|
||||
|
||||
public static SlackImageMessageBlock of(String imageUrl, String altText) {
|
||||
return SlackImageMessageBlock.builder()
|
||||
.type(SlackMessageBlockType.IMAGE)
|
||||
.imageUrl(imageUrl)
|
||||
.altText(altText)
|
||||
.build();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getType() {
|
||||
return type.getType();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
package com.myoa.engineering.crawl.ppomppu.sender.dto;
|
||||
|
||||
import java.io.Serializable;
|
||||
|
||||
/**
|
||||
* SlackMessageBlock
|
||||
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
|
||||
* @since 2021-12-01
|
||||
*
|
||||
*/
|
||||
public interface SlackMessageBlock extends Serializable {
|
||||
|
||||
String getType();
|
||||
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
package com.myoa.engineering.crawl.ppomppu.sender.dto;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Getter;
|
||||
|
||||
/**
|
||||
* BlockType
|
||||
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
|
||||
* @since 2021-11-30
|
||||
*
|
||||
*/
|
||||
@Getter
|
||||
@AllArgsConstructor
|
||||
public enum SlackMessageBlockType {
|
||||
SECTION("section"),
|
||||
MARKDOWN("mrkdwn"),
|
||||
DIVIDER("divider"),
|
||||
IMAGE("image"),
|
||||
;
|
||||
|
||||
private String type;
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
package com.myoa.engineering.crawl.ppomppu.sender.dto;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.dto.BlockMessageDTO;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.dto.BlockMessageDTO.Block;
|
||||
|
||||
import lombok.Builder;
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
/**
|
||||
* SlackMessageDTO
|
||||
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
|
||||
* @since 2021-11-14
|
||||
*
|
||||
*/
|
||||
@Getter
|
||||
@NoArgsConstructor
|
||||
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||
public class SlackMessageDTO implements MessageDTO {
|
||||
|
||||
private final static long serialVersionUID = 4737608709660494713L;
|
||||
|
||||
private String text;
|
||||
private String channel;
|
||||
private String username;
|
||||
private List<SlackMessageBlock> blocks;
|
||||
|
||||
@JsonProperty("icon_emoji")
|
||||
private String iconEmoji;
|
||||
|
||||
@Builder
|
||||
public SlackMessageDTO(String text, String channel, String username,
|
||||
List<SlackMessageBlock> blocks, String iconEmoji) {
|
||||
this.text = text;
|
||||
this.channel = channel;
|
||||
this.username = username;
|
||||
this.blocks = blocks;
|
||||
this.iconEmoji = iconEmoji;
|
||||
}
|
||||
|
||||
public void applyText(String text) {
|
||||
this.text = text;
|
||||
}
|
||||
|
||||
public void addSectionBlock(BlockMessageDTO.Block block) {
|
||||
SlackSectionMessageBlock slackSectionMessageBlock = SlackSectionMessageBlock.ofMarkDown(block.getText());
|
||||
slackSectionMessageBlock.applyImageaccessory(block.getImageUrl(), block.getAltText());
|
||||
addBlock(slackSectionMessageBlock);
|
||||
}
|
||||
public void addSectionBlock(String rawBlockMessage) {
|
||||
addBlock(SlackSectionMessageBlock.ofMarkDown(rawBlockMessage));
|
||||
}
|
||||
|
||||
public void addBlock(SlackMessageBlock block) {
|
||||
blocks.add(block);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
package com.myoa.engineering.crawl.ppomppu.sender.dto;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||
|
||||
import lombok.Builder;
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
/**
|
||||
* SectionBlock
|
||||
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
|
||||
* @since 2021-11-30
|
||||
*
|
||||
*/
|
||||
@Getter
|
||||
@NoArgsConstructor
|
||||
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||
public class SlackSectionMessageBlock implements SlackMessageBlock {
|
||||
private static final long serialVersionUID = -7600944576753160168L;
|
||||
|
||||
private SlackMessageBlockType type;
|
||||
private SlackBaseMessageBlock text;
|
||||
private SlackImageMessageBlock accessory;
|
||||
|
||||
@Builder
|
||||
private SlackSectionMessageBlock(SlackMessageBlockType type, SlackBaseMessageBlock text,
|
||||
SlackImageMessageBlock accessory) {
|
||||
this.type = type;
|
||||
this.text = text;
|
||||
this.accessory = accessory;
|
||||
}
|
||||
|
||||
public static SlackSectionMessageBlock ofMarkDown(String message) {
|
||||
return SlackSectionMessageBlock.builder()
|
||||
.type(SlackMessageBlockType.SECTION)
|
||||
.text(SlackBaseMessageBlock.ofMarkDown(message))
|
||||
.build();
|
||||
}
|
||||
|
||||
public SlackSectionMessageBlock applyImageaccessory(String imageUrl, String altText) {
|
||||
this.accessory = SlackImageMessageBlock.of(imageUrl, altText);
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getType() {
|
||||
return type.getType();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
package com.myoa.engineering.crawl.ppomppu.sender.infrastructure.client;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.sender.dto.MessageDTO;
|
||||
|
||||
import reactor.core.publisher.Mono;
|
||||
|
||||
/**
|
||||
* MessageSender
|
||||
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
|
||||
* @since 2021-11-14
|
||||
*
|
||||
*/
|
||||
public interface MessageSender<T extends MessageDTO> {
|
||||
|
||||
Mono<String> sendMessage(T message);
|
||||
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
package com.myoa.engineering.crawl.ppomppu.sender.infrastructure.client;
|
||||
|
||||
import java.util.ArrayList;
|
||||
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.sender.configuration.properties.SlackSecretProperties;
|
||||
import com.myoa.engineering.crawl.ppomppu.sender.configuration.properties.SlackSecretProperties.SlackSecretPropertiesUnit;
|
||||
import com.myoa.engineering.crawl.ppomppu.sender.dto.SlackMessageDTO;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
@Component
|
||||
public class MongeShoppingBotSlackMessageSender extends SlackMessageSender {
|
||||
|
||||
private static final String SLACK_SECRET_UNIT_NAME = "monge_shopping_bot";
|
||||
|
||||
private final SlackSecretPropertiesUnit slackProperties;
|
||||
|
||||
public MongeShoppingBotSlackMessageSender(SlackSecretProperties slackSecretProperties) {
|
||||
super(slackSecretProperties.find(SLACK_SECRET_UNIT_NAME).getToken());
|
||||
this.slackProperties = slackSecretProperties.find(SLACK_SECRET_UNIT_NAME);
|
||||
}
|
||||
|
||||
public SlackMessageDTO ofMessageTemplate() {
|
||||
return SlackMessageDTO.builder()
|
||||
.channel(slackProperties.getChannel())
|
||||
.iconEmoji(slackProperties.getIconEmoji())
|
||||
.username(slackProperties.getUsername())
|
||||
.build();
|
||||
}
|
||||
|
||||
public SlackMessageDTO ofMessage(String text) {
|
||||
return SlackMessageDTO.builder()
|
||||
.channel(slackProperties.getChannel())
|
||||
.iconEmoji(slackProperties.getIconEmoji())
|
||||
.username(slackProperties.getUsername())
|
||||
.text(text)
|
||||
.build();
|
||||
}
|
||||
|
||||
public SlackMessageDTO ofBlockMessageBased() {
|
||||
return SlackMessageDTO.builder()
|
||||
.channel(slackProperties.getChannel())
|
||||
.iconEmoji(slackProperties.getIconEmoji())
|
||||
.username(slackProperties.getUsername())
|
||||
.blocks(new ArrayList<>())
|
||||
.build();
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
package com.myoa.engineering.crawl.ppomppu.sender.infrastructure.client;
|
||||
|
||||
import org.springframework.http.HttpHeaders;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.web.reactive.function.client.WebClient;
|
||||
import org.springframework.web.reactive.function.client.WebClientRequestException;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.sender.dto.SlackMessageDTO;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebClientFilterFactory;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebFluxExchangeStragiesFactory;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import reactor.core.publisher.Mono;
|
||||
import reactor.core.scheduler.Schedulers;
|
||||
|
||||
/**
|
||||
* SlackMessageSender
|
||||
*
|
||||
* @author Shin Woo-jin (woozu.shin@kakaoent.com)
|
||||
* @since 2021-09-08
|
||||
*/
|
||||
@Slf4j
|
||||
public class SlackMessageSender implements MessageSender<SlackMessageDTO> {
|
||||
|
||||
private static final String SLACK_API_URL = "https://slack.com/api";
|
||||
|
||||
private final WebClient webClient;
|
||||
|
||||
public SlackMessageSender(String apiSecret) {
|
||||
this.webClient = WebClient.builder()
|
||||
.baseUrl(SLACK_API_URL)
|
||||
.defaultHeader(HttpHeaders.AUTHORIZATION, "Bearer " + apiSecret)
|
||||
.exchangeStrategies(WebFluxExchangeStragiesFactory.ofDefault())
|
||||
.defaultHeader(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_UTF8_VALUE)
|
||||
.defaultHeader(HttpHeaders.ACCEPT_CHARSET, "UTF-8")
|
||||
.filter(WebClientFilterFactory.logRequest())
|
||||
.filter(WebClientFilterFactory.logResponse())
|
||||
.build();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Mono<String> sendMessage(SlackMessageDTO message) {
|
||||
return webClient.post()
|
||||
.uri("/chat.postMessage")
|
||||
.bodyValue(message)
|
||||
.exchangeToMono(e -> e.bodyToMono(String.class))
|
||||
.publishOn(Schedulers.boundedElastic())
|
||||
.onErrorResume(WebClientRequestException.class, t -> {
|
||||
log.info("Exception occured, ignoring. : {}", t.getClass().getSimpleName());
|
||||
return Mono.empty();
|
||||
})
|
||||
.doOnNext(e -> log.info("[sendMessage] {}", e));
|
||||
}
|
||||
|
||||
}
|
||||
10
sender/src/main/resources/application-development.yml
Normal file
10
sender/src/main/resources/application-development.yml
Normal file
@@ -0,0 +1,10 @@
|
||||
spring:
|
||||
config:
|
||||
activate:
|
||||
on-profile: development
|
||||
import:
|
||||
- "configserver:http://192.168.0.100:11080"
|
||||
# import: optional:configserver:http://localhost:11080 # can be start up even config server was not found.
|
||||
|
||||
server:
|
||||
port: 20082
|
||||
10
sender/src/main/resources/application-local.yml
Normal file
10
sender/src/main/resources/application-local.yml
Normal file
@@ -0,0 +1,10 @@
|
||||
spring:
|
||||
config:
|
||||
activate:
|
||||
on-profile: local
|
||||
import:
|
||||
- "configserver:http://localhost:20085"
|
||||
# import: optional:configserver:http://localhost:11080 # can be start up even config server was not found.
|
||||
|
||||
server:
|
||||
port: 20082
|
||||
6
sender/src/main/resources/application-production.yml
Normal file
6
sender/src/main/resources/application-production.yml
Normal file
@@ -0,0 +1,6 @@
|
||||
spring:
|
||||
config:
|
||||
activate:
|
||||
on-profile: production
|
||||
import:
|
||||
- "configserver:http://ppn-config-server:20080"
|
||||
@@ -0,0 +1,25 @@
|
||||
spring:
|
||||
application:
|
||||
name: ppn-sender
|
||||
main:
|
||||
allow-bean-definition-overriding: true
|
||||
profiles:
|
||||
active: ${SPRING_ACTIVE_PROFILE:local}
|
||||
group:
|
||||
local: "local,slackapi-local,webclient-local"
|
||||
development: "development,slackapi-development,webclient-development"
|
||||
production: "production,slackapi-production,webclient-production"
|
||||
freemarker:
|
||||
enabled: false
|
||||
|
||||
server:
|
||||
port: 20080
|
||||
error:
|
||||
whitelabel:
|
||||
enabled: false
|
||||
|
||||
management:
|
||||
endpoints:
|
||||
web:
|
||||
exposure:
|
||||
include: refresh,health
|
||||
|
||||
15
sender/src/main/resources/logback-spring.xml
Normal file
15
sender/src/main/resources/logback-spring.xml
Normal file
@@ -0,0 +1,15 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<configuration>
|
||||
<springProfile name="local">
|
||||
<include resource="logback/logback-development.xml" />
|
||||
<logger name="org.apache.kafka" level="INFO" />
|
||||
</springProfile>
|
||||
<springProperty name="DEFAULT_LEVEL_CONFIG" source="log.defaultLevel" />
|
||||
<springProfile name="development">
|
||||
<include resource="logback/logback-development.xml" />
|
||||
<logger name="org.apache.kafka" level="INFO" />
|
||||
</springProfile>
|
||||
<springProfile name="production">
|
||||
<include resource="logback/logback-production.xml" />
|
||||
</springProfile>
|
||||
</configuration>
|
||||
23
sender/src/main/resources/logback/component/logback-file.xml
Normal file
23
sender/src/main/resources/logback/component/logback-file.xml
Normal file
@@ -0,0 +1,23 @@
|
||||
<included>
|
||||
<property name="FILE_LOG_PATTERN"
|
||||
value="%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{45}:%L - %msg%n" />
|
||||
<property name="LOG_FILE_BASE" value="lcp-benefit-benefit-api" />
|
||||
<appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
|
||||
<file>${DIRECTORY}/${LOG_FILE_BASE}_log</file>
|
||||
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
|
||||
<fileNamePattern>${DIRECTORY}/${LOG_FILE_BASE}_log.%d{yyyyMMdd}.%i</fileNamePattern>
|
||||
<timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
|
||||
<maxFileSize>1000MB</maxFileSize>
|
||||
</timeBasedFileNamingAndTriggeringPolicy>
|
||||
<maxHistory>60</maxHistory>
|
||||
</rollingPolicy>
|
||||
<encoder>
|
||||
<pattern>${FILE_LOG_PATTERN}</pattern>
|
||||
<immediateFlush>${IMMEDIATE_FLUSH}</immediateFlush>
|
||||
</encoder>
|
||||
</appender>
|
||||
<appender name="ASYNC_FILE" class="ch.qos.logback.classic.AsyncAppender">
|
||||
<queueSize>1024</queueSize>
|
||||
<appender-ref ref="FILE" />
|
||||
</appender>
|
||||
</included>
|
||||
19
sender/src/main/resources/logback/logback-development.xml
Normal file
19
sender/src/main/resources/logback/logback-development.xml
Normal file
@@ -0,0 +1,19 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<included>
|
||||
<!-- =========== property BETA ========= -->
|
||||
<property name="DEFAULT_LEVEL" value="${DEFAULT_LEVEL_CONFIG:-INFO}"/>
|
||||
<!--file-->
|
||||
<property name="DIRECTORY" value="/home1/www/logs/supervisor"/>
|
||||
<property name="IMMEDIATE_FLUSH" value="true"/>
|
||||
<!--nelo2-->
|
||||
<property name="NELO2_LEVEL" value="WARN"/>
|
||||
<!-- =========== include appender =========== -->
|
||||
<include resource="org/springframework/boot/logging/logback/defaults.xml"/>
|
||||
<include resource="org/springframework/boot/logging/logback/console-appender.xml"/>
|
||||
<include resource="logback/component/logback-nelo2.xml"/>
|
||||
<include resource="logback/component/logback-datachain.xml"/>
|
||||
<!-- =========== root logger ============== -->
|
||||
<root level="${DEFAULT_LEVEL}">
|
||||
<appender-ref ref="CONSOLE"/>
|
||||
</root>
|
||||
</included>
|
||||
19
sender/src/main/resources/logback/logback-production.xml
Normal file
19
sender/src/main/resources/logback/logback-production.xml
Normal file
@@ -0,0 +1,19 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<included>
|
||||
<!-- =========== property RELEASE ========= -->
|
||||
<property name="DEFAULT_LEVEL" value="${DEFAULT_LEVEL_CONFIG:-INFO}"/>
|
||||
<!--file-->
|
||||
<property name="DIRECTORY" value="/home1/www/logs/supervisor"/>
|
||||
<property name="IMMEDIATE_FLUSH" value="true"/>
|
||||
<!--nelo2-->
|
||||
<property name="NELO2_LEVEL" value="WARN"/>
|
||||
<!-- =========== include appender =========== -->
|
||||
<include resource="org/springframework/boot/logging/logback/defaults.xml"/>
|
||||
<include resource="org/springframework/boot/logging/logback/console-appender.xml"/>
|
||||
<include resource="logback/component/logback-nelo2.xml"/>
|
||||
<include resource="logback/component/logback-datachain.xml"/>
|
||||
<!-- =========== root logger ============== -->
|
||||
<root level="${DEFAULT_LEVEL}">
|
||||
<appender-ref ref="CONSOLE"/>
|
||||
</root>
|
||||
</included>
|
||||
23
support/build.gradle
Normal file
23
support/build.gradle
Normal file
@@ -0,0 +1,23 @@
|
||||
dependencies {
|
||||
compileOnly 'org.projectlombok:lombok'
|
||||
|
||||
// https://projectreactor.io/docs/core/release/reference/#debug-activate
|
||||
annotationProcessor 'org.projectlombok:lombok'
|
||||
implementation 'org.springframework.boot:spring-boot-starter-webflux'
|
||||
|
||||
}
|
||||
|
||||
test {
|
||||
useJUnitPlatform()
|
||||
testLogging {
|
||||
events "passed", "skipped", "failed"
|
||||
}
|
||||
}
|
||||
|
||||
jar {
|
||||
enabled = true
|
||||
}
|
||||
|
||||
bootJar {
|
||||
enabled = false
|
||||
}
|
||||
31
support/next_message_tempalte(block_kit_builder).json
Normal file
31
support/next_message_tempalte(block_kit_builder).json
Normal file
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"blocks": [
|
||||
{
|
||||
"type": "section",
|
||||
"text": {
|
||||
"type": "mrkdwn",
|
||||
"text": "기타)) `[쿠팡] 타푸코 이중 진공 스텐 세라믹 코팅 텀블러 (화이트만 11,130원/로켓와우무료) 4 [기타]`<https://www.ppomppu.co.kr/zboardview.php?id=ppomppu&page=1&divpage=69&category=1&&no=404126|LINK>"
|
||||
},
|
||||
"accessory": {
|
||||
"type": "image",
|
||||
"image_url": "cdn.ppomppu.co.kr/zboard/data3/2021/1121/m_20211121184835_zfyahnow.png",
|
||||
"alt_text": "alt text for image"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "divider"
|
||||
},
|
||||
{
|
||||
"type": "section",
|
||||
"text": {
|
||||
"type": "mrkdwn",
|
||||
"text": "기타)) `[롯데온] 1매 149원 올국산 KF94새부리마스크 대형 200매 (29,920원/무료배송) 58 [기타]`<https://www.ppomppu.co.kr/zboardview.php?id=ppomppu&page=1&divpage=69&category=1&&no=404113|LINK>"
|
||||
},
|
||||
"accessory": {
|
||||
"type": "image",
|
||||
"image_url": "https://s3-media3.fl.yelpcdn.com/bphoto/c7ed05m9lC2EmA3Aruue7A/o.jpg",
|
||||
"alt_text": "alt text for image"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
package com.myoa.engineering.crawl.ppomppu.support.dto;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.Map;
|
||||
import lombok.Getter;
|
||||
|
||||
/**
|
||||
* APIResponse
|
||||
*
|
||||
* @author Shin Woo-jin (woozu.shin@kakaoent.com)
|
||||
* @since 2021-09-07
|
||||
*/
|
||||
@Getter
|
||||
public class APIResponse<T> implements Serializable {
|
||||
private static final long serialVersionUID = 1523350013713908487L;
|
||||
|
||||
private boolean success;
|
||||
private T result;
|
||||
private APIResponseError error;
|
||||
|
||||
public APIResponse(boolean success, T result, APIResponseError error) {
|
||||
this.success = success;
|
||||
this.error = error;
|
||||
this.result = result;
|
||||
}
|
||||
|
||||
public APIResponse(boolean success, T result) {
|
||||
this.success = success;
|
||||
this.result = result;
|
||||
}
|
||||
|
||||
public APIResponse(boolean success, APIResponseError error) {
|
||||
this.success = success;
|
||||
this.error = error;
|
||||
}
|
||||
|
||||
public APIResponse() {}
|
||||
|
||||
public static <T> APIResponse<T> success(T result) {
|
||||
return new APIResponse<>(true, result);
|
||||
}
|
||||
|
||||
public static APIResponse<Void> success() {
|
||||
return new APIResponse<>(true, null);
|
||||
}
|
||||
|
||||
public static <T> APIResponse<T> fail(T result, String code) {
|
||||
return new APIResponse<T>(false, result, APIResponseError.of(code));
|
||||
}
|
||||
|
||||
public static APIResponse<Void> fail(String code) {
|
||||
return new APIResponse<Void>(false, APIResponseError.of(code));
|
||||
}
|
||||
|
||||
public static APIResponse<Void> fail(String code, String message) {
|
||||
return new APIResponse<Void>(false, APIResponseError.of(code, message));
|
||||
}
|
||||
|
||||
public static <K, V> APIResponse<Void> fail(String code, String message, Map<K, V> reasons) {
|
||||
return new APIResponse<Void>(false, APIResponseError.of(code, message, reasons));
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
package com.myoa.engineering.crawl.ppomppu.support.dto;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import lombok.Getter;
|
||||
|
||||
/**
|
||||
* ResponseError
|
||||
*
|
||||
* @author Shin Woo-jin (woozu.shin@kakaoent.com)
|
||||
* @since 2021-09-07
|
||||
*/
|
||||
@Getter
|
||||
public class APIResponseError<K, V> implements Serializable {
|
||||
|
||||
private static final long serialVersionUID = 931593091836887301L;
|
||||
|
||||
private String code;
|
||||
private String message;
|
||||
private Map<K, V> reasons;
|
||||
|
||||
public APIResponseError() {}
|
||||
|
||||
public APIResponseError(String code, String message, Map<K, V> reasons) {
|
||||
this.code = code;
|
||||
this.message = message;
|
||||
this.reasons = reasons;
|
||||
}
|
||||
|
||||
public static <K, V> APIResponseError<K, V> of(String code, String message, Map<K, V> reasons) {
|
||||
return new APIResponseError<>(code, message, reasons);
|
||||
}
|
||||
|
||||
public static APIResponseError<String, String> of(
|
||||
String code, String message, String reasonKey, String reasonValue) {
|
||||
final Map<String, String> reasons = new HashMap<>();
|
||||
reasons.put(reasonKey, reasonValue);
|
||||
return new APIResponseError<String, String>(code, message, reasons);
|
||||
}
|
||||
|
||||
public static APIResponseError<Void, Void> of(String code, String message) {
|
||||
return new APIResponseError<>(code, message, null);
|
||||
}
|
||||
|
||||
public static APIResponseError<Void, Void> of(String code) {
|
||||
return new APIResponseError<>(code, null, null);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
package com.myoa.engineering.crawl.ppomppu.support.dto;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.time.Instant;
|
||||
import java.util.List;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
|
||||
|
||||
import lombok.Builder;
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
/**
|
||||
* SimpleMessageDTO
|
||||
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
|
||||
* @since 2021-11-21
|
||||
*
|
||||
*/
|
||||
@Getter
|
||||
@NoArgsConstructor
|
||||
public class BlockMessageDTO implements Serializable {
|
||||
|
||||
private static final long serialVersionUID = -6992039884035135523L;
|
||||
|
||||
private PpomppuBoardName boardName;
|
||||
private String title;
|
||||
private List<Block> blocks;
|
||||
private String url;
|
||||
private Instant publishedAt;
|
||||
private Instant requestedAt;
|
||||
|
||||
@Builder
|
||||
public BlockMessageDTO(PpomppuBoardName boardName, String title,
|
||||
List<Block> blocks, String url, Instant publishedAt,
|
||||
Instant requestedAt) {
|
||||
this.boardName = boardName;
|
||||
this.title = title;
|
||||
this.blocks = blocks;
|
||||
this.url = url;
|
||||
this.publishedAt = publishedAt;
|
||||
this.requestedAt = requestedAt;
|
||||
}
|
||||
|
||||
@Getter
|
||||
@NoArgsConstructor
|
||||
public static class Block implements Serializable {
|
||||
private static final long serialVersionUID = 3633781631892663709L;
|
||||
|
||||
private String text;
|
||||
private String imageUrl;
|
||||
private String altText;
|
||||
|
||||
public Block(String text, String imageUrl, String altText) {
|
||||
this.text = text;
|
||||
this.imageUrl = imageUrl;
|
||||
this.altText = altText;
|
||||
}
|
||||
}
|
||||
|
||||
public static Block createBlock(String text, String imageUrl) {
|
||||
return new Block(text, imageUrl, "");
|
||||
}
|
||||
|
||||
public static Block createBlock(String text, String imageUrl, String altText) {
|
||||
return new Block(text, imageUrl, altText);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
package com.myoa.engineering.crawl.ppomppu.support.dto;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.time.Instant;
|
||||
|
||||
import lombok.Builder;
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
/**
|
||||
* SimpleMessageDTO
|
||||
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
|
||||
* @since 2021-11-21
|
||||
*
|
||||
*/
|
||||
@Getter
|
||||
@NoArgsConstructor
|
||||
public class SimpleMessageDTO implements Serializable {
|
||||
|
||||
private static final long serialVersionUID = 2203955567672404428L;
|
||||
|
||||
private String title;
|
||||
private String body;
|
||||
private String url;
|
||||
private Instant publishedAt;
|
||||
private Instant requestedAt;
|
||||
|
||||
@Builder
|
||||
public SimpleMessageDTO(String title, String body, String url, Instant publishedAt, Instant requestedAt) {
|
||||
this.title = title;
|
||||
this.body = body;
|
||||
this.url = url;
|
||||
this.publishedAt = publishedAt;
|
||||
this.requestedAt = requestedAt;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
package com.myoa.engineering.crawl.ppomppu.support.dto.code;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Getter;
|
||||
|
||||
/**
|
||||
* PpomppuBoardName
|
||||
*
|
||||
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
|
||||
* @since 2021-09-05
|
||||
*/
|
||||
@Getter
|
||||
@AllArgsConstructor
|
||||
public enum PpomppuBoardName {
|
||||
PPOMPPU_DOMESTIC_ALL("/zboard/zboard.php?id=ppomppu", "국내-전체", false),
|
||||
PPOMPPU_DOMESTIC_ETC("/zboard/zboard.php?id=ppomppu&category=1", "국내-기타", true),
|
||||
PPOMPPU_DOMESTIC_COMPUTER("/zboard/zboard.php?id=ppomppu&category=4", "국내-컴퓨터", true),
|
||||
PPOMPPU_DOMESTIC_DIGITAL("/zboard/zboard.php?id=ppomppu&category=5", "국내-디지털", true),
|
||||
PPOMPPU_DOMESTIC_FOOD("/zboard/zboard.php?id=ppomppu&category=6", "국내-식품/건강", true),
|
||||
PPOMPPU_DOMESTIC_BOOK("/zboard/zboard.php?id=ppomppu&category=8", "국내-서적", true),
|
||||
PPOMPPU_DOMESTIC_APPLIANCES("/zboard/zboard.php?id=ppomppu&category=9", "국내-가전/가구", true),
|
||||
PPOMPPU_DOMESTIC_PARENTING("/zboard/zboard.php?id=ppomppu&category=10", "국내-육아", true),
|
||||
PPOMPPU_DOMESTIC_GIFTCARD("/zboard/zboard.php?id=ppomppu&category=11", "국내-상품권", true),
|
||||
PPOMPPU_DOMESTIC_CLOTHES("/zboard/zboard.php?id=ppomppu&category=12", "국내-의류/잡화", true),
|
||||
PPOMPPU_DOMESTIC_COSMETIC("/zboard/zboard.php?id=ppomppu&category=13", "국내-화장품", true),
|
||||
PPOMPPU_DOMESTIC_OUTDOOR("/zboard/zboard.php?id=ppomppu&category=15", "국내-등산/캠핑", true),
|
||||
PPOMPPU_OVERSEA_ALL("/zboard/zboard.php?id=ppomppu4", "해외-전체", false),
|
||||
PPOMPPU_OVERSEA_ETC("/zboard/zboard.php?id=ppomppu4&category=1", "해외-기타", true),
|
||||
PPOMPPU_OVERSEA_APPLIANCES("/zboard/zboard.php?id=ppomppu4&category=7", "해외-가전", true),
|
||||
PPOMPPU_OVERSEA_TVAV("/zboard/zboard.php?id=ppomppu4&category=8", "해외-TV/영상", true),
|
||||
PPOMPPU_OVERSEA_COMPUTER("/zboard/zboard.php?id=ppomppu4&category=3", "해외-컴퓨터", true),
|
||||
PPOMPPU_OVERSEA_DIGITAL("/zboard/zboard.php?id=ppomppu4&category=4", "해외-디지털", true),
|
||||
PPOMPPU_OVERSEA_MOBILEACCESSORY("/zboard/zboard.php?id=ppomppu4&category=9", "해외-액세서리", true),
|
||||
PPOMPPU_OVERSEA_CLOTHES("/zboard/zboard.php?id=ppomppu4&category=5", "해외-의류/잡화", true),
|
||||
PPOMPPU_OVERSEA_WATCH("/zboard/zboard.php?id=ppomppu4&category=2", "해외-시계", true),
|
||||
PPOMPPU_OVERSEA_SHOES("/zboard/zboard.php?id=ppomppu4&category=11", "해외-신발", true),
|
||||
PPOMPPU_OVERSEA_FOOD("/zboard/zboard.php?id=ppomppu4&category=10", "해외-식품/건강", true),
|
||||
PPOMPPU_OVERSEA_PARENTING("/zboard/zboard.php?id=ppomppu4&category=6", "해외-육아", true),
|
||||
;
|
||||
|
||||
private String resourcePath;
|
||||
private String menuName;
|
||||
private boolean crawlWithDefaultTimer;
|
||||
|
||||
public static final String PPOMPPU_URL = "https://www.ppomppu.co.kr";
|
||||
|
||||
public static String ofViewPageUrl(String articleUrl) {
|
||||
return PPOMPPU_URL + "/zboard/" + articleUrl;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
package com.myoa.engineering.crawl.ppomppu.support.util;
|
||||
|
||||
import java.time.ZoneId;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
|
||||
/**
|
||||
* DateUtil
|
||||
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
|
||||
* @since 2022-01-02
|
||||
*
|
||||
*/
|
||||
public final class DateUtil {
|
||||
|
||||
private DateUtil() { }
|
||||
|
||||
public static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yy.MM.dd HH:mm:ss")
|
||||
.withZone(ZoneId.of("Asia/Seoul"));
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.myoa.engineering.music.soundhoundfound.support.util;
|
||||
package com.myoa.engineering.crawl.ppomppu.support.util;
|
||||
|
||||
public final class NumberUtil {
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
package com.myoa.engineering.crawl.ppomppu.support.util;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonGenerator;
|
||||
import com.fasterxml.jackson.core.JsonParser.Feature;
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.databind.DeserializationFeature;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.databind.SerializationFeature;
|
||||
import com.fasterxml.jackson.databind.SerializerProvider;
|
||||
import com.fasterxml.jackson.databind.ser.std.StdSerializer;
|
||||
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
|
||||
import java.io.IOException;
|
||||
import org.springframework.cache.support.NullValue;
|
||||
import org.springframework.util.StringUtils;
|
||||
|
||||
/**
|
||||
* ObjectMapperFactory
|
||||
*
|
||||
* @author Shin Woo-jin (woozu.shin@kakaoent.com)
|
||||
* @since 2021-08-31
|
||||
*/
|
||||
public final class ObjectMapperFactory {
|
||||
|
||||
private ObjectMapperFactory() {
|
||||
}
|
||||
|
||||
private static final ObjectMapper defaultMapper;
|
||||
private static final ObjectMapper httpMapper;
|
||||
|
||||
static {
|
||||
defaultMapper = initDefaultMapper();
|
||||
httpMapper = initHttpMapper();
|
||||
}
|
||||
|
||||
public static ObjectMapper defaultMapper() {
|
||||
return defaultMapper;
|
||||
}
|
||||
|
||||
public static ObjectMapper httpMapper() {
|
||||
return httpMapper;
|
||||
}
|
||||
|
||||
private static ObjectMapper initDefaultMapper() {
|
||||
final ObjectMapper objectMapper = new ObjectMapper();
|
||||
objectMapper.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES);
|
||||
objectMapper.disable(SerializationFeature.FAIL_ON_EMPTY_BEANS);
|
||||
objectMapper.disable(SerializationFeature.FAIL_ON_UNWRAPPED_TYPE_IDENTIFIERS);
|
||||
objectMapper.enable(DeserializationFeature.READ_UNKNOWN_ENUM_VALUES_AS_NULL);
|
||||
objectMapper.configure(Feature.ALLOW_UNQUOTED_CONTROL_CHARS, true);
|
||||
objectMapper.enable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS);
|
||||
|
||||
objectMapper.registerModule(new JavaTimeModule());
|
||||
return objectMapper;
|
||||
}
|
||||
|
||||
private static ObjectMapper initHttpMapper() {
|
||||
final ObjectMapper objectMapper = initDefaultMapper();
|
||||
objectMapper.configure(DeserializationFeature.READ_UNKNOWN_ENUM_VALUES_USING_DEFAULT_VALUE, true);
|
||||
return objectMapper;
|
||||
}
|
||||
|
||||
public static String writeAsString(Object o) {
|
||||
try {
|
||||
return defaultMapper().writeValueAsString(o);
|
||||
} catch (JsonProcessingException e) {
|
||||
e.printStackTrace();
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Copy from {@link GenericJackson2JsonRedisSerializer.NullValueSerializer}.
|
||||
*/
|
||||
public static class NullValueSerializer extends StdSerializer<NullValue> {
|
||||
|
||||
private static final long serialVersionUID = 6776419544239897328L;
|
||||
private final String classIdentifier;
|
||||
|
||||
/**
|
||||
* @param classIdentifier can be {@literal null} and will be defaulted to {@code @class}.
|
||||
*/
|
||||
NullValueSerializer(String classIdentifier) {
|
||||
super(NullValue.class);
|
||||
this.classIdentifier = StringUtils.hasText(classIdentifier) ? classIdentifier : "@class";
|
||||
}
|
||||
|
||||
/*
|
||||
* (non-Javadoc)
|
||||
* @see com.fasterxml.jackson.databind.ser.std.StdSerializer#serialize(java.lang.Object, com
|
||||
* .fasterxml.jackson.core.JsonGenerator, com.fasterxml.jackson.databind.SerializerProvider)
|
||||
*/
|
||||
@Override
|
||||
public void serialize(NullValue value, JsonGenerator jgen, SerializerProvider provider)
|
||||
throws IOException {
|
||||
|
||||
jgen.writeStartObject();
|
||||
jgen.writeStringField(classIdentifier, NullValue.class.getName());
|
||||
jgen.writeEndObject();
|
||||
}
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user