5 Commits

Author SHA1 Message Date
woo-jin.shin
6d315e2a9f [PPN-13] Deal with new page board structure 2022-01-02 12:48:10 +09:00
woo-jin.shin
8eb431a812 Prettify section message 2021-12-04 01:29:32 +09:00
woo-jin.shin
520a651a70 Apply accessory image 2021-12-04 01:07:42 +09:00
woo-jin.shin
7b230fdb74 [NO-ISSUE] Fix bug 2021-12-01 01:16:15 +09:00
woo-jin.shin
b204b70b79 [NO-ISSUE] Fix bug 2021-12-01 01:16:01 +09:00
166 changed files with 2287 additions and 3025 deletions

3
.gitignore vendored
View File

@@ -37,6 +37,3 @@ out/
.vscode/ .vscode/
temppassword.yml temppassword.yml
data.sql
**/src/main/resources/slack
**/src/main/resources/datasource

View File

@@ -1,14 +1,13 @@
plugins { plugins {
id 'org.springframework.boot' version '2.5.4'
id 'io.spring.dependency-management' version '1.0.11.RELEASE'
id 'java' id 'java'
id 'idea' id 'idea'
id 'org.springframework.boot' version '3.2.5'
id 'io.spring.dependency-management' version '1.1.4'
} }
group = 'com.myoa.engineering.crawl.ppomppu' group = 'com.myoa.engineering.crawl.ppomppu'
version = '1.0.1' version = '1.1.1'
sourceCompatibility = JavaVersion.VERSION_21 sourceCompatibility = '11'
targetCompatibility = JavaVersion.VERSION_21
configurations { configurations {
compileOnly { compileOnly {
@@ -21,8 +20,8 @@ repositories {
} }
allprojects { allprojects {
group = 'com.myoa.engineering.crawl.shopping' group = 'com.myoa.engineering.crawl.ppomppu'
version = '2.0.0' version = '1.1.1'
apply plugin: 'java' apply plugin: 'java'
apply plugin: 'idea' apply plugin: 'idea'
@@ -37,7 +36,7 @@ allprojects {
} }
ext { ext {
set('springCloudVersion', "2023.0.1") set('springCloudVersion', "2020.0.4")
} }
dependencyManagement { dependencyManagement {

3
copy.bat Normal file
View File

@@ -0,0 +1,3 @@
xcopy /y .\processor\build\libs\*.jar .\
xcopy /y .\receiver\build\libs\*.jar .\
xcopy /y .\sender\build\libs\*.jar .\

View File

@@ -1,6 +1,5 @@
#Sun Apr 28 23:47:38 KST 2024
distributionBase=GRADLE_USER_HOME distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists distributionPath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-8.5-bin.zip distributionUrl=https\://services.gradle.org/distributions/gradle-7.1.1-bin.zip
zipStoreBase=GRADLE_USER_HOME zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists zipStorePath=wrapper/dists

0
gradlew vendored Executable file → Normal file
View File

30
processor/build.gradle Normal file
View File

@@ -0,0 +1,30 @@
dependencies {
developmentOnly 'org.springframework.boot:spring-boot-devtools'
runtimeOnly 'com.h2database:h2'
runtimeOnly 'mysql:mysql-connector-java'
compileOnly 'org.projectlombok:lombok'
implementation project(':support')
// https://projectreactor.io/docs/core/release/reference/#debug-activate
implementation 'org.springframework.boot:spring-boot-starter-webflux'
implementation 'org.springframework.boot:spring-boot-starter-data-jpa'
implementation 'org.springframework.boot:spring-boot-starter-jdbc'
implementation 'org.springframework.cloud:spring-cloud-starter-config'
implementation 'org.springframework.boot:spring-boot-starter-actuator'
implementation 'com.rometools:rome:1.16.0'
implementation 'org.jsoup:jsoup:1.14.2'
implementation 'com.h2database:h2:1.4.200'
annotationProcessor 'org.springframework.boot:spring-boot-configuration-processor'
annotationProcessor 'org.projectlombok:lombok'
testImplementation 'org.springframework.boot:spring-boot-starter-test'
}
test {
useJUnitPlatform()
testLogging {
events "passed", "skipped", "failed"
}
}

View File

@@ -0,0 +1,23 @@
package com.myoa.engineering.crawl.ppomppu.processor;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.context.annotation.Import;
import com.myoa.engineering.crawl.ppomppu.support.webclient.PpomppuNotifierWebClientConfiguration;
/**
* ProcessorApplication
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-08-20
*
*/
@Import({ PpomppuNotifierWebClientConfiguration.class })
@SpringBootApplication
public class ProcessorApplication {
public static void main(String[] args) {
SpringApplication.run(ProcessorApplication.class, args);
}
}

View File

@@ -1,5 +1,6 @@
package com.myoa.engineering.crawl.shopping.configuration.datasource; package com.myoa.engineering.crawl.ppomppu.processor.configuration;
import java.sql.SQLException;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.h2.tools.Server; import org.h2.tools.Server;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;
@@ -9,8 +10,6 @@ import org.springframework.context.event.ContextClosedEvent;
import org.springframework.context.event.ContextRefreshedEvent; import org.springframework.context.event.ContextRefreshedEvent;
import org.springframework.context.event.EventListener; import org.springframework.context.event.EventListener;
import java.sql.SQLException;
@Slf4j @Slf4j
@Profile({"datasource-local", "datasource-development"}) @Profile({"datasource-local", "datasource-development"})
@Configuration @Configuration

View File

@@ -1,58 +1,54 @@
package com.myoa.engineering.crawl.shopping.configuration.datasource; package com.myoa.engineering.crawl.ppomppu.processor.configuration;
import com.myoa.engineering.crawl.shopping.configuration.datasource.properties.DatasourceProperties; import com.myoa.engineering.crawl.ppomppu.processor.configuration.properties.DatasourceProperties;
import com.myoa.engineering.crawl.shopping.configuration.datasource.properties.HibernateProperties; import com.myoa.engineering.crawl.ppomppu.processor.configuration.properties.DatasourceProperties.DataSourcePropertiesUnit;
import com.myoa.engineering.crawl.shopping.configuration.datasource.properties.HikariProperties; import com.myoa.engineering.crawl.ppomppu.processor.configuration.properties.HibernateProperties;
import com.myoa.engineering.crawl.shopping.domain.entity.BaseScanDomain; import com.myoa.engineering.crawl.ppomppu.processor.configuration.properties.HikariProperties;
import com.myoa.engineering.crawl.shopping.infra.repository.BaseScanRepository; import com.myoa.engineering.crawl.ppomppu.processor.domain.BaseScanDomain;
import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository.BaseScanRepository;
import com.zaxxer.hikari.HikariConfig; import com.zaxxer.hikari.HikariConfig;
import com.zaxxer.hikari.HikariDataSource; import com.zaxxer.hikari.HikariDataSource;
import jakarta.persistence.EntityManagerFactory; import java.util.Enumeration;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
import javax.persistence.EntityManagerFactory;
import javax.sql.DataSource;
import lombok.NonNull; import lombok.NonNull;
import org.hibernate.boot.model.naming.CamelCaseToUnderscoresNamingStrategy;
import org.hibernate.boot.model.naming.ImplicitNamingStrategyJpaCompliantImpl;
import org.hibernate.cfg.AvailableSettings; import org.hibernate.cfg.AvailableSettings;
import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.boot.orm.jpa.EntityManagerFactoryBuilder; import org.springframework.boot.orm.jpa.EntityManagerFactoryBuilder;
import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Configuration;
import org.springframework.data.jpa.repository.config.EnableJpaAuditing;
import org.springframework.data.jpa.repository.config.EnableJpaRepositories; import org.springframework.data.jpa.repository.config.EnableJpaRepositories;
import org.springframework.orm.jpa.JpaTransactionManager; import org.springframework.orm.jpa.JpaTransactionManager;
import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean; import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean;
import org.springframework.transaction.PlatformTransactionManager; import org.springframework.transaction.PlatformTransactionManager;
import javax.sql.DataSource;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
@Configuration @Configuration
@EnableJpaAuditing
@EnableJpaRepositories(basePackageClasses = BaseScanRepository.class, @EnableJpaRepositories(basePackageClasses = BaseScanRepository.class,
entityManagerFactoryRef = "shoppingCrawlerEntityManagerFactory", entityManagerFactoryRef = "ppomppuNotifierProcessorEntityManagerFactory",
transactionManagerRef = "shoppingCrawlerTransactionManager" transactionManagerRef = "ppomppuNotifierProcessorTransactionManager"
) )
public class ShoppingCrawlerDatasourceConfiguration { public class PpomppuDatasourceConfiguration {
private static final String DATA_SOURCE_UNIT_NAME = "crawler-shopping"; private static final String DATA_SOURCE_UNIT_NAME = "ppn_mysql";
private final DatasourceProperties dataSourceProeprties; private final DatasourceProperties dataSourceProeprties;
private final HikariProperties hikariProperties; private final HikariProperties hikariProperties;
private final HibernateProperties hibernateProperties; private final HibernateProperties hibernateProperties;
public ShoppingCrawlerDatasourceConfiguration(DatasourceProperties dataSourceProeprties, public PpomppuDatasourceConfiguration(DatasourceProperties dataSourceProeprties,
HikariProperties hikariProperties, HikariProperties hikariProperties,
HibernateProperties hibernateProperties) { HibernateProperties hibernateProperties) {
this.dataSourceProeprties = dataSourceProeprties; this.dataSourceProeprties = dataSourceProeprties;
this.hikariProperties = hikariProperties; this.hikariProperties = hikariProperties;
this.hibernateProperties = hibernateProperties; this.hibernateProperties = hibernateProperties;
} }
@Bean(name = "shoppingCrawlerDataSource") @Bean(name = "ppomppuNotifierProcessorDataSource")
public DataSource dataSource() { public DataSource dataSource() {
DatasourceProperties.DataSourcePropertiesUnit dataSourcePropertiesUnit = dataSourceProeprties.find(DATA_SOURCE_UNIT_NAME); DataSourcePropertiesUnit dataSourcePropertiesUnit = dataSourceProeprties.find(DATA_SOURCE_UNIT_NAME);
final HikariConfig hikariConfig = new HikariConfig(); final HikariConfig hikariConfig = new HikariConfig();
hikariConfig.setJdbcUrl(dataSourcePropertiesUnit.toCompletedJdbcUrl()); hikariConfig.setJdbcUrl(dataSourcePropertiesUnit.toCompletedJdbcUrl());
@@ -73,19 +69,19 @@ public class ShoppingCrawlerDatasourceConfiguration {
return dataSource; return dataSource;
} }
@Bean("shoppingCrawlerEntityManagerFactory") @Bean("ppomppuNotifierProcessorEntityManagerFactory")
public LocalContainerEntityManagerFactoryBean entityManagerFactory( public LocalContainerEntityManagerFactoryBean entityManagerFactory(
EntityManagerFactoryBuilder builder, EntityManagerFactoryBuilder builder,
@Qualifier("shoppingCrawlerDataSource") DataSource dataSource) { @Qualifier("ppomppuNotifierProcessorDataSource") DataSource dataSource) {
return builder.dataSource(dataSource) return builder.dataSource(dataSource)
.packages(BaseScanDomain.class) .packages(BaseScanDomain.class)
.properties(getPropsMap(DATA_SOURCE_UNIT_NAME)) .properties(getPropsMap(DATA_SOURCE_UNIT_NAME))
.build(); .build();
} }
@Bean("shoppingCrawlerTransactionManager") @Bean("ppomppuNotifierProcessorTransactionManager")
public PlatformTransactionManager transactionManager( public PlatformTransactionManager transactionManager(
@Qualifier("shoppingCrawlerEntityManagerFactory") EntityManagerFactory entityManagerFactory) { @Qualifier("ppomppuNotifierProcessorEntityManagerFactory") EntityManagerFactory entityManagerFactory) {
return new JpaTransactionManager(entityManagerFactory); return new JpaTransactionManager(entityManagerFactory);
} }
@@ -95,17 +91,20 @@ public class ShoppingCrawlerDatasourceConfiguration {
properties.put(AvailableSettings.FORMAT_SQL, hibernateProperties.getFormatSql()); properties.put(AvailableSettings.FORMAT_SQL, hibernateProperties.getFormatSql());
properties.put(AvailableSettings.SHOW_SQL, hibernateProperties.getShowSql()); properties.put(AvailableSettings.SHOW_SQL, hibernateProperties.getShowSql());
properties.put(AvailableSettings.HBM2DDL_AUTO, hibernateProperties.getHbm2ddlAuto()); properties.put(AvailableSettings.HBM2DDL_AUTO, hibernateProperties.getHbm2ddlAuto());
properties.put(AvailableSettings.CONNECTION_PROVIDER_DISABLES_AUTOCOMMIT, hibernateProperties.getDisableAutoCommit()); properties.put(AvailableSettings.CONNECTION_PROVIDER_DISABLES_AUTOCOMMIT,
properties.put(AvailableSettings.IMPLICIT_NAMING_STRATEGY, ImplicitNamingStrategyJpaCompliantImpl.class.getName()); hibernateProperties.getDisableAutoCommit());
properties.put(AvailableSettings.PHYSICAL_NAMING_STRATEGY, CamelCaseToUnderscoresNamingStrategy.class.getName()); properties.put(AvailableSettings.IMPLICIT_NAMING_STRATEGY,
"org.springframework.boot.orm.jpa.hibernate.SpringImplicitNamingStrategy");
properties.put(AvailableSettings.PHYSICAL_NAMING_STRATEGY,
"org.springframework.boot.orm.jpa.hibernate.SpringPhysicalNamingStrategy");
properties.put(AvailableSettings.GENERATE_STATISTICS, "false"); properties.put(AvailableSettings.GENERATE_STATISTICS, "false");
// properties.put(AvailableSettings.GLOBALLY_QUOTED_IDENTIFIERS, "true"); properties.put(AvailableSettings.GLOBALLY_QUOTED_IDENTIFIERS, "true");
// properties.put(AvailableSettings.GLOBALLY_QUOTED_IDENTIFIERS_SKIP_COLUMN_DEFINITIONS, "true"); properties.put(AvailableSettings.GLOBALLY_QUOTED_IDENTIFIERS_SKIP_COLUMN_DEFINITIONS, "true");
properties.put(AvailableSettings.STATEMENT_BATCH_SIZE, "20"); properties.put(AvailableSettings.STATEMENT_BATCH_SIZE, "20");
properties.put(AvailableSettings.ORDER_INSERTS, "true"); properties.put(AvailableSettings.ORDER_INSERTS, "true");
properties.put(AvailableSettings.ORDER_UPDATES, "true"); properties.put(AvailableSettings.ORDER_UPDATES, "true");
properties.put(AvailableSettings.BATCH_VERSIONED_DATA, "true"); properties.put(AvailableSettings.BATCH_VERSIONED_DATA, "true");
// properties.put(AvailableSettings.JPA_ID_GENERATOR_GLOBAL_SCOPE_COMPLIANCE, "false"); properties.put(AvailableSettings.USE_NEW_ID_GENERATOR_MAPPINGS, "false");
return properties; return properties;
} }

View File

@@ -1,26 +1,29 @@
package com.myoa.engineering.crawl.shopping.configuration.datasource.properties; package com.myoa.engineering.crawl.ppomppu.processor.configuration.properties;
import com.myoa.engineering.crawl.shopping.support.util.ObjectUtil; import com.myoa.engineering.crawl.ppomppu.support.util.ObjectUtil;
import lombok.Data; import java.util.List;
import lombok.Getter;
import lombok.Setter;
import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import java.util.List;
@Component @Component
@Data @Setter
@ConfigurationProperties(prefix = "datasource") @Getter
@ConfigurationProperties(prefix = "infra.database")
public class DatasourceProperties { public class DatasourceProperties {
private List<DataSourcePropertiesUnit> units; private List<DataSourcePropertiesUnit> units;
@Data @Getter
@Setter
public static class DataSourcePropertiesUnit { public static class DataSourcePropertiesUnit {
private String unitName; private String unitName;
private String schemaName; private String schemaName;
private String connectionParameters; private String connectionParameters;
private String dbConnectionUrl; private String datasourceUrl;
private Boolean isSimpleConnectionUrl; private Boolean isSimpleConnectionUrl;
private String username; private String username;
private String password; private String password;
@@ -28,9 +31,9 @@ public class DatasourceProperties {
public String toCompletedJdbcUrl() { public String toCompletedJdbcUrl() {
if (ObjectUtil.isEmpty(isSimpleConnectionUrl) || isSimpleConnectionUrl == false) { if (ObjectUtil.isEmpty(isSimpleConnectionUrl) || isSimpleConnectionUrl == false) {
return String.format("%s/%s?%s", dbConnectionUrl, schemaName, connectionParameters); return String.format("%s/%s?%s", datasourceUrl, schemaName, connectionParameters);
} }
return dbConnectionUrl; return datasourceUrl;
} }
} }
@@ -39,7 +42,7 @@ public class DatasourceProperties {
.filter(e -> e.getUnitName().equals(unitName)) .filter(e -> e.getUnitName().equals(unitName))
.findFirst() .findFirst()
.orElseThrow( .orElseThrow(
() -> new IllegalArgumentException(this.getClass().getName() + ": unitName Not found. " + unitName)); () -> new IllegalArgumentException(this.getClass().getName() + ": unitName Not found. " + unitName));
} }
} }

View File

@@ -1,11 +1,13 @@
package com.myoa.engineering.crawl.shopping.configuration.datasource.properties; package com.myoa.engineering.crawl.ppomppu.processor.configuration.properties;
import lombok.Getter;
import lombok.Setter;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;
import java.util.List; import java.util.List;
import lombok.Getter;
import lombok.Setter;
import org.hibernate.dialect.MySQL8Dialect;
import org.hibernate.dialect.MySQLDialect;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;
@Component @Component
@Setter @Setter

View File

@@ -1,4 +1,4 @@
package com.myoa.engineering.crawl.shopping.configuration.datasource.properties; package com.myoa.engineering.crawl.ppomppu.processor.configuration.properties;
import lombok.Getter; import lombok.Getter;
import lombok.Setter; import lombok.Setter;

View File

@@ -0,0 +1,70 @@
package com.myoa.engineering.crawl.ppomppu.processor.controller;
import java.util.List;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
import com.myoa.engineering.crawl.ppomppu.processor.dto.FeedParsedResult;
import com.myoa.engineering.crawl.ppomppu.processor.service.MessageSenderService;
import com.myoa.engineering.crawl.ppomppu.processor.service.PpomppuArticleService;
import com.myoa.engineering.crawl.ppomppu.processor.service.PpomppuFeedService;
import com.myoa.engineering.crawl.ppomppu.support.dto.APIResponse;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import lombok.extern.slf4j.Slf4j;
import reactor.core.publisher.Mono;
/**
* CrawlAPIController
*
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-09-05
*/
@Slf4j
@RestController
@RequestMapping("/api/v1/crawl")
public class CrawlAPIController {
private final PpomppuFeedService ppomppuRSSFeedService;
private final PpomppuArticleService ppomppuArticleService;
private final MessageSenderService messageSenderService;
public CrawlAPIController(PpomppuFeedService ppomppuRSSFeedService,
PpomppuArticleService ppomppuArticleService,
MessageSenderService messageSenderService) {
this.ppomppuRSSFeedService = ppomppuRSSFeedService;
this.ppomppuArticleService = ppomppuArticleService;
this.messageSenderService = messageSenderService;
}
@PostMapping("/boards/{boardName}")
public Mono<APIResponse<FeedParsedResult>> crawlBoard(@PathVariable("boardName") PpomppuBoardName boardName) {
log.info("got request... {}", boardName);
FeedParsedResult result = FeedParsedResult.of(boardName);
Mono<String> publishedMessages =
ppomppuRSSFeedService.getArticles(boardName)
.map(e -> ppomppuArticleService.filterOnlyNewArticles(boardName, e))
.map(e -> ppomppuArticleService.save(boardName, e))
.filter(e -> !e.isEmpty())
.flatMap(e -> messageSenderService.sendBlockMessageToSlack(boardName, e));
return publishedMessages.then(Mono.just(APIResponse.success(result.done())));
}
@PostMapping("/exploit/boards/{boardName}")
public Mono<APIResponse<String>> crawlBoardDryRun(
@PathVariable("boardName") PpomppuBoardName boardName) {
log.info("got request... {}", boardName);
Mono<String> publishedMessages =
ppomppuRSSFeedService.getArticles(boardName)
.flatMap(e -> messageSenderService.sendBlockMessageToSlack(boardName, e));
return publishedMessages.map(APIResponse::success);
}
}

View File

@@ -1,18 +1,17 @@
package com.myoa.engineering.crawl.shopping.domain.entity; package com.myoa.engineering.crawl.ppomppu.processor.domain;
import lombok.Getter;
import org.springframework.data.annotation.CreatedDate;
import org.springframework.data.annotation.LastModifiedDate;
import jakarta.persistence.*;
import org.springframework.data.jpa.domain.support.AuditingEntityListener;
import java.io.Serializable; import java.io.Serializable;
import java.time.Instant; import java.time.Instant;
import javax.persistence.Column;
import org.springframework.data.annotation.CreatedDate;
import org.springframework.data.annotation.LastModifiedDate;
@Getter /**
@MappedSuperclass * Auditable
@EntityListeners(AuditingEntityListener.class) *
* @author Shin Woo-jin (woozu.shin@kakaoent.com)
* @since 2021-09-08
*/
public abstract class Auditable implements Serializable { public abstract class Auditable implements Serializable {
private static final long serialVersionUID = -7105030870015828551L; private static final long serialVersionUID = -7105030870015828551L;

View File

@@ -1,4 +1,4 @@
package com.myoa.engineering.crawl.shopping.domain.entity; package com.myoa.engineering.crawl.ppomppu.processor.domain;
/** /**
* BaseScanDomain * BaseScanDomain

View File

@@ -1,16 +1,19 @@
package com.myoa.engineering.crawl.shopping.domain.entity.v1; package com.myoa.engineering.crawl.ppomppu.processor.domain;
import com.myoa.engineering.crawl.shopping.domain.entity.Auditable; import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName; import java.time.Instant;
import jakarta.persistence.*; import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.EnumType;
import javax.persistence.Enumerated;
import javax.persistence.GeneratedValue;
import javax.persistence.GenerationType;
import javax.persistence.Id;
import javax.persistence.Table;
import lombok.Builder; import lombok.Builder;
import lombok.Getter; import lombok.Getter;
import lombok.NoArgsConstructor; import lombok.NoArgsConstructor;
import lombok.ToString;
import java.time.ZonedDateTime;
@ToString
@Getter @Getter
@NoArgsConstructor @NoArgsConstructor
@Entity @Entity
@@ -31,6 +34,9 @@ public class PpomppuArticle extends Auditable {
@Column @Column
private String articleUrl; private String articleUrl;
@Column
private String thumbnailUrl;
@Column @Column
private String title; private String title;
@@ -41,19 +47,25 @@ public class PpomppuArticle extends Auditable {
private Integer recommended; private Integer recommended;
@Column @Column
private ZonedDateTime registeredAt; private Instant registeredAt;
@Builder @Builder
public PpomppuArticle(Long id, Long articleId, PpomppuBoardName boardName, String articleUrl, public PpomppuArticle(Long id, Long articleId, PpomppuBoardName boardName, String articleUrl,
String title, Integer recommended, Integer hit, ZonedDateTime registeredAt) { String thumbnailUrl, String title, Integer recommended, Integer hit,
Instant registeredAt) {
this.id = id; this.id = id;
this.articleId = articleId; this.articleId = articleId;
this.boardName = boardName; this.boardName = boardName;
this.articleUrl = articleUrl; this.articleUrl = articleUrl;
this.thumbnailUrl = thumbnailUrl;
this.title = title; this.title = title;
this.recommended = recommended; this.recommended = recommended;
this.hit = hit; this.hit = hit;
this.registeredAt = registeredAt; this.registeredAt = registeredAt;
} }
public PpomppuArticle updateBoardName(PpomppuBoardName boardName) {
this.boardName = boardName;
return this;
}
} }

View File

@@ -1,14 +1,19 @@
package com.myoa.engineering.crawl.shopping.domain.entity.v1; package com.myoa.engineering.crawl.ppomppu.processor.domain;
import com.myoa.engineering.crawl.shopping.domain.entity.Auditable; import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName; import java.time.Instant;
import jakarta.persistence.*; import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.EnumType;
import javax.persistence.Enumerated;
import javax.persistence.GeneratedValue;
import javax.persistence.GenerationType;
import javax.persistence.Id;
import javax.persistence.Table;
import lombok.Builder; import lombok.Builder;
import lombok.Getter; import lombok.Getter;
import lombok.NoArgsConstructor; import lombok.NoArgsConstructor;
import java.time.Instant;
@Getter @Getter
@NoArgsConstructor @NoArgsConstructor
@Entity @Entity

View File

@@ -1,12 +1,15 @@
package com.myoa.engineering.crawl.shopping.domain.entity.v1; package com.myoa.engineering.crawl.ppomppu.processor.domain;
import com.myoa.engineering.crawl.shopping.domain.entity.Auditable; import java.time.Instant;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.GeneratedValue;
import javax.persistence.GenerationType;
import javax.persistence.Id;
import javax.persistence.Table;
import lombok.Getter; import lombok.Getter;
import lombok.NoArgsConstructor; import lombok.NoArgsConstructor;
import jakarta.persistence.*;
import java.time.Instant;
@Getter @Getter
@NoArgsConstructor @NoArgsConstructor
@Entity @Entity

View File

@@ -0,0 +1,32 @@
package com.myoa.engineering.crawl.ppomppu.processor.domain;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.EnumType;
import javax.persistence.Enumerated;
import javax.persistence.GeneratedValue;
import javax.persistence.GenerationType;
import javax.persistence.Id;
import javax.persistence.Table;
import lombok.Getter;
import lombok.NoArgsConstructor;
@Getter
@NoArgsConstructor
@Entity
@Table(name = "subscribed_board")
public class SubscribedBoard extends Auditable {
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
private Long id;
@Column
private Long userId;
@Column
@Enumerated(EnumType.STRING)
private PpomppuBoardName boardName;
}

View File

@@ -1,17 +1,20 @@
package com.myoa.engineering.crawl.shopping.domain.entity.v1; package com.myoa.engineering.crawl.ppomppu.processor.domain;
import com.myoa.engineering.crawl.shopping.domain.entity.Auditable; import java.time.Instant;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.GeneratedValue;
import javax.persistence.GenerationType;
import javax.persistence.Id;
import javax.persistence.Table;
import lombok.Getter; import lombok.Getter;
import lombok.NoArgsConstructor; import lombok.NoArgsConstructor;
import jakarta.persistence.*;
import java.time.Instant;
@Getter @Getter
@NoArgsConstructor @NoArgsConstructor
@Entity @Entity
@Table(name = "subscribed_user") @Table(name = "subscribed_user")
public class SubscribedUser extends Auditable { public class SubscribedUser extends Auditable{
@Id @Id
@GeneratedValue(strategy = GenerationType.IDENTITY) @GeneratedValue(strategy = GenerationType.IDENTITY)

View File

@@ -1,13 +1,12 @@
package com.myoa.engineering.crawl.shopping.dto; package com.myoa.engineering.crawl.ppomppu.processor.dto;
import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName; import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import java.io.Serializable;
import java.time.Instant;
import lombok.Builder; import lombok.Builder;
import lombok.Getter; import lombok.Getter;
import lombok.NoArgsConstructor; import lombok.NoArgsConstructor;
import java.io.Serializable;
import java.time.Instant;
/** /**
* FeedParsedResult * FeedParsedResult
* *

View File

@@ -0,0 +1,66 @@
package com.myoa.engineering.crawl.ppomppu.processor.dto;
import java.time.Instant;
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
import com.myoa.engineering.crawl.ppomppu.support.util.DateUtil;
import lombok.Builder;
import lombok.Getter;
import lombok.NoArgsConstructor;
/**
* PpomppuArticleParseDTO
*
* @author Shin Woo-jin (woozu.shin@kakaoent.com)
* @since 2021-09-08
*/
@Getter
@NoArgsConstructor
public class PpomppuArticleParseDTO {
private String id;
private String articleId;
private String boardName;
private String articleUrl;
private String thumbnailUrl;
private String title;
private String hit;
private Integer recommended;
private String registeredAt;
@Builder
public PpomppuArticleParseDTO(String id, String articleId, String boardName, String articleUrl,
String thumbnailUrl, String title, String hit, Integer recommended,
String registeredAt) {
this.id = id;
this.articleId = articleId;
this.boardName = boardName;
this.articleUrl = articleUrl;
this.thumbnailUrl = thumbnailUrl;
this.title = title;
this.hit = hit;
this.recommended = recommended;
this.registeredAt = registeredAt;
}
public boolean isInValidated() {
return articleId == null || articleId.isEmpty()
|| hit == null || hit.isEmpty();
}
public PpomppuArticle convert() {
if (isInValidated()) {
throw new IllegalArgumentException("PpomppuArticleParseDTO was invalidated");
}
return PpomppuArticle.builder()
.articleId(Long.parseLong(articleId))
.title(title)
.articleUrl(articleUrl)
.thumbnailUrl(thumbnailUrl)
.recommended(recommended)
.hit(Integer.parseInt(hit))
.registeredAt(DateUtil.DATE_TIME_FORMATTER.parse(registeredAt, Instant::from))
.build();
}
}

View File

@@ -0,0 +1,78 @@
package com.myoa.engineering.crawl.ppomppu.processor.dto;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
/**
* PpomppuArticleTransformer
*
* @author Shin Woo-jin (woozu.shin@kakaoent.com)
* @since 2021-09-08
*/
public final class PpomppuArticleParser {
private PpomppuArticleParser() {}
public static PpomppuArticle toArticle(Elements articleElement) {
final String articleIdString = PpomppuArticleParser.parseArticleId(articleElement.get(0));
final String title = PpomppuArticleParser.parseTitle(articleElement.get(2));
final String articleUrl = PpomppuArticleParser.parseArticleUrl(articleElement.get(2));
final String thumbnailUrl = PpomppuArticleParser.parseThumbnailUrl(articleElement.get(3));
final Integer recommended = PpomppuArticleParser.parseRecommended(articleElement.get(6));
final String hitString = PpomppuArticleParser.parseHit(articleElement.get(7));
final String registeredAtString = PpomppuArticleParser.parseRegisteredAt(articleElement.get(5));
return PpomppuArticleParseDTO.builder()
.articleId(articleIdString)
.title(title)
.articleUrl(articleUrl)
.thumbnailUrl(thumbnailUrl)
.recommended(recommended)
.hit(hitString)
.registeredAt(registeredAtString)
.build()
.convert();
}
public static String parseArticleId(Element td) {
return td.text().trim();
}
public static String parseTitle(Element td) {
return td.getElementsByTag("a").text();
}
public static String parseArticleUrl(Element td) {
return PpomppuBoardName.ofViewPageUrl(td.getElementsByTag("a").attr("href"));
}
public static String parseThumbnailUrl(Element td) {
return "https:" + td.getElementsByTag("img").get(0).attr("src");
}
public static Integer parseRecommended(Element td) {
final String voteString = td.text();
final int recommended;
if (voteString.isEmpty()) {
recommended = 0;
} else {
final int voteUp = Integer.parseInt(td.text().split(" - ")[0]);
final int voteDown = Integer.parseInt(td.text().split(" - ")[1]);
recommended = voteUp - voteDown;
}
return recommended;
}
public static String parseHit(Element td) {
return td.text();
}
public static String parseRegisteredAt(Element td) {
return td.attr("title");
}
}

View File

@@ -0,0 +1,70 @@
package com.myoa.engineering.crawl.ppomppu.processor.dto;
import java.time.Instant;
import java.util.List;
import java.util.function.Function;
import java.util.stream.Collectors;
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
import com.myoa.engineering.crawl.ppomppu.support.dto.BlockMessageDTO;
import com.myoa.engineering.crawl.ppomppu.support.dto.SimpleMessageDTO;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import com.myoa.engineering.crawl.ppomppu.support.util.DateUtil;
/**
* PpomppuArticleTransformer
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-11-21
*
*/
public final class PpomppuArticleTransformer {
private PpomppuArticleTransformer() {}
private static final String MESSAGE_FORMAT_V1 = "%s)) <%s|LINK> `%s` ";
private static final String MESSAGE_FORMAT_V2 = "%s *<%s|LINK>*\n%s";
private static final String TITLE_FORMAT_V1 = "_*:hearts: %s | %s*_";
public static final Function<PpomppuArticle, SimpleMessageDTO> TRANSFORM_TO_MESSAGE_DTO = article ->
SimpleMessageDTO.builder()
.requestedAt(Instant.now())
.publishedAt(article.getRegisteredAt())
.title(String.format(MESSAGE_FORMAT_V1,
article.getBoardName().getMenuName(), article.getArticleUrl(),
article.getTitle()))
.body(article.getArticleUrl())
.build();
// https://stackoverflow.com/questions/24882927/using-streams-to-convert-a-list-of-objects-into-a-string-obtained-from-the-tostr
public static SimpleMessageDTO transformToSimpleMessage(List<PpomppuArticle> articles) {
Instant requestedAt = Instant.now();
String body = articles.stream()
.map(PpomppuArticleTransformer::convertToInlineMessage)
.collect(Collectors.joining("\n\n"));
return SimpleMessageDTO.builder()
.requestedAt(requestedAt)
.title(DateUtil.DATE_TIME_FORMATTER.format(requestedAt))
.body(body)
.build();
}
public static BlockMessageDTO transformToBlockMessage(PpomppuBoardName boardName, List<PpomppuArticle> articles) {
Instant requestedAt = Instant.now();
List<BlockMessageDTO.Block> body = articles.stream()
.map(e -> BlockMessageDTO.createBlock(convertToInlineMessage(e),
e.getThumbnailUrl()))
.collect(Collectors.toList());
return BlockMessageDTO.builder()
.requestedAt(requestedAt)
.title(String.format(TITLE_FORMAT_V1,
boardName.getMenuName(),
DateUtil.DATE_TIME_FORMATTER.format(requestedAt)))
.blocks(body)
.build();
}
public static String convertToInlineMessage(PpomppuArticle article) {
return String.format(MESSAGE_FORMAT_V2,
article.getBoardName().getMenuName(), article.getArticleUrl(), article.getTitle());
}
}

View File

@@ -1,4 +1,4 @@
package com.myoa.engineering.crawl.shopping.dto.constant; package com.myoa.engineering.crawl.ppomppu.processor.dto.constant;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;
import lombok.Getter; import lombok.Getter;

View File

@@ -0,0 +1,70 @@
package com.myoa.engineering.crawl.ppomppu.processor.infrastructure.client;
import org.springframework.core.ParameterizedTypeReference;
import org.springframework.http.HttpHeaders;
import org.springframework.http.MediaType;
import org.springframework.stereotype.Component;
import org.springframework.web.reactive.function.client.WebClient;
import org.springframework.web.reactive.function.client.WebClientRequestException;
import com.myoa.engineering.crawl.ppomppu.processor.dto.constant.WebClientPropertiesUnitName;
import com.myoa.engineering.crawl.ppomppu.support.dto.BlockMessageDTO;
import com.myoa.engineering.crawl.ppomppu.support.dto.SimpleMessageDTO;
import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebClientFilterFactory;
import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebFluxExchangeStragiesFactory;
import com.myoa.engineering.crawl.ppomppu.support.webclient.properties.WebClientProperties;
import com.myoa.engineering.crawl.ppomppu.support.webclient.properties.WebClientProperties.WebClientPropertiesUnit;
import lombok.extern.slf4j.Slf4j;
import reactor.core.publisher.Mono;
import reactor.core.scheduler.Schedulers;
/**
* PpomppuNotifierSenderAPIClient
*
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-11-17
*/
@Slf4j
@Component
public class MessageSenderAPIClient {
private final WebClient webClient;
public MessageSenderAPIClient(WebClientProperties webClientProperties) {
WebClientPropertiesUnit webClientPropertiesUnit =
webClientProperties.find(WebClientPropertiesUnitName.PPOMPPU_NOTIFIER_SENDER_API.getUnitName());
this.webClient = WebClient.builder()
.baseUrl(webClientPropertiesUnit.getBaseUrl())
.exchangeStrategies(WebFluxExchangeStragiesFactory.ofDefault())
.defaultHeader(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_VALUE)
// .filter(WebClientFilterFactory.logRequest())
// .filter(WebClientFilterFactory.logResponse())
.build();
}
public Mono<String> sendSimpleMessageToSlack(SimpleMessageDTO dto) {
return webClient.post()
.uri("/api/v1/messages/sendSimpleMessage/messengers/slack")
.bodyValue(dto)
.exchangeToMono(e -> e.bodyToMono(new ParameterizedTypeReference<String>() {}))
.publishOn(Schedulers.boundedElastic())
.onErrorResume(WebClientRequestException.class, t -> {
log.info("Exception occured, ignoring. : {}", t.getClass().getSimpleName());
return Mono.empty();
});
}
public Mono<String> sendBlockMessageToSlack(BlockMessageDTO dto) {
return webClient.post()
.uri("/api/v1/messages/sendBlockMessage/messengers/slack")
.bodyValue(dto)
.exchangeToMono(e -> e.bodyToMono(new ParameterizedTypeReference<String>() {}))
.publishOn(Schedulers.boundedElastic())
.onErrorResume(WebClientRequestException.class, t -> {
log.info("Exception occured, ignoring. : {}", t.getClass().getSimpleName());
return Mono.empty();
});
}
}

View File

@@ -1,7 +1,16 @@
package com.myoa.engineering.crawl.shopping.infra.client.ppomppu; package com.myoa.engineering.crawl.ppomppu.processor.infrastructure.client;
import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebClientFilterFactory;
import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebFluxExchangeStragiesFactory;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.core.ParameterizedTypeReference;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import org.springframework.web.reactive.function.client.WebClient;
import org.springframework.web.reactive.function.client.WebClientRequestException;
import reactor.core.publisher.Mono;
import reactor.core.scheduler.Schedulers;
/** /**
* PpomppuBoardFeedRetriever * PpomppuBoardFeedRetriever
@@ -11,11 +20,11 @@ import org.springframework.stereotype.Component;
*/ */
@Slf4j @Slf4j
@Component @Component
public class PpomppuBoardClient { public class PpomppuBoardFeedRetriever {
/*
private final WebClient webClient; private final WebClient webClient;
public PpomppuBoardClient(WebClient.Builder webClientBuilder) { public PpomppuBoardFeedRetriever(WebClient.Builder webClientBuilder) {
this.webClient = webClientBuilder.baseUrl(PpomppuBoardName.PPOMPPU_URL) this.webClient = webClientBuilder.baseUrl(PpomppuBoardName.PPOMPPU_URL)
.exchangeStrategies(WebFluxExchangeStragiesFactory.ofTextHtml()) .exchangeStrategies(WebFluxExchangeStragiesFactory.ofTextHtml())
.filter(WebClientFilterFactory.logRequest()) .filter(WebClientFilterFactory.logRequest())
@@ -34,5 +43,5 @@ public class PpomppuBoardClient {
}); });
// .doOnNext(e -> log.info("[getHtml] {}", e)); // .doOnNext(e -> log.info("[getHtml] {}", e));
} }
*/
} }

View File

@@ -0,0 +1,4 @@
package com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository;
public interface BaseScanRepository {
}

View File

@@ -1,6 +1,6 @@
package com.myoa.engineering.crawl.shopping.infra.repository.v1; package com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository;
import com.myoa.engineering.crawl.shopping.domain.entity.v1.PpomppuArticle; import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.stereotype.Repository; import org.springframework.stereotype.Repository;

View File

@@ -1,12 +1,11 @@
package com.myoa.engineering.crawl.shopping.infra.repository.v1; package com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository;
import com.myoa.engineering.crawl.shopping.domain.entity.v1.PpomppuBoardFeedStatus; import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuBoardFeedStatus;
import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName; import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import java.util.Optional;
import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.stereotype.Repository; import org.springframework.stereotype.Repository;
import java.util.Optional;
@Repository @Repository
public interface PpomppuBoardFeedStatusRepository extends JpaRepository<PpomppuBoardFeedStatus, Long> { public interface PpomppuBoardFeedStatusRepository extends JpaRepository<PpomppuBoardFeedStatus, Long> {

View File

@@ -0,0 +1,44 @@
package com.myoa.engineering.crawl.ppomppu.processor.service;
import java.util.List;
import org.springframework.stereotype.Service;
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
import com.myoa.engineering.crawl.ppomppu.processor.dto.PpomppuArticleTransformer;
import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.client.MessageSenderAPIClient;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import lombok.extern.slf4j.Slf4j;
import reactor.core.publisher.Mono;
/**
* MessageSenderService
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-11-21
*
*/
@Slf4j
@Service
public class MessageSenderService {
private final MessageSenderAPIClient messageSenderAPIClient;
public MessageSenderService(MessageSenderAPIClient messageSenderAPIClient) {
this.messageSenderAPIClient = messageSenderAPIClient;
}
public Mono<String> sendSimpleMessageToSlack(PpomppuArticle article) {
return messageSenderAPIClient.sendSimpleMessageToSlack(PpomppuArticleTransformer.TRANSFORM_TO_MESSAGE_DTO.apply(article));
}
public Mono<String> sendSimpleMessageToSlack(List<PpomppuArticle> articles) {
return messageSenderAPIClient.sendSimpleMessageToSlack(PpomppuArticleTransformer.transformToSimpleMessage(articles));
}
public Mono<String> sendBlockMessageToSlack(PpomppuBoardName boardName, List<PpomppuArticle> articles) {
return messageSenderAPIClient.sendBlockMessageToSlack(
PpomppuArticleTransformer.transformToBlockMessage(boardName, articles));
}
}

View File

@@ -1,18 +1,20 @@
package com.myoa.engineering.crawl.shopping.service; package com.myoa.engineering.crawl.ppomppu.processor.service;
import com.myoa.engineering.crawl.shopping.domain.entity.v1.PpomppuArticle;
import com.myoa.engineering.crawl.shopping.domain.entity.v1.PpomppuBoardFeedStatus;
import com.myoa.engineering.crawl.shopping.infra.repository.v1.PpomppuArticleRepository;
import com.myoa.engineering.crawl.shopping.infra.repository.v1.PpomppuBoardFeedStatusRepository;
import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import java.util.List; import java.util.List;
import java.util.Optional; import java.util.Optional;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuBoardFeedStatus;
import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository.PpomppuArticleRepository;
import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository.PpomppuBoardFeedStatusRepository;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import lombok.extern.slf4j.Slf4j;
@Slf4j @Slf4j
@Service @Service
public class PpomppuArticleService { public class PpomppuArticleService {
@@ -48,7 +50,10 @@ public class PpomppuArticleService {
// save PpomppuBoardFeedStatus // save PpomppuBoardFeedStatus
Optional<PpomppuBoardFeedStatus> boardFeedStatus = ppomppuBoardFeedStatusRepository.findByBoardName(boardName); Optional<PpomppuBoardFeedStatus> boardFeedStatus = ppomppuBoardFeedStatusRepository.findByBoardName(boardName);
log.info("boardName: {}, isPresent?: {}", boardName, boardFeedStatus.isPresent()); log.info("[save] boardName: {}, isPresent?: {}, latestArticleId: {}",
boardName, boardFeedStatus.isPresent(), latestArticleId);
log.info("[save] articles count: {}, article ids: {}",
articles.size(), articles.stream().map(PpomppuArticle::getArticleId).toArray());
boardFeedStatus.ifPresentOrElse(e -> { boardFeedStatus.ifPresentOrElse(e -> {
if (latestArticleId.longValue() > 0L) { if (latestArticleId.longValue() > 0L) {
e.updateArticleId(latestArticleId); e.updateArticleId(latestArticleId);

View File

@@ -1,12 +1,20 @@
package com.myoa.engineering.crawl.shopping.service; package com.myoa.engineering.crawl.ppomppu.processor.service;
import java.util.Comparator;
import java.util.List;
import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName;
import lombok.extern.slf4j.Slf4j;
import org.jsoup.Jsoup; import org.jsoup.Jsoup;
import org.jsoup.nodes.Element; import org.jsoup.nodes.Element;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import java.util.List; import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
import com.myoa.engineering.crawl.ppomppu.processor.dto.PpomppuArticleParser;
import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.client.PpomppuBoardFeedRetriever;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import lombok.extern.slf4j.Slf4j;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
/** /**
* PpomppuFeedService * PpomppuFeedService
@@ -17,7 +25,7 @@ import java.util.List;
@Slf4j @Slf4j
@Component @Component
public class PpomppuFeedService { public class PpomppuFeedService {
/*
private final PpomppuBoardFeedRetriever ppomppuBoardFeedRetriever; private final PpomppuBoardFeedRetriever ppomppuBoardFeedRetriever;
public PpomppuFeedService(PpomppuBoardFeedRetriever ppomppuBoardFeedRetriever) { public PpomppuFeedService(PpomppuBoardFeedRetriever ppomppuBoardFeedRetriever) {
@@ -27,9 +35,12 @@ public class PpomppuFeedService {
public Mono<List<PpomppuArticle>> getArticles(PpomppuBoardName boardName) { public Mono<List<PpomppuArticle>> getArticles(PpomppuBoardName boardName) {
final Mono<String> html = ppomppuBoardFeedRetriever.getHtml(boardName.getResourcePath()); final Mono<String> html = ppomppuBoardFeedRetriever.getHtml(boardName.getResourcePath());
final Mono<Element> tbody = extractTbodyFromHtml(html); final Mono<Element> tbody = extractTbodyFromHtml(html);
// .doOnNext(e -> log.info("pre tbody - {}", e.html())); // .doOnNext(e -> log.info("pre tbody - {}", e.html()));
return extractArticlesFromTbody(tbody).map(this::convertFromElement) return extractArticlesFromTbody(tbody).map(this::convertFromElement)
.onErrorContinue((t, e) -> log.error("Error occured : {}, value: {}",
e, t.getLocalizedMessage()))
.map(e -> e.updateBoardName(boardName)) .map(e -> e.updateBoardName(boardName))
.sort(Comparator.comparing(PpomppuArticle::getArticleId))
// .doOnNext(e -> log.info("parsed Result: {}", e)) // .doOnNext(e -> log.info("parsed Result: {}", e))
.collectList(); .collectList();
} }
@@ -52,6 +63,4 @@ public class PpomppuFeedService {
private PpomppuArticle convertFromElement(Element element) { private PpomppuArticle convertFromElement(Element element) {
return PpomppuArticleParser.toArticle(element.getElementsByTag("td")); return PpomppuArticleParser.toArticle(element.getElementsByTag("td"));
} }
*/
} }

View File

@@ -3,7 +3,7 @@
* LINE Corporation PROPRIETARY/CONFIDENTIAL. Use is subject to license terms. * LINE Corporation PROPRIETARY/CONFIDENTIAL. Use is subject to license terms.
*/ */
package com.myoa.engineering.crawl.shopping.util; package com.myoa.engineering.crawl.ppomppu.processor.util;
import java.util.Arrays; import java.util.Arrays;
import java.util.Collection; import java.util.Collection;
@@ -14,9 +14,9 @@ import java.util.Collection;
* @author Shin Woo-jin (lp12254@linecorp.com) * @author Shin Woo-jin (lp12254@linecorp.com)
* @since 2019-10-28 * @since 2019-10-28
*/ */
public final class ObjectUtils { public final class ObjectUtil {
private ObjectUtils() { private ObjectUtil() {
} }
/** /**
@@ -58,7 +58,7 @@ public final class ObjectUtils {
* @return Is there objects array has null? * @return Is there objects array has null?
*/ */
public static boolean hasNullObject(Object... args) { public static boolean hasNullObject(Object... args) {
return Arrays.stream(args).anyMatch(ObjectUtils::isNullObject); return Arrays.stream(args).anyMatch(ObjectUtil::isNullObject);
} }
/** /**
@@ -72,7 +72,7 @@ public final class ObjectUtils {
* @return Is there objects array has null? * @return Is there objects array has null?
*/ */
public static boolean hasAllObject(Object... args) { public static boolean hasAllObject(Object... args) {
return Arrays.stream(args).noneMatch(ObjectUtils::isNullObject); return Arrays.stream(args).noneMatch(ObjectUtil::isNullObject);
} }
/** /**
@@ -86,7 +86,7 @@ public final class ObjectUtils {
* @return Is there null all of given objects? * @return Is there null all of given objects?
*/ */
public static boolean hasAllNullObjects(final Object... args) { public static boolean hasAllNullObjects(final Object... args) {
return Arrays.stream(args).allMatch(ObjectUtils::isNullObject); return Arrays.stream(args).allMatch(ObjectUtil::isNullObject);
} }
/** /**

View File

@@ -2,11 +2,11 @@ spring:
config: config:
activate: activate:
on-profile: development on-profile: development
# import: import:
# - "configserver:http://192.168.0.100:20085" - "configserver:http://192.168.0.100:11080"
server: server:
port: 20080 port: 20081
# import: optional:configserver:http://localhost:11080 # can be start up even config server was not found. # import: optional:configserver:http://localhost:11080 # can be start up even config server was not found.

View File

@@ -0,0 +1,11 @@
spring:
config:
activate:
on-profile: local
import:
- "configserver:http://localhost:20085"
server:
port: 20081
# import: optional:configserver:http://localhost:11080 # can be start up even config server was not found.

View File

@@ -1,6 +1,6 @@
spring: spring:
application: application:
name: crawler-shopping name: ppn-processor
main: main:
allow-bean-definition-overriding: true allow-bean-definition-overriding: true
profiles: profiles:
@@ -8,12 +8,9 @@ spring:
group: group:
local: "local,datasource-local,webclient-local" local: "local,datasource-local,webclient-local"
development: "development,datasource-development,webclient-development" development: "development,datasource-development,webclient-development"
production: "production, datasource-production,webclient-production" production: "production,datasource-production,webclient-production"
freemarker: freemarker:
enabled: false enabled: false
cloud:
config:
enabled: false
server: server:
port: 20080 port: 20080
@@ -25,4 +22,4 @@ management:
endpoints: endpoints:
web: web:
exposure: exposure:
include: refresh include: refresh,health

26
receiver/build.gradle Normal file
View File

@@ -0,0 +1,26 @@
dependencies {
developmentOnly 'org.springframework.boot:spring-boot-devtools'
runtimeOnly 'com.h2database:h2'
runtimeOnly 'mysql:mysql-connector-java'
compileOnly 'org.projectlombok:lombok'
implementation project(':support')
// https://projectreactor.io/docs/core/release/reference/#debug-activate
implementation 'org.springframework.boot:spring-boot-starter-webflux'
implementation 'org.springframework.boot:spring-boot-configuration-processor'
implementation 'org.springframework.cloud:spring-cloud-starter-config'
implementation 'org.springframework.boot:spring-boot-starter-actuator'
implementation 'org.telegram:telegrambots:5.3.0'
annotationProcessor 'org.springframework.boot:spring-boot-configuration-processor'
annotationProcessor 'org.projectlombok:lombok'
testImplementation 'org.springframework.boot:spring-boot-starter-test'
}
test {
useJUnitPlatform()
testLogging {
events "passed", "skipped", "failed"
}
}

View File

@@ -0,0 +1,25 @@
package com.myoa.engineering.crawl.ppomppu.receiver;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.annotation.Import;
import com.myoa.engineering.crawl.ppomppu.receiver.configuration.properties.TelegramBotProperties;
import com.myoa.engineering.crawl.ppomppu.support.webclient.PpomppuNotifierWebClientConfiguration;
/**
* ReceiverApplication
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-08-20
*
*/
@Import({ PpomppuNotifierWebClientConfiguration.class})
@SpringBootApplication
// @EnableConfigurationProperties({ TelegramBotProperties.class })
public class ReceiverApplication {
public static void main(String[] args) {
SpringApplication.run(ReceiverApplication.class, args);
}
}

View File

@@ -0,0 +1,39 @@
package com.myoa.engineering.crawl.ppomppu.receiver.configuration;
import java.util.List;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.telegram.telegrambots.meta.TelegramBotsApi;
import org.telegram.telegrambots.meta.exceptions.TelegramApiException;
import org.telegram.telegrambots.updatesreceivers.DefaultBotSession;
import com.myoa.engineering.crawl.ppomppu.receiver.configuration.properties.TelegramBotProperties;
import com.myoa.engineering.crawl.ppomppu.receiver.configuration.properties.TelegramBotProperties.TelegramBotPropertiesUnit;
import com.myoa.engineering.crawl.ppomppu.receiver.dispatch.MessageDispatcher;
import com.myoa.engineering.crawl.ppomppu.receiver.handler.message.MessageHandler;
/**
* TelegramBotConfiguration
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-08-21
*
*/
@Configuration
public class TelegramBotConfiguration {
private static final String BOT_PROPERTIES_UNIT_NAME = "ppomppu_notify_bot";
@Bean
public TelegramBotsApi telegramBotsApi(MessageDispatcher messageDispatcher) throws TelegramApiException {
TelegramBotsApi api = new TelegramBotsApi(DefaultBotSession.class);
api.registerBot(messageDispatcher);
return api;
}
@Bean
public MessageDispatcher messageDispatcher(List<MessageHandler> messageHandlers,
TelegramBotProperties telegramBotProperties) {
TelegramBotPropertiesUnit propertiesUnit = telegramBotProperties.find(BOT_PROPERTIES_UNIT_NAME);
return new MessageDispatcher(messageHandlers, propertiesUnit.getName(), propertiesUnit.getToken());
}
}

View File

@@ -0,0 +1,44 @@
package com.myoa.engineering.crawl.ppomppu.receiver.configuration.properties;
import java.util.ArrayList;
import java.util.List;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.boot.context.properties.ConstructorBinding;
import org.springframework.stereotype.Component;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
/**
* TelegramBotProperties
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-09-05
*
*/
@Component
@Setter
@Getter
@ConfigurationProperties(prefix = "infra.telegram.bot")
public class TelegramBotProperties {
private List<TelegramBotPropertiesUnit> units = new ArrayList<>();
@Data
public static class TelegramBotPropertiesUnit {
private String unitName;
private String name;
private String token;
}
public TelegramBotPropertiesUnit find(String unitName) {
return units.stream()
.filter(e -> e.getUnitName().equals(unitName))
.findFirst()
.orElseThrow(
() -> new IllegalArgumentException(this.getClass().getName() + ": unitName Not found. " + unitName));
}
}

View File

@@ -0,0 +1,33 @@
package com.myoa.engineering.crawl.ppomppu.receiver.controller.v1;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import com.myoa.engineering.crawl.ppomppu.receiver.service.ProcessorAPIService;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import reactor.core.publisher.Mono;
/**
* EventAPIController
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-09-05
*
*/
@RestController
@RequestMapping("/api/v1/")
public class EventAPIController {
private final ProcessorAPIService processorAPIService;
public EventAPIController(ProcessorAPIService processorAPIService) {
this.processorAPIService = processorAPIService;
}
@PostMapping("/exploit/parsePpomppuRSS/{boardName}")
public Mono<String> parsePpomppuRSS(@PathVariable("boardName") PpomppuBoardName boardName) {
return processorAPIService.emitParseEvent(boardName);
}
}

View File

@@ -0,0 +1,50 @@
package com.myoa.engineering.crawl.ppomppu.receiver.dispatch;
import java.util.List;
import org.telegram.telegrambots.bots.TelegramLongPollingBot;
import org.telegram.telegrambots.meta.api.objects.Message;
import org.telegram.telegrambots.meta.api.objects.Update;
import com.myoa.engineering.crawl.ppomppu.receiver.handler.message.MessageHandler;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class MessageDispatcher extends TelegramLongPollingBot {
private final List<MessageHandler> messageHandlers;
private final String botName;
private final String botToken;
public MessageDispatcher(List<MessageHandler> messageHandlers, String botName, String botToken) {
this.messageHandlers = messageHandlers;
this.botName = botName;
this.botToken = botToken;
}
@Override
public String getBotToken() {
return botToken;
}
@Override
public void onUpdateReceived(Update update) {
Message message = update.getMessage();
MessageHandler handler = getMessageHandler(message);
handler.handle(message);
}
private MessageHandler getMessageHandler(Message message) {
return messageHandlers.stream()
.filter(e -> e.isApplicable(message))
.findFirst()
.orElseThrow(() -> new IllegalArgumentException("Can not found applicable handler"));
}
@Override
public String getBotUsername() {
return botName;
}
}

View File

@@ -0,0 +1,25 @@
package com.myoa.engineering.crawl.ppomppu.receiver.dto;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import java.io.Serializable;
import java.time.Instant;
import lombok.Getter;
import lombok.NoArgsConstructor;
/**
* FeedParsedResult
*
* @author Shin Woo-jin (woozu.shin@kakaoent.com)
* @since 2021-09-08
*/
@Getter
@NoArgsConstructor
public class FeedParsedResult implements Serializable {
private static final long serialVersionUID = -3771310078623481348L;
private PpomppuBoardName boardName;
private Instant requestedAt;
private Instant processedAt;
}

View File

@@ -0,0 +1,19 @@
package com.myoa.engineering.crawl.ppomppu.receiver.dto.constant;
import lombok.AllArgsConstructor;
import lombok.Getter;
/**
* WebClientPropertiesUnitName
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-11-18
*
*/
@Getter
@AllArgsConstructor
public enum WebClientPropertiesUnitName {
PPOMPPU_NOTIFIER_PROCESSOR_API("ppn-processor-api"),
;
private String unitName;
}

View File

@@ -0,0 +1,21 @@
package com.myoa.engineering.crawl.ppomppu.receiver.handler.message;
import com.myoa.engineering.crawl.ppomppu.support.util.ObjectUtil;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import org.telegram.telegrambots.meta.api.objects.Message;
@Slf4j
@Component
public class HelloWorldMessageHandler implements MessageHandler {
@Override
public boolean isApplicable(Message message) {
return ObjectUtil.isEmpty(message);
}
@Override
public void handle(Message message) {
// skip empty event message.
}
}

View File

@@ -0,0 +1,12 @@
package com.myoa.engineering.crawl.ppomppu.receiver.handler.message;
/**
* ImageHandler
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-08-21
*
*/
public interface ImageMessageHandler extends MessageHandler {
}

View File

@@ -0,0 +1,17 @@
package com.myoa.engineering.crawl.ppomppu.receiver.handler.message;
import org.telegram.telegrambots.meta.api.objects.Message;
/**
* MessageHandler
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-08-21
*
*/
public interface MessageHandler {
boolean isApplicable(Message message);
void handle(Message message);
}

View File

@@ -0,0 +1,18 @@
package com.myoa.engineering.crawl.ppomppu.receiver.handler.message;
import com.myoa.engineering.crawl.ppomppu.support.util.ObjectUtil;
import org.telegram.telegrambots.meta.api.objects.Message;
/**
* TextMessageHandler
*
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-08-21
*/
public interface TextMessageHandler extends MessageHandler {
@Override
default boolean isApplicable(Message message) {
return ObjectUtil.isNotEmpty(message) && message.isUserMessage() && message.hasText();
}
}

View File

@@ -0,0 +1,48 @@
package com.myoa.engineering.crawl.ppomppu.receiver.handler.message.text;
import java.util.List;
import org.springframework.stereotype.Component;
import org.telegram.telegrambots.meta.api.objects.Message;
import com.myoa.engineering.crawl.ppomppu.receiver.handler.message.TextMessageHandler;
import lombok.extern.slf4j.Slf4j;
/**
* CommandHandler
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-08-21
*
*/
@Slf4j
@Component
public class CommandHandler implements TextMessageHandler {
private final List<TextCommandProcessor> processors;
public CommandHandler(List<TextCommandProcessor> processors) {
this.processors = processors;
}
@Override
public boolean isApplicable(Message message) {
return TextMessageHandler.super.isApplicable(message)
&& message.isCommand(); // && message.getText().startsWith("/");
}
@Override
public void handle(Message message) {
log.info("CommandHandler : {}", message.getText());
TextCommandCode commandCode = TextCommandCode.find(message.getText());
TextCommandProcessor applicableProcessor = getApplicableProcessor(commandCode);
applicableProcessor.process(message);
}
private TextCommandProcessor getApplicableProcessor(TextCommandCode commandCode) {
return processors.stream()
.filter(e -> e.isApplicable(commandCode))
.findFirst()
.orElseThrow(() -> new IllegalArgumentException("Can not found"));
}
}

View File

@@ -0,0 +1,30 @@
package com.myoa.engineering.crawl.ppomppu.receiver.handler.message.text;
import org.springframework.stereotype.Component;
import org.telegram.telegrambots.meta.api.objects.Message;
import com.myoa.engineering.crawl.ppomppu.receiver.handler.message.TextMessageHandler;
import lombok.extern.slf4j.Slf4j;
/**
* NormalTextHandler
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-08-21
*
*/
@Slf4j
@Component
public class CommonTextHandler implements TextMessageHandler {
@Override
public boolean isApplicable(Message message) {
return TextMessageHandler.super.isApplicable(message) && message.isCommand() == false;
}
@Override
public void handle(Message message) {
log.info("CommonTextHandler : {}", message.getText());
}
}

View File

@@ -0,0 +1,24 @@
package com.myoa.engineering.crawl.ppomppu.receiver.handler.message.text;
import org.springframework.stereotype.Component;
import org.telegram.telegrambots.meta.api.objects.Message;
/**
* EmptyTextCommandProcessor
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-09-05
*
*/
@Component
public class EmptyTextCommandProcessor implements TextCommandProcessor {
@Override
public boolean isApplicable(TextCommandCode commandCode) {
return commandCode == TextCommandCode.EMPTY;
}
@Override
public void process(Message message) {
}
}

View File

@@ -0,0 +1,27 @@
package com.myoa.engineering.crawl.ppomppu.receiver.handler.message.text;
import org.springframework.stereotype.Component;
import org.telegram.telegrambots.meta.api.objects.Message;
import lombok.extern.slf4j.Slf4j;
/**
* StartCommandProcessor
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-09-05
*
*/
@Slf4j
@Component
public class StartTextCommandProcessor implements TextCommandProcessor {
@Override
public boolean isApplicable(TextCommandCode commandCode) {
return TextCommandCode.START == commandCode;
}
@Override
public void process(Message message) {
log.info("[process] user: {}, command: {}", message.getChatId(), message.getText());
}
}

View File

@@ -0,0 +1,33 @@
package com.myoa.engineering.crawl.ppomppu.receiver.handler.message.text;
import java.util.Arrays;
import lombok.Getter;
import lombok.NoArgsConstructor;
/**
* CommandTextCode
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-09-05
*
*/
@Getter
@NoArgsConstructor
public enum TextCommandCode {
EMPTY(null),
START("/start"),
;
private String value;
TextCommandCode(String value) {
this.value = value;
}
public static TextCommandCode find(String value) {
return Arrays.stream(TextCommandCode.values())
.filter(e -> e != EMPTY)
.filter(e -> value.startsWith(e.getValue()))
.findFirst()
.orElse(TextCommandCode.EMPTY);
}
}

View File

@@ -0,0 +1,17 @@
package com.myoa.engineering.crawl.ppomppu.receiver.handler.message.text;
import org.telegram.telegrambots.meta.api.objects.Message;
/**
* TextCommandProcessor
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-09-05
*
*/
public interface TextCommandProcessor {
boolean isApplicable(TextCommandCode commandCode);
void process(Message message);
}

View File

@@ -0,0 +1,44 @@
package com.myoa.engineering.crawl.ppomppu.receiver.infrastructure.client;
import org.springframework.core.ParameterizedTypeReference;
import org.springframework.stereotype.Component;
import org.springframework.web.reactive.function.client.WebClient;
import org.springframework.web.reactive.function.client.WebClientRequestException;
import com.myoa.engineering.crawl.ppomppu.receiver.dto.constant.WebClientPropertiesUnitName;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebClientFilterFactory;
import com.myoa.engineering.crawl.ppomppu.support.webclient.properties.WebClientProperties;
import com.myoa.engineering.crawl.ppomppu.support.webclient.properties.WebClientProperties.WebClientPropertiesUnit;
import lombok.extern.slf4j.Slf4j;
import reactor.core.publisher.Mono;
import reactor.core.scheduler.Schedulers;
@Slf4j
@Component
public class ProcessorAPIWebClient {
private final WebClient webClient;
public ProcessorAPIWebClient(WebClientProperties webClientProperties) {
WebClientPropertiesUnit webClientPropertiesUnit =
webClientProperties.find(WebClientPropertiesUnitName.PPOMPPU_NOTIFIER_PROCESSOR_API.getUnitName());
this.webClient = WebClient.builder()
.baseUrl(webClientPropertiesUnit.getBaseUrl())
.filter(WebClientFilterFactory.logRequest())
.filter(WebClientFilterFactory.logResponse())
.build();
}
public Mono<String> emitParseEvent(PpomppuBoardName boardName) {
return webClient.post()
.uri("/api/v1/crawl/boards/{boardName}", boardName)
.exchangeToMono(e -> e.bodyToMono(new ParameterizedTypeReference<String>() {}))
.publishOn(Schedulers.boundedElastic())
.onErrorResume(WebClientRequestException.class, t -> {
log.info("Exception occured, ignoring. : {}", t.getClass().getSimpleName());
return Mono.empty();
});
}
}

View File

@@ -0,0 +1,38 @@
package com.myoa.engineering.crawl.ppomppu.receiver.scheduler;
import java.util.Arrays;
import org.springframework.scheduling.annotation.EnableScheduling;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import com.myoa.engineering.crawl.ppomppu.receiver.service.ProcessorAPIService;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import lombok.extern.slf4j.Slf4j;
/**
* ParseEventEmitter
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-09-05
*
*/
@Slf4j
@Component
@EnableScheduling
public class ParseEventEmitter {
private final ProcessorAPIService processorAPIService;
public ParseEventEmitter(ProcessorAPIService processorAPIService) {
this.processorAPIService = processorAPIService;
}
@Scheduled(fixedRate = 600 * 1000L)
public void emitBoards() {
log.info("[emitDomesticBoard] trigger fired!");
Arrays.stream(PpomppuBoardName.values())
.filter(PpomppuBoardName::isCrawlWithDefaultTimer)
.forEach(boardName -> processorAPIService.emitParseEvent(boardName).block());
}
}

View File

@@ -0,0 +1,30 @@
package com.myoa.engineering.crawl.ppomppu.receiver.service;
import org.springframework.stereotype.Service;
import com.myoa.engineering.crawl.ppomppu.receiver.infrastructure.client.ProcessorAPIWebClient;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import lombok.extern.slf4j.Slf4j;
import reactor.core.publisher.Mono;
/**
* ProcessorAPIService
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-09-05
*
*/
@Slf4j
@Service
public class ProcessorAPIService {
private final ProcessorAPIWebClient processorAPIWebClient;
public ProcessorAPIService(ProcessorAPIWebClient processorAPIWebClient) {
this.processorAPIWebClient = processorAPIWebClient;
}
public Mono<String> emitParseEvent(PpomppuBoardName boardName) {
return processorAPIWebClient.emitParseEvent(boardName);
}
}

View File

@@ -0,0 +1,6 @@
spring:
config:
activate:
on-profile: development
import:
- "configserver:http://192.168.0.100:11080"

View File

@@ -0,0 +1,6 @@
spring:
config:
activate:
on-profile: local
import:
- "configserver:http://localhost:20085"

View File

@@ -0,0 +1,6 @@
spring:
config:
activate:
on-profile: production
import:
- "configserver:http://ppn-config-server:20080"

View File

@@ -0,0 +1,25 @@
spring:
application:
name: ppn-receiver
main:
allow-bean-definition-overriding: true
profiles:
active: ${SPRING_ACTIVE_PROFILE:local}
group:
local: "local,webclient-local"
development: "development,webclient-development"
production: "production,webclient-production"
freemarker:
enabled: false
server:
port: 20080
error:
whitelabel:
enabled: false
management:
endpoints:
web:
exposure:
include: refresh,health

View File

@@ -0,0 +1,15 @@
<?xml version="1.0" encoding="UTF-8"?>
<configuration>
<springProperty name="DEFAULT_LEVEL_CONFIG" source="log.defaultLevel" />
<springProfile name="local">
<include resource="logback/logback-development.xml" />
<logger name="org.apache.kafka" level="INFO" />
</springProfile>
<springProfile name="development">
<include resource="logback/logback-development.xml" />
<logger name="org.apache.kafka" level="INFO" />
</springProfile>
<springProfile name="production">
<include resource="logback/logback-production.xml" />
</springProfile>
</configuration>

View File

@@ -0,0 +1,23 @@
<included>
<property name="FILE_LOG_PATTERN"
value="%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{45}:%L - %msg%n" />
<property name="LOG_FILE_BASE" value="lcp-benefit-benefit-api" />
<appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${DIRECTORY}/${LOG_FILE_BASE}_log</file>
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<fileNamePattern>${DIRECTORY}/${LOG_FILE_BASE}_log.%d{yyyyMMdd}.%i</fileNamePattern>
<timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
<maxFileSize>1000MB</maxFileSize>
</timeBasedFileNamingAndTriggeringPolicy>
<maxHistory>60</maxHistory>
</rollingPolicy>
<encoder>
<pattern>${FILE_LOG_PATTERN}</pattern>
<immediateFlush>${IMMEDIATE_FLUSH}</immediateFlush>
</encoder>
</appender>
<appender name="ASYNC_FILE" class="ch.qos.logback.classic.AsyncAppender">
<queueSize>1024</queueSize>
<appender-ref ref="FILE" />
</appender>
</included>

View File

@@ -0,0 +1,19 @@
<?xml version="1.0" encoding="UTF-8"?>
<included>
<!-- =========== property BETA ========= -->
<property name="DEFAULT_LEVEL" value="${DEFAULT_LEVEL_CONFIG:-INFO}"/>
<!--file-->
<property name="DIRECTORY" value="/home1/www/logs/supervisor"/>
<property name="IMMEDIATE_FLUSH" value="true"/>
<!--nelo2-->
<property name="NELO2_LEVEL" value="WARN"/>
<!-- =========== include appender =========== -->
<include resource="org/springframework/boot/logging/logback/defaults.xml"/>
<include resource="org/springframework/boot/logging/logback/console-appender.xml"/>
<include resource="logback/component/logback-nelo2.xml"/>
<include resource="logback/component/logback-datachain.xml"/>
<!-- =========== root logger ============== -->
<root level="${DEFAULT_LEVEL}">
<appender-ref ref="CONSOLE"/>
</root>
</included>

View File

@@ -0,0 +1,19 @@
<?xml version="1.0" encoding="UTF-8"?>
<included>
<!-- =========== property RELEASE ========= -->
<property name="DEFAULT_LEVEL" value="${DEFAULT_LEVEL_CONFIG:-INFO}"/>
<!--file-->
<property name="DIRECTORY" value="/home1/www/logs/supervisor"/>
<property name="IMMEDIATE_FLUSH" value="true"/>
<!--nelo2-->
<property name="NELO2_LEVEL" value="WARN"/>
<!-- =========== include appender =========== -->
<include resource="org/springframework/boot/logging/logback/defaults.xml"/>
<include resource="org/springframework/boot/logging/logback/console-appender.xml"/>
<include resource="logback/component/logback-nelo2.xml"/>
<include resource="logback/component/logback-datachain.xml"/>
<!-- =========== root logger ============== -->
<root level="${DEFAULT_LEVEL}">
<appender-ref ref="CONSOLE"/>
</root>
</included>

23
sender/build.gradle Normal file
View File

@@ -0,0 +1,23 @@
dependencies {
developmentOnly 'org.springframework.boot:spring-boot-devtools'
runtimeOnly 'mysql:mysql-connector-java'
compileOnly 'org.projectlombok:lombok'
implementation project(':support')
// https://projectreactor.io/docs/core/release/reference/#debug-activate
implementation 'org.springframework.boot:spring-boot-starter-webflux'
implementation 'org.springframework.boot:spring-boot-starter-actuator'
implementation 'org.springframework.cloud:spring-cloud-starter-config'
annotationProcessor 'org.springframework.boot:spring-boot-configuration-processor'
annotationProcessor 'org.projectlombok:lombok'
testImplementation 'org.springframework.boot:spring-boot-starter-test'
}
test {
useJUnitPlatform()
testLogging {
events "passed", "skipped", "failed"
}
}

View File

@@ -0,0 +1,22 @@
package com.myoa.engineering.crawl.ppomppu.sender;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.context.annotation.Import;
import com.myoa.engineering.crawl.ppomppu.support.webclient.PpomppuNotifierWebClientConfiguration;
/**
* SenderApplication
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-08-20
*
*/
@Import({ PpomppuNotifierWebClientConfiguration.class })
@SpringBootApplication
public class SenderApplication {
public static void main(String[] args) {
SpringApplication.run(SenderApplication.class, args);
}
}

View File

@@ -1,4 +1,4 @@
package com.myoa.engineering.crawl.shopping.configuration.slack.properties; package com.myoa.engineering.crawl.ppomppu.sender.configuration.properties;
import java.util.List; import java.util.List;
import lombok.Data; import lombok.Data;
@@ -7,16 +7,18 @@ import lombok.Setter;
import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
@Data @Getter
@Setter
@Component @Component
@ConfigurationProperties(prefix = "slack.bot") @ConfigurationProperties("infra.slack.bot")
public class SlackSecretProperties { public class SlackSecretProperties {
private List<SlackSecretPropertiesUnit> units; private List<SlackSecretPropertiesUnit> units;
@Data @Data
public static class SlackSecretPropertiesUnit { public static class SlackSecretPropertiesUnit {
private String botUnitName;
private String botName;
private String username; private String username;
private String iconEmoji; private String iconEmoji;
private String channel; private String channel;
@@ -25,7 +27,7 @@ public class SlackSecretProperties {
public SlackSecretPropertiesUnit find(String botUnitName) { public SlackSecretPropertiesUnit find(String botUnitName) {
return units.stream() return units.stream()
.filter(e -> e.getBotUnitName().equals(botUnitName)) .filter(e -> e.getBotName().equals(botUnitName))
.findFirst() .findFirst()
.orElseThrow(() -> new IllegalArgumentException("not found bot unit name : " + botUnitName)); .orElseThrow(() -> new IllegalArgumentException("not found bot unit name : " + botUnitName));
} }

View File

@@ -0,0 +1,62 @@
package com.myoa.engineering.crawl.ppomppu.sender.controller;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import com.myoa.engineering.crawl.ppomppu.sender.dto.SlackBaseMessageBlock;
import com.myoa.engineering.crawl.ppomppu.sender.dto.SlackMessageDTO;
import com.myoa.engineering.crawl.ppomppu.sender.infrastructure.client.MongeShoppingBotSlackMessageSender;
import com.myoa.engineering.crawl.ppomppu.support.dto.APIResponse;
import com.myoa.engineering.crawl.ppomppu.support.dto.BlockMessageDTO;
import com.myoa.engineering.crawl.ppomppu.support.dto.SimpleMessageDTO;
import com.myoa.engineering.crawl.ppomppu.support.util.ObjectMapperFactory;
import lombok.extern.slf4j.Slf4j;
import reactor.core.publisher.Mono;
/**
* MessageSenderAPIController
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-11-21
*
*/
@Slf4j
@RestController
@RequestMapping("/api/v1")
public class MessageSenderAPIController {
private final MongeShoppingBotSlackMessageSender sender;
public MessageSenderAPIController(MongeShoppingBotSlackMessageSender sender) {
this.sender = sender;
}
@PostMapping("/messages/sendSimpleMessage/messengers/slack")
public Mono<APIResponse<SimpleMessageDTO>> sendSimpleMessageToSlack(@RequestBody SimpleMessageDTO dto) {
return sender.sendMessage(sender.ofMessage(dto.getBody()))
.then(Mono.just(APIResponse.success(dto)));
}
@PostMapping("/messages/sendBlockMessage/messengers/slack")
public Mono<APIResponse<BlockMessageDTO>> sendBlockMessageToSlack(@RequestBody BlockMessageDTO dto) {
if (dto.getBlocks().isEmpty()) {
return Mono.just(APIResponse.fail(dto, "empty blocks"));
}
return sender.sendMessage(buildSlackMessageDTO(dto))
// .doOnNext(e -> log.info("[sendBlockMessageToSlack] slackMessageDTO: {}",
// ObjectMapperFactory.writeAsString(buildSlackMessageDTO(dto))))
.then(Mono.just(APIResponse.success(dto)));
}
private SlackMessageDTO buildSlackMessageDTO(BlockMessageDTO dto) {
SlackMessageDTO slackMessageDTO = sender.ofBlockMessageBased();
slackMessageDTO.addSectionBlock(dto.getTitle());
dto.getBlocks().forEach(slackMessageDTO::addSectionBlock);
slackMessageDTO.addBlock(SlackBaseMessageBlock.ofDivider());
return slackMessageDTO;
}
}

View File

@@ -0,0 +1,33 @@
package com.myoa.engineering.crawl.ppomppu.sender.controller;
import com.myoa.engineering.crawl.ppomppu.sender.infrastructure.client.MongeShoppingBotSlackMessageSender;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import lombok.extern.slf4j.Slf4j;
import reactor.core.publisher.Mono;
/**
* TestAPIController
*
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-11-15
*/
@Slf4j
@RestController
@RequestMapping("/api/v1")
public class TestAPIController {
private final MongeShoppingBotSlackMessageSender sender;
public TestAPIController(MongeShoppingBotSlackMessageSender sender) {
this.sender = sender;
}
@GetMapping("/test")
public Mono<String> test() {
log.info("received!!!");
return sender.sendMessage(sender.ofMessage("testtesttest!!!"));
}
}

View File

@@ -1,4 +1,4 @@
package com.myoa.engineering.crawl.shopping.dto.slack; package com.myoa.engineering.crawl.ppomppu.sender.dto;
import java.io.Serializable; import java.io.Serializable;

View File

@@ -0,0 +1,47 @@
package com.myoa.engineering.crawl.ppomppu.sender.dto;
import com.fasterxml.jackson.annotation.JsonInclude;
import lombok.Builder;
import lombok.Getter;
import lombok.NoArgsConstructor;
/**
* MessageBlock
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-11-30
*
*/
@Getter
@NoArgsConstructor
@JsonInclude(JsonInclude.Include.NON_NULL)
public class SlackBaseMessageBlock implements SlackMessageBlock {
private static final long serialVersionUID = 1597984001727808419L;
private SlackMessageBlockType type;
private String text;
@Builder
private SlackBaseMessageBlock(SlackMessageBlockType type, String text) {
this.type = type;
this.text = text;
}
public static SlackBaseMessageBlock ofMarkDown(String message) {
return SlackBaseMessageBlock.builder()
.type(SlackMessageBlockType.MARKDOWN)
.text(message)
.build();
}
public static SlackBaseMessageBlock ofDivider() {
return SlackBaseMessageBlock.builder()
.type(SlackMessageBlockType.DIVIDER)
.build();
}
@Override
public String getType() {
return type.getType();
}
}

View File

@@ -0,0 +1,49 @@
package com.myoa.engineering.crawl.ppomppu.sender.dto;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.Builder;
import lombok.Getter;
import lombok.NoArgsConstructor;
/**
* SlackImageMessageBlock
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-11-30
*
*/
@Getter
@NoArgsConstructor
@JsonInclude(JsonInclude.Include.NON_NULL)
public class SlackImageMessageBlock implements SlackMessageBlock {
private static final long serialVersionUID = 1597984001727808419L;
private SlackMessageBlockType type;
@JsonProperty(value = "image_url", required = true)
private String imageUrl;
@JsonProperty(value = "alt_text", required = true)
private String altText;
@Builder
private SlackImageMessageBlock(SlackMessageBlockType type, String imageUrl, String altText) {
this.type = type;
this.imageUrl = imageUrl;
this.altText = altText;
}
public static SlackImageMessageBlock of(String imageUrl, String altText) {
return SlackImageMessageBlock.builder()
.type(SlackMessageBlockType.IMAGE)
.imageUrl(imageUrl)
.altText(altText)
.build();
}
@Override
public String getType() {
return type.getType();
}
}

View File

@@ -0,0 +1,15 @@
package com.myoa.engineering.crawl.ppomppu.sender.dto;
import java.io.Serializable;
/**
* SlackMessageBlock
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-12-01
*
*/
public interface SlackMessageBlock extends Serializable {
String getType();
}

View File

@@ -0,0 +1,22 @@
package com.myoa.engineering.crawl.ppomppu.sender.dto;
import lombok.AllArgsConstructor;
import lombok.Getter;
/**
* BlockType
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-11-30
*
*/
@Getter
@AllArgsConstructor
public enum SlackMessageBlockType {
SECTION("section"),
MARKDOWN("mrkdwn"),
DIVIDER("divider"),
IMAGE("image"),
;
private String type;
}

View File

@@ -0,0 +1,62 @@
package com.myoa.engineering.crawl.ppomppu.sender.dto;
import java.util.List;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.myoa.engineering.crawl.ppomppu.support.dto.BlockMessageDTO;
import com.myoa.engineering.crawl.ppomppu.support.dto.BlockMessageDTO.Block;
import lombok.Builder;
import lombok.Getter;
import lombok.NoArgsConstructor;
/**
* SlackMessageDTO
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-11-14
*
*/
@Getter
@NoArgsConstructor
@JsonInclude(JsonInclude.Include.NON_NULL)
public class SlackMessageDTO implements MessageDTO {
private final static long serialVersionUID = 4737608709660494713L;
private String text;
private String channel;
private String username;
private List<SlackMessageBlock> blocks;
@JsonProperty("icon_emoji")
private String iconEmoji;
@Builder
public SlackMessageDTO(String text, String channel, String username,
List<SlackMessageBlock> blocks, String iconEmoji) {
this.text = text;
this.channel = channel;
this.username = username;
this.blocks = blocks;
this.iconEmoji = iconEmoji;
}
public void applyText(String text) {
this.text = text;
}
public void addSectionBlock(BlockMessageDTO.Block block) {
SlackSectionMessageBlock slackSectionMessageBlock = SlackSectionMessageBlock.ofMarkDown(block.getText());
slackSectionMessageBlock.applyImageaccessory(block.getImageUrl(), block.getAltText());
addBlock(slackSectionMessageBlock);
}
public void addSectionBlock(String rawBlockMessage) {
addBlock(SlackSectionMessageBlock.ofMarkDown(rawBlockMessage));
}
public void addBlock(SlackMessageBlock block) {
blocks.add(block);
}
}

View File

@@ -0,0 +1,49 @@
package com.myoa.engineering.crawl.ppomppu.sender.dto;
import com.fasterxml.jackson.annotation.JsonInclude;
import lombok.Builder;
import lombok.Getter;
import lombok.NoArgsConstructor;
/**
* SectionBlock
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-11-30
*
*/
@Getter
@NoArgsConstructor
@JsonInclude(JsonInclude.Include.NON_NULL)
public class SlackSectionMessageBlock implements SlackMessageBlock {
private static final long serialVersionUID = -7600944576753160168L;
private SlackMessageBlockType type;
private SlackBaseMessageBlock text;
private SlackImageMessageBlock accessory;
@Builder
private SlackSectionMessageBlock(SlackMessageBlockType type, SlackBaseMessageBlock text,
SlackImageMessageBlock accessory) {
this.type = type;
this.text = text;
this.accessory = accessory;
}
public static SlackSectionMessageBlock ofMarkDown(String message) {
return SlackSectionMessageBlock.builder()
.type(SlackMessageBlockType.SECTION)
.text(SlackBaseMessageBlock.ofMarkDown(message))
.build();
}
public SlackSectionMessageBlock applyImageaccessory(String imageUrl, String altText) {
this.accessory = SlackImageMessageBlock.of(imageUrl, altText);
return this;
}
@Override
public String getType() {
return type.getType();
}
}

View File

@@ -0,0 +1,17 @@
package com.myoa.engineering.crawl.ppomppu.sender.infrastructure.client;
import com.myoa.engineering.crawl.ppomppu.sender.dto.MessageDTO;
import reactor.core.publisher.Mono;
/**
* MessageSender
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-11-14
*
*/
public interface MessageSender<T extends MessageDTO> {
Mono<String> sendMessage(T message);
}

View File

@@ -0,0 +1,52 @@
package com.myoa.engineering.crawl.ppomppu.sender.infrastructure.client;
import java.util.ArrayList;
import org.springframework.stereotype.Component;
import com.myoa.engineering.crawl.ppomppu.sender.configuration.properties.SlackSecretProperties;
import com.myoa.engineering.crawl.ppomppu.sender.configuration.properties.SlackSecretProperties.SlackSecretPropertiesUnit;
import com.myoa.engineering.crawl.ppomppu.sender.dto.SlackMessageDTO;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Component
public class MongeShoppingBotSlackMessageSender extends SlackMessageSender {
private static final String SLACK_SECRET_UNIT_NAME = "monge_shopping_bot";
private final SlackSecretPropertiesUnit slackProperties;
public MongeShoppingBotSlackMessageSender(SlackSecretProperties slackSecretProperties) {
super(slackSecretProperties.find(SLACK_SECRET_UNIT_NAME).getToken());
this.slackProperties = slackSecretProperties.find(SLACK_SECRET_UNIT_NAME);
}
public SlackMessageDTO ofMessageTemplate() {
return SlackMessageDTO.builder()
.channel(slackProperties.getChannel())
.iconEmoji(slackProperties.getIconEmoji())
.username(slackProperties.getUsername())
.build();
}
public SlackMessageDTO ofMessage(String text) {
return SlackMessageDTO.builder()
.channel(slackProperties.getChannel())
.iconEmoji(slackProperties.getIconEmoji())
.username(slackProperties.getUsername())
.text(text)
.build();
}
public SlackMessageDTO ofBlockMessageBased() {
return SlackMessageDTO.builder()
.channel(slackProperties.getChannel())
.iconEmoji(slackProperties.getIconEmoji())
.username(slackProperties.getUsername())
.blocks(new ArrayList<>())
.build();
}
}

View File

@@ -1,6 +1,17 @@
package com.myoa.engineering.crawl.shopping.infra.client.slack; package com.myoa.engineering.crawl.ppomppu.sender.infrastructure.client;
import org.springframework.http.HttpHeaders;
import org.springframework.http.MediaType;
import org.springframework.web.reactive.function.client.WebClient;
import org.springframework.web.reactive.function.client.WebClientRequestException;
import com.myoa.engineering.crawl.ppomppu.sender.dto.SlackMessageDTO;
import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebClientFilterFactory;
import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebFluxExchangeStragiesFactory;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import reactor.core.publisher.Mono;
import reactor.core.scheduler.Schedulers;
/** /**
* SlackMessageSender * SlackMessageSender
@@ -9,9 +20,8 @@ import lombok.extern.slf4j.Slf4j;
* @since 2021-09-08 * @since 2021-09-08
*/ */
@Slf4j @Slf4j
public class SlackMessageSender { /* implements MessageSender<SlackMessageDTO> {*/ public class SlackMessageSender implements MessageSender<SlackMessageDTO> {
/*
private static final String SLACK_API_URL = "https://slack.com/api"; private static final String SLACK_API_URL = "https://slack.com/api";
private final WebClient webClient; private final WebClient webClient;
@@ -23,6 +33,7 @@ public class SlackMessageSender { /* implements MessageSender<SlackMessageDTO> {
.exchangeStrategies(WebFluxExchangeStragiesFactory.ofDefault()) .exchangeStrategies(WebFluxExchangeStragiesFactory.ofDefault())
.defaultHeader(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_UTF8_VALUE) .defaultHeader(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_UTF8_VALUE)
.defaultHeader(HttpHeaders.ACCEPT_CHARSET, "UTF-8") .defaultHeader(HttpHeaders.ACCEPT_CHARSET, "UTF-8")
.filter(WebClientFilterFactory.logRequest())
.filter(WebClientFilterFactory.logResponse()) .filter(WebClientFilterFactory.logResponse())
.build(); .build();
} }
@@ -37,9 +48,8 @@ public class SlackMessageSender { /* implements MessageSender<SlackMessageDTO> {
.onErrorResume(WebClientRequestException.class, t -> { .onErrorResume(WebClientRequestException.class, t -> {
log.info("Exception occured, ignoring. : {}", t.getClass().getSimpleName()); log.info("Exception occured, ignoring. : {}", t.getClass().getSimpleName());
return Mono.empty(); return Mono.empty();
}); })
.doOnNext(e -> log.info("[sendMessage] {}", e));
} }
*/
} }

View File

@@ -0,0 +1,10 @@
spring:
config:
activate:
on-profile: development
import:
- "configserver:http://192.168.0.100:11080"
# import: optional:configserver:http://localhost:11080 # can be start up even config server was not found.
server:
port: 20082

View File

@@ -3,16 +3,8 @@ spring:
activate: activate:
on-profile: local on-profile: local
import: import:
- classpath:/datasource/local.yml - "configserver:http://localhost:20085"
- classpath:/slack/local.yml
server:
port: 20080
# import: optional:configserver:http://localhost:11080 # can be start up even config server was not found. # import: optional:configserver:http://localhost:11080 # can be start up even config server was not found.
feign: server:
client: port: 20082
config:
default:
loggerLevel: FULL

View File

@@ -0,0 +1,6 @@
spring:
config:
activate:
on-profile: production
import:
- "configserver:http://ppn-config-server:20080"

View File

@@ -0,0 +1,25 @@
spring:
application:
name: ppn-sender
main:
allow-bean-definition-overriding: true
profiles:
active: ${SPRING_ACTIVE_PROFILE:local}
group:
local: "local,slackapi-local,webclient-local"
development: "development,slackapi-development,webclient-development"
production: "production,slackapi-production,webclient-production"
freemarker:
enabled: false
server:
port: 20080
error:
whitelabel:
enabled: false
management:
endpoints:
web:
exposure:
include: refresh,health

View File

@@ -0,0 +1,15 @@
<?xml version="1.0" encoding="UTF-8"?>
<configuration>
<springProfile name="local">
<include resource="logback/logback-development.xml" />
<logger name="org.apache.kafka" level="INFO" />
</springProfile>
<springProperty name="DEFAULT_LEVEL_CONFIG" source="log.defaultLevel" />
<springProfile name="development">
<include resource="logback/logback-development.xml" />
<logger name="org.apache.kafka" level="INFO" />
</springProfile>
<springProfile name="production">
<include resource="logback/logback-production.xml" />
</springProfile>
</configuration>

View File

@@ -0,0 +1,23 @@
<included>
<property name="FILE_LOG_PATTERN"
value="%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{45}:%L - %msg%n" />
<property name="LOG_FILE_BASE" value="lcp-benefit-benefit-api" />
<appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${DIRECTORY}/${LOG_FILE_BASE}_log</file>
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<fileNamePattern>${DIRECTORY}/${LOG_FILE_BASE}_log.%d{yyyyMMdd}.%i</fileNamePattern>
<timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
<maxFileSize>1000MB</maxFileSize>
</timeBasedFileNamingAndTriggeringPolicy>
<maxHistory>60</maxHistory>
</rollingPolicy>
<encoder>
<pattern>${FILE_LOG_PATTERN}</pattern>
<immediateFlush>${IMMEDIATE_FLUSH}</immediateFlush>
</encoder>
</appender>
<appender name="ASYNC_FILE" class="ch.qos.logback.classic.AsyncAppender">
<queueSize>1024</queueSize>
<appender-ref ref="FILE" />
</appender>
</included>

View File

@@ -0,0 +1,19 @@
<?xml version="1.0" encoding="UTF-8"?>
<included>
<!-- =========== property BETA ========= -->
<property name="DEFAULT_LEVEL" value="${DEFAULT_LEVEL_CONFIG:-INFO}"/>
<!--file-->
<property name="DIRECTORY" value="/home1/www/logs/supervisor"/>
<property name="IMMEDIATE_FLUSH" value="true"/>
<!--nelo2-->
<property name="NELO2_LEVEL" value="WARN"/>
<!-- =========== include appender =========== -->
<include resource="org/springframework/boot/logging/logback/defaults.xml"/>
<include resource="org/springframework/boot/logging/logback/console-appender.xml"/>
<include resource="logback/component/logback-nelo2.xml"/>
<include resource="logback/component/logback-datachain.xml"/>
<!-- =========== root logger ============== -->
<root level="${DEFAULT_LEVEL}">
<appender-ref ref="CONSOLE"/>
</root>
</included>

View File

@@ -0,0 +1,19 @@
<?xml version="1.0" encoding="UTF-8"?>
<included>
<!-- =========== property RELEASE ========= -->
<property name="DEFAULT_LEVEL" value="${DEFAULT_LEVEL_CONFIG:-INFO}"/>
<!--file-->
<property name="DIRECTORY" value="/home1/www/logs/supervisor"/>
<property name="IMMEDIATE_FLUSH" value="true"/>
<!--nelo2-->
<property name="NELO2_LEVEL" value="WARN"/>
<!-- =========== include appender =========== -->
<include resource="org/springframework/boot/logging/logback/defaults.xml"/>
<include resource="org/springframework/boot/logging/logback/console-appender.xml"/>
<include resource="logback/component/logback-nelo2.xml"/>
<include resource="logback/component/logback-datachain.xml"/>
<!-- =========== root logger ============== -->
<root level="${DEFAULT_LEVEL}">
<appender-ref ref="CONSOLE"/>
</root>
</included>

View File

@@ -1,14 +1,14 @@
rootProject.name = 'PpomppuNotifier' rootProject.name = 'PpomppuNotifier'
//include 'reader' include 'receiver'
//project(':reader').projectDir = "$rootDir/reader" as File project(':receiver').projectDir = "$rootDir/receiver" as File
//include 'processor'
//project(':processor').projectDir = "$rootDir/processor" as File include 'processor'
//include 'writer' project(':processor').projectDir = "$rootDir/processor" as File
//project(':writer').projectDir = "$rootDir/writer" as File
include 'sender'
project(':sender').projectDir = "$rootDir/sender" as File
include 'shopping-crawler'
project(':shopping-crawler').projectDir = "$rootDir/shopping-crawler" as File
include 'support' include 'support'
project(':support').projectDir = "$rootDir/support" as File project(':support').projectDir = "$rootDir/support" as File

Binary file not shown.

View File

@@ -1,42 +0,0 @@
dependencies {
developmentOnly 'org.springframework.boot:spring-boot-devtools'
runtimeOnly 'com.h2database:h2'
runtimeOnly 'com.mysql:mysql-connector-j'
compileOnly 'org.projectlombok:lombok'
implementation project(':support')
// https://projectreactor.io/docs/core/release/reference/#debug-activate
implementation("org.springframework.boot:spring-boot-starter-web") {
exclude group: "org.springframework.boot", module: "spring-boot-starter-tomcat"
}
implementation("org.springframework.boot:spring-boot-starter-undertow") {
exclude group: "io.undertow", module: "undertow-websockets-jsr"
}
implementation 'org.springframework.boot:spring-boot-starter-data-jpa'
implementation 'org.springframework.boot:spring-boot-configuration-processor'
implementation 'org.springframework.cloud:spring-cloud-starter-config'
implementation 'org.springframework.boot:spring-boot-starter-actuator'
implementation 'com.rometools:rome:2.1.0'
implementation 'org.jsoup:jsoup:1.17.2'
implementation 'com.h2database:h2:2.2.224'
implementation "org.springframework.cloud:spring-cloud-starter-openfeign"
implementation "io.github.openfeign:feign-hc5"
implementation 'org.ahocorasick:ahocorasick:0.6.3'
annotationProcessor 'org.springframework.boot:spring-boot-configuration-processor'
annotationProcessor 'org.projectlombok:lombok'
testImplementation 'org.springframework.boot:spring-boot-starter-test'
testImplementation('org.assertj:assertj-core:3.25.3')
testImplementation("org.jeasy:easy-random-core:5.0.0")
testCompileOnly 'org.projectlombok:lombok'
testAnnotationProcessor('org.projectlombok:lombok')
}
test {
useJUnitPlatform()
testLogging {
events "passed", "skipped", "failed"
}
}

View File

@@ -1,19 +0,0 @@
package com.myoa.engineering.crawl.shopping;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.cloud.openfeign.EnableFeignClients;
import org.springframework.scheduling.annotation.EnableScheduling;
@EnableScheduling
@EnableFeignClients
@EnableConfigurationProperties
@SpringBootApplication
public class ShoppingCrawlerApplication {
public static void main(String[] args) {
SpringApplication.run(ShoppingCrawlerApplication.class, args);
}
}

Some files were not shown because too many files have changed in this diff Show More