MyoaEngineering
/
ShoppingCrawler
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request '[PPN-210926] Persist feed articles' (#8) from feature/PPN-210926-5 into develop 

Reviewed-on: OutworldDestroyer/PpomppuNotifier#8
This commit is contained in:
nthfuncx 2021-09-26 22:25:36 +09:00
parent 398a94ce3f b03fe85ad5
commit 22ac349d26
34 changed files with 747 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
PpomppuNotifier_DB.sql Normal file
View File

@ -0,0 +1,46 @@
create table "ppomppu_article"
(
"id" bigint generated by default as identity,
"article_id" bigint,
"article_url" varchar(255),
"board_name" integer,
"hit" integer,
"recommended" integer,
"registered_at" timestamp,
"title" varchar(255),
primary key ("id")
)
create table "ppomppu_board_feed_status"
(
"id" bigint generated by default as identity,
"board_name" integer,
"latest_parsed_article_id" bigint,
"updated_at" timestamp,
primary key ("id")
)
create table "published_history"
(
"id" bigint generated by default as identity,
"board_name_list" varchar(255),
"published_at" timestamp,
"user_id" bigint,
primary key ("id")
)
create table "subscribed_board"
(
"id" bigint generated by default as identity,
"board_name" integer,
"user_id" bigint,
primary key ("id")
)
create table "subscribed_user"
(
"id" bigint generated by default as identity,
"registered_at" timestamp,
"user_id" bigint,
primary key ("id")
)

54
PpomppuNotifier_ERD.puml Normal file
View File

@ -0,0 +1,54 @@
@startuml
'https://plantuml.com/sequence-diagram
class SubscribedUser {
- id
+ user_id
+ registered_at
+ created_at
+ modified_at
}
class SubscribedBoard {
- id
- user_id
+ board_name
+ created_at
+ modified_at
}
class PublishedHisotry {
- id
+ user_id
+ board_name_list
+ published_at
+ created_at
+ modified_at
}
class PpomppuArticle {
- id
+ article_id
+ board_name
+ article_url
+ title
+ recommended
+ hit
+ registered_at
+ created_at
+ modified_at
}
class PpomppuBoardFeedStatus {
- id
+ board_name
+ latest_parsed_article_id
+ updated_at
+ created_at
+ modified_at
}
SubscribedUser --o{ SubscribedBoard
SubscribedUser --o{ PublishedHisotry
@enduml

2
processor/build.gradle
View File

@ -8,8 +8,10 @@ dependencies {
// https://projectreactor.io/docs/core/release/reference/#debug-activate
implementation 'org.springframework.boot:spring-boot-starter-webflux'
implementation 'org.springframework.boot:spring-boot-starter-data-jpa'
implementation 'org.springframework.boot:spring-boot-starter-jdbc'
implementation 'com.rometools:rome:1.16.0'
implementation 'org.jsoup:jsoup:1.14.2'
implementation 'com.h2database:h2:1.4.200'
annotationProcessor 'org.springframework.boot:spring-boot-configuration-processor'
annotationProcessor 'org.projectlombok:lombok'

35
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/H2ConsoleConfiguration.java Normal file
View File

@ -0,0 +1,35 @@
package com.myoa.engineering.crawl.ppomppu.processor.configuration;
import java.sql.SQLException;
import lombok.extern.slf4j.Slf4j;
import org.h2.tools.Server;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Profile;
import org.springframework.context.event.ContextClosedEvent;
import org.springframework.context.event.ContextRefreshedEvent;
import org.springframework.context.event.EventListener;
@Slf4j
@Profile("development")
@Configuration
public class H2ConsoleConfiguration {
private Server webServer;
@Value("${spring.h2.console.port}")
private String port;
@EventListener(ContextRefreshedEvent.class)
public void start() throws SQLException {
log.info("starting h2 console");
this.webServer = Server.createWebServer("-webPort", port, "-tcpAllowOthers").start();
}
@EventListener(ContextClosedEvent.class)
public void stop() {
log.info("stopping h2 console");
this.webServer.stop(); ;
}
}

124
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/PpomppuDatasourceConfiguration.java Normal file
View File

@ -0,0 +1,124 @@
package com.myoa.engineering.crawl.ppomppu.processor.configuration;
import com.myoa.engineering.crawl.ppomppu.processor.configuration.properties.DatasourceProperties;
import com.myoa.engineering.crawl.ppomppu.processor.configuration.properties.DatasourceProperties.DataSourcePropertiesUnit;
import com.myoa.engineering.crawl.ppomppu.processor.configuration.properties.HibernateProperties;
import com.myoa.engineering.crawl.ppomppu.processor.configuration.properties.HikariProperties;
import com.myoa.engineering.crawl.ppomppu.processor.domain.BaseScanDomain;
import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository.BaseScanRepository;
import com.zaxxer.hikari.HikariConfig;
import com.zaxxer.hikari.HikariDataSource;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
import javax.persistence.EntityManagerFactory;
import javax.sql.DataSource;
import lombok.NonNull;
import org.hibernate.cfg.AvailableSettings;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.boot.orm.jpa.EntityManagerFactoryBuilder;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.data.jpa.repository.config.EnableJpaRepositories;
import org.springframework.orm.jpa.JpaTransactionManager;
import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean;
import org.springframework.transaction.PlatformTransactionManager;
@Configuration
@EnableJpaRepositories(basePackageClasses = BaseScanRepository.class,
entityManagerFactoryRef = "ppomppuNotifierProcessorEntityManagerFactory",
transactionManagerRef = "ppomppuNotifierProcessorTransactionManager"
)
public class PpomppuDatasourceConfiguration {
private static final String DATA_SOURCE_UNIT_NAME = "ppomppu";
private final DatasourceProperties dataSourceProeprties;
private final HikariProperties hikariProperties;
private final HibernateProperties hibernateProperties;
public PpomppuDatasourceConfiguration(DatasourceProperties dataSourceProeprties,
HikariProperties hikariProperties,
HibernateProperties hibernateProperties) {
this.dataSourceProeprties = dataSourceProeprties;
this.hikariProperties = hikariProperties;
this.hibernateProperties = hibernateProperties;
}
@Bean(name = "ppomppuNotifierProcessorDataSource")
public DataSource dataSource() {
DataSourcePropertiesUnit dataSourcePropertiesUnit = dataSourceProeprties.find(DATA_SOURCE_UNIT_NAME);
final HikariConfig hikariConfig = new HikariConfig();
hikariConfig.setJdbcUrl(dataSourcePropertiesUnit.toCompletedJdbcUrl());
hikariConfig.setUsername("sa");
hikariConfig.setPassword("sa");
hikariConfig.setAutoCommit(hikariProperties.getAutoCommit());
hikariConfig.setMaximumPoolSize(hikariProperties.getMaximumPoolSize());
hikariConfig.setMinimumIdle(hikariProperties.getMinimumIdle());
if (hikariProperties.getMaximumPoolSize() > hikariProperties.getMinimumIdle()) {
hikariConfig.setIdleTimeout(hikariProperties.getIdleTimeout());
}
hikariConfig.setValidationTimeout(hikariProperties.getValidationTimeout());
hikariConfig.setConnectionTimeout(hikariProperties.getConnectionTimeout());
hikariConfig.setMaxLifetime(hikariProperties.getMaxLifetime());
final DataSource dataSource = new HikariDataSource(hikariConfig);
return dataSource;
}
@Bean("ppomppuNotifierProcessorEntityManagerFactory")
public LocalContainerEntityManagerFactoryBean entityManagerFactory(
EntityManagerFactoryBuilder builder,
@Qualifier("ppomppuNotifierProcessorDataSource") DataSource dataSource) {
return builder.dataSource(dataSource)
.packages(BaseScanDomain.class)
.properties(getPropsMap(DATA_SOURCE_UNIT_NAME))
.build();
}
@Bean("ppomppuNotifierProcessorTransactionManager")
public PlatformTransactionManager transactionManager(
@Qualifier("ppomppuNotifierProcessorEntityManagerFactory") EntityManagerFactory entityManagerFactory) {
return new JpaTransactionManager(entityManagerFactory);
}
public static Properties getProps(@NonNull HibernateProperties.HibernatePropertiesUnit hibernateProperties) {
Properties properties = new Properties();
properties.put(AvailableSettings.DIALECT, hibernateProperties.getDialect());
properties.put(AvailableSettings.FORMAT_SQL, hibernateProperties.getFormatSql());
properties.put(AvailableSettings.SHOW_SQL, hibernateProperties.getShowSql());
properties.put(AvailableSettings.HBM2DDL_AUTO, hibernateProperties.getHbm2ddlAuto());
properties.put(AvailableSettings.CONNECTION_PROVIDER_DISABLES_AUTOCOMMIT,
hibernateProperties.getDisableAutoCommit());
properties.put(AvailableSettings.IMPLICIT_NAMING_STRATEGY,
"org.springframework.boot.orm.jpa.hibernate.SpringImplicitNamingStrategy");
properties.put(AvailableSettings.PHYSICAL_NAMING_STRATEGY,
"org.springframework.boot.orm.jpa.hibernate.SpringPhysicalNamingStrategy");
properties.put(AvailableSettings.GENERATE_STATISTICS, "false");
properties.put(AvailableSettings.GLOBALLY_QUOTED_IDENTIFIERS, "true");
properties.put(AvailableSettings.GLOBALLY_QUOTED_IDENTIFIERS_SKIP_COLUMN_DEFINITIONS, "true");
properties.put(AvailableSettings.STATEMENT_BATCH_SIZE, "20");
properties.put(AvailableSettings.ORDER_INSERTS, "true");
properties.put(AvailableSettings.ORDER_UPDATES, "true");
properties.put(AvailableSettings.BATCH_VERSIONED_DATA, "true");
properties.put(AvailableSettings.USE_NEW_ID_GENERATOR_MAPPINGS, "false");
return properties;
}
public Map<String, String> getPropsMap(@NonNull String unitName) {
return convertPropertiestoMaps(getProps(hibernateProperties.find(unitName)));
}
public Map<String, String> convertPropertiestoMaps(Properties properties) {
Map<String, String> propertiesMap = new HashMap<>();
for (Enumeration<?> e = properties.propertyNames(); e.hasMoreElements(); ) {
String key = (String) e.nextElement();
propertiesMap.put(key, properties.getProperty(key));
}
return propertiesMap;
}
}

44
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/properties/DatasourceProperties.java Normal file
View File

@ -0,0 +1,44 @@
package com.myoa.engineering.crawl.ppomppu.processor.configuration.properties;
import com.myoa.engineering.crawl.ppomppu.support.util.ObjectUtil;
import java.util.List;
import lombok.Getter;
import lombok.Setter;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;
@Component
@Setter
@Getter
@ConfigurationProperties(prefix = "datasource")
public class DatasourceProperties {
private List<DataSourcePropertiesUnit> units;
@Getter
@Setter
public static class DataSourcePropertiesUnit {
private String unitName;
private String schemaName;
private String connectionParameters;
private String dbConnectionUrl;
private Boolean simpleConnectionUrl;
public String toCompletedJdbcUrl() {
if (ObjectUtil.isEmpty(simpleConnectionUrl) || simpleConnectionUrl == false) {
return String.format("%s/%s?%s", dbConnectionUrl, schemaName, connectionParameters);
}
return dbConnectionUrl;
}
}
public DataSourcePropertiesUnit find(String unitName) {
return units.stream()
.filter(e -> e.getUnitName().equals(unitName))
.findFirst()
.orElseThrow(
() -> new IllegalArgumentException(this.getClass().getName() + ": unitName Not found. " + unitName));
}
}

38
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/properties/HibernateProperties.java Normal file
View File

@ -0,0 +1,38 @@
package com.myoa.engineering.crawl.ppomppu.processor.configuration.properties;
import java.util.List;
import lombok.Getter;
import lombok.Setter;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;
@Component
@Setter
@Getter
@ConfigurationProperties(prefix = "hibernate")
public class HibernateProperties {
private List<HibernatePropertiesUnit> units;
@Getter
@Setter
public static class HibernatePropertiesUnit {
private String unitName;
private String dialect;
private String formatSql;
private String showSql;
private String hbm2ddlAuto;
private String disableAutoCommit;
}
public HibernatePropertiesUnit find(String unitName) {
return units.stream()
.filter(x -> x.getUnitName().equals(unitName))
.findFirst()
.orElseThrow(
() -> new IllegalArgumentException(this.getClass().getName() + ": unitName Not found. " + unitName));
}
}

22
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/properties/HikariProperties.java Normal file
View File

@ -0,0 +1,22 @@
package com.myoa.engineering.crawl.ppomppu.processor.configuration.properties;
import lombok.Getter;
import lombok.Setter;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;
@Component
@Setter
@Getter
@ConfigurationProperties(prefix = "spring.datasource.hikari")
public class HikariProperties {
private Integer minimumIdle;
private Integer maximumPoolSize;
private Integer idleTimeout;
private Integer validationTimeout;
private Integer connectionTimeout;
private Integer maxLifetime;
private Boolean autoCommit;
}

14
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/controller/CrawlAPIController.java
View File

@ -1,16 +1,17 @@
package com.myoa.engineering.crawl.ppomppu.processor.controller;
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
import com.myoa.engineering.crawl.ppomppu.support.dto.APIResponse;
import com.myoa.engineering.crawl.ppomppu.processor.dto.FeedParsedResult;
import com.myoa.engineering.crawl.ppomppu.processor.service.PpomppuArticleService;
import com.myoa.engineering.crawl.ppomppu.processor.service.PpomppuFeedService;
import com.myoa.engineering.crawl.ppomppu.support.dto.APIResponse;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import java.util.List;
import lombok.extern.slf4j.Slf4j;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
/**
@ -25,16 +26,21 @@ import reactor.core.publisher.Mono;
public class CrawlAPIController {
private final PpomppuFeedService ppomppuRSSFeedService;
private final PpomppuArticleService ppomppuArticleService;
public CrawlAPIController(PpomppuFeedService ppomppuRSSFeedService) {
public CrawlAPIController(PpomppuFeedService ppomppuRSSFeedService,
PpomppuArticleService ppomppuArticleService) {
this.ppomppuRSSFeedService = ppomppuRSSFeedService;
this.ppomppuArticleService = ppomppuArticleService;
}
@PostMapping("/boards/{boardName}")
public Mono<APIResponse<FeedParsedResult>> crawlBoard(@PathVariable("boardName") PpomppuBoardName boardName) {
log.info("got request... {}", boardName);
FeedParsedResult result = FeedParsedResult.of(boardName);
Flux<PpomppuArticle> articles = ppomppuRSSFeedService.getArticles(boardName);
Mono<List<PpomppuArticle>> articles = ppomppuRSSFeedService.getArticles(boardName)
.doOnNext(e -> ppomppuArticleService.filterOnlyNewArticles(boardName, e))
.doOnNext(e -> ppomppuArticleService.save(boardName, e));
return articles.then(Mono.just(APIResponse.success(result.done())));
}

36
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/PpomppuArticle.java
View File

@ -1,22 +1,23 @@
package com.myoa.engineering.crawl.ppomppu.processor.domain;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import java.time.Instant;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.EnumType;
import javax.persistence.Enumerated;
import javax.persistence.GeneratedValue;
import javax.persistence.GenerationType;
import javax.persistence.Id;
import javax.persistence.Table;
import lombok.Builder;
import lombok.Getter;
import lombok.NoArgsConstructor;
/**
* PpomppuArticle
*
* @author Shin Woo-jin (woozu.shin@kakaoent.com)
* @since 2021-09-08
*/
@Getter
@NoArgsConstructor
@Entity
@Table(name = "ppomppu_article")
public class PpomppuArticle extends Auditable {
@Id
@ -26,6 +27,10 @@ public class PpomppuArticle extends Auditable {
@Column
private Long articleId;
@Column
@Enumerated(EnumType.STRING)
private PpomppuBoardName boardName;
@Column
private String articleUrl;
@ -42,10 +47,11 @@ public class PpomppuArticle extends Auditable {
private Instant registeredAt;
@Builder
public PpomppuArticle(Long id, Long articleId, String articleUrl, String title,
Integer recommended, Integer hit, Instant registeredAt) {
public PpomppuArticle(Long id, Long articleId, PpomppuBoardName boardName, String articleUrl,
String title, Integer recommended, Integer hit, Instant registeredAt) {
this.id = id;
this.articleId = articleId;
this.boardName = boardName;
this.articleUrl = articleUrl;
this.title = title;
this.recommended = recommended;
@ -53,16 +59,8 @@ public class PpomppuArticle extends Auditable {
this.registeredAt = registeredAt;
}
@Override
public String toString() {
return "PpomppuArticle{" +
"id=" + id +
", articleId=" + articleId +
", articleUrl='" + articleUrl + '\'' +
", title='" + title + '\'' +
", hit=" + hit +
", recommended=" + recommended +
", registeredAt=" + registeredAt +
'}';
public PpomppuArticle updateBoardName(PpomppuBoardName boardName) {
this.boardName = boardName;
return this;
}
}

57
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/PpomppuBoardFeedStatus.java Normal file
View File

@ -0,0 +1,57 @@
package com.myoa.engineering.crawl.ppomppu.processor.domain;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import java.time.Instant;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.EnumType;
import javax.persistence.Enumerated;
import javax.persistence.GeneratedValue;
import javax.persistence.GenerationType;
import javax.persistence.Id;
import javax.persistence.Table;
import lombok.Builder;
import lombok.Getter;
import lombok.NoArgsConstructor;
@Getter
@NoArgsConstructor
@Entity
@Table(name = "ppomppu_board_feed_status")
public class PpomppuBoardFeedStatus extends Auditable {
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
private Long id;
@Column
private Long latestParsedArticleId;
@Column
@Enumerated(EnumType.STRING)
private PpomppuBoardName boardName;
@Column
private Instant updatedAt;
public static PpomppuBoardFeedStatus of(PpomppuBoardName boardName, Long latestArticleId) {
return PpomppuBoardFeedStatus.builder()
.boardName(boardName)
.latestParsedArticleId(latestArticleId)
.updatedAt(Instant.now())
.build();
}
public void updateArticleId(Long latestArticleId) {
this.updatedAt = Instant.now();
this.latestParsedArticleId = latestArticleId;
}
@Builder
public PpomppuBoardFeedStatus(Long id, Long latestParsedArticleId, PpomppuBoardName boardName, Instant updatedAt) {
this.id = id;
this.latestParsedArticleId = latestParsedArticleId;
this.boardName = boardName;
this.updatedAt = updatedAt;
}
}

32
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/PublishedHistory.java Normal file
View File

@ -0,0 +1,32 @@
package com.myoa.engineering.crawl.ppomppu.processor.domain;
import java.time.Instant;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.GeneratedValue;
import javax.persistence.GenerationType;
import javax.persistence.Id;
import javax.persistence.Table;
import lombok.Getter;
import lombok.NoArgsConstructor;
@Getter
@NoArgsConstructor
@Entity
@Table(name = "published_history")
public class PublishedHistory extends Auditable {
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
private Long id;
@Column
private Long userId;
@Column
private String boardNameList;
@Column
private Instant publishedAt;
}

32
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/SubscribedBoard.java Normal file
View File

@ -0,0 +1,32 @@
package com.myoa.engineering.crawl.ppomppu.processor.domain;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.EnumType;
import javax.persistence.Enumerated;
import javax.persistence.GeneratedValue;
import javax.persistence.GenerationType;
import javax.persistence.Id;
import javax.persistence.Table;
import lombok.Getter;
import lombok.NoArgsConstructor;
@Getter
@NoArgsConstructor
@Entity
@Table(name = "subscribed_board")
public class SubscribedBoard extends Auditable {
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
private Long id;
@Column
private Long userId;
@Column
@Enumerated(EnumType.STRING)
private PpomppuBoardName boardName;
}

29
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/SubscribedUser.java Normal file
View File

@ -0,0 +1,29 @@
package com.myoa.engineering.crawl.ppomppu.processor.domain;
import java.time.Instant;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.GeneratedValue;
import javax.persistence.GenerationType;
import javax.persistence.Id;
import javax.persistence.Table;
import lombok.Getter;
import lombok.NoArgsConstructor;
@Getter
@NoArgsConstructor
@Entity
@Table(name = "subscribed_user")
public class SubscribedUser extends Auditable{
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
private Long id;
@Column
private Long userId;
@Column
private Instant registeredAt;
}

3
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/PpomppuArticleTransformer.java
View File

@ -1,6 +1,7 @@
package com.myoa.engineering.crawl.ppomppu.processor.dto;
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import java.time.Instant;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
@ -47,7 +48,7 @@ public final class PpomppuArticleTransformer {
}
public static String toArticleUrl(Element td) {
return td.getElementsByTag("a").attr("href");
return PpomppuBoardName.ofViewPageUrl(td.getElementsByTag("a").attr("href"));
}
public static Integer toRecommended(Element td) {

7
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/client/PpomppuBoardFeedRetriever.java
View File

@ -2,6 +2,7 @@ package com.myoa.engineering.crawl.ppomppu.processor.infrastructure.client;
import com.myoa.engineering.crawl.ppomppu.processor.configuration.factory.WebClientFilterFactory;
import com.myoa.engineering.crawl.ppomppu.processor.configuration.factory.WebFluxExchangeStragiesFactory;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import org.springframework.web.reactive.function.client.WebClient;
@ -19,12 +20,10 @@ import reactor.core.scheduler.Schedulers;
@Component
public class PpomppuBoardFeedRetriever {
private static final String PPOMPPU_URL = "https://www.ppomppu.co.kr/";
private final WebClient webClient;
public PpomppuBoardFeedRetriever(WebClient.Builder webClientBuilder) {
this.webClient = webClientBuilder.baseUrl(PPOMPPU_URL)
this.webClient = webClientBuilder.baseUrl(PpomppuBoardName.PPOMPPU_URL)
.exchangeStrategies(WebFluxExchangeStragiesFactory.ofTextHtml())
.filter(WebClientFilterFactory.logRequest())
.filter(WebClientFilterFactory.logResponse())
@ -33,7 +32,7 @@ public class PpomppuBoardFeedRetriever {
public Mono<String> getHtml(String uri) {
return webClient.get()
.uri("/zboard/zboard.php?id=ppomppu")
.uri(uri)
.exchangeToMono(e -> e.bodyToMono(String.class))
.publishOn(Schedulers.boundedElastic())
.onErrorResume(WebClientRequestException.class, t -> {

4
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/repository/BaseScanRepository.java Normal file
View File

@ -0,0 +1,4 @@
package com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository;
public interface BaseScanRepository {
}

10
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/repository/PpomppuArticleRepository.java Normal file
View File

@ -0,0 +1,10 @@
package com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository;
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.stereotype.Repository;
@Repository
public interface PpomppuArticleRepository extends JpaRepository<PpomppuArticle, Long> {
}

14
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/repository/PpomppuBoardFeedStatusRepository.java Normal file
View File

@ -0,0 +1,14 @@
package com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository;
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuBoardFeedStatus;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import java.util.Optional;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.stereotype.Repository;
@Repository
public interface PpomppuBoardFeedStatusRepository extends JpaRepository<PpomppuBoardFeedStatus, Long> {
Optional<PpomppuBoardFeedStatus> findByBoardName(PpomppuBoardName boardName);
}

59
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/PpomppuArticleService.java Normal file
View File

@ -0,0 +1,59 @@
package com.myoa.engineering.crawl.ppomppu.processor.service;
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuBoardFeedStatus;
import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository.PpomppuArticleRepository;
import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository.PpomppuBoardFeedStatusRepository;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import java.util.List;
import java.util.Optional;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
@Slf4j
@Service
public class PpomppuArticleService {
private final PpomppuArticleRepository ppomppuArticleRepository;
private final PpomppuBoardFeedStatusRepository ppomppuBoardFeedStatusRepository;
public PpomppuArticleService(PpomppuArticleRepository ppomppuArticleRepository,
PpomppuBoardFeedStatusRepository ppomppuBoardFeedStatusRepository) {
this.ppomppuArticleRepository = ppomppuArticleRepository;
this.ppomppuBoardFeedStatusRepository = ppomppuBoardFeedStatusRepository;
}
@Transactional(readOnly = true)
public List<PpomppuArticle> filterOnlyNewArticles(PpomppuBoardName boardName, List<PpomppuArticle> articles) {
Optional<PpomppuBoardFeedStatus> boardFeedStatus = ppomppuBoardFeedStatusRepository.findByBoardName(boardName);
Long latestArticleId = boardFeedStatus.map(PpomppuBoardFeedStatus::getLatestParsedArticleId)
.orElse(0L);
return articles.stream()
.filter(e -> e.getArticleId().compareTo(latestArticleId) > 0)
.collect(Collectors.toList());
}
@Transactional
public void save(PpomppuBoardName boardName, List<PpomppuArticle> articles) {
Long latestArticleId = articles.stream()
.map(PpomppuArticle::getArticleId)
.max(Long::compareTo)
.orElse(0L);
// save PpomppuBoardFeedStatus
Optional<PpomppuBoardFeedStatus> boardFeedStatus = ppomppuBoardFeedStatusRepository.findByBoardName(boardName);
boardFeedStatus.ifPresentOrElse(e -> {
e.updateArticleId(latestArticleId);
ppomppuBoardFeedStatusRepository.save(e);
},
() -> ppomppuBoardFeedStatusRepository.save(PpomppuBoardFeedStatus.of(boardName,
latestArticleId)));
// save real articles.
ppomppuArticleRepository.saveAll(articles);
}
}

7
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/PpomppuFeedService.java
View File

@ -4,6 +4,7 @@ import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
import com.myoa.engineering.crawl.ppomppu.processor.dto.PpomppuArticleTransformer;
import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.client.PpomppuBoardFeedRetriever;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import java.util.List;
import lombok.extern.slf4j.Slf4j;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Element;
@ -27,12 +28,14 @@ public class PpomppuFeedService {
this.ppomppuBoardFeedRetriever = ppomppuBoardFeedRetriever;
}
public Flux<PpomppuArticle> getArticles(PpomppuBoardName boardName) {
public Mono<List<PpomppuArticle>> getArticles(PpomppuBoardName boardName) {
final Mono<String> html = ppomppuBoardFeedRetriever.getHtml(boardName.getResourcePath());
final Mono<Element> tbody = extractTbodyFromHtml(html)
.doOnNext(e -> log.info("pre tbody - {}", e.html()));
return extractArticlesFromTbody(tbody).map(this::convertFromElement)
.doOnNext(e -> log.info("parsed Result: {}", e));
.map(e -> e.updateBoardName(boardName))
.doOnNext(e -> log.info("parsed Result: {}", e))
.collectList();
}
private Mono<Element> extractTbodyFromHtml(Mono<String> html) {

1
processor/src/main/resources/application-development.yml
View File

@ -4,4 +4,5 @@ spring:
on-profile: development
import:
- classpath:/development/webclient.yml
- classpath:/development/temppassword.yml
- classpath:/development/database.yml

2
processor/src/main/resources/application.yml
View File

@ -11,4 +11,4 @@ server:
port: 20081
error:
whitelabel:
enabled: false
enabled: false

0
support/src/main/resources/development/webclient.yml → processor/src/main/resources/development/webclient.yml
View File

0
support/src/main/resources/production/webclient.yml → processor/src/main/resources/production/webclient.yml
View File

1
receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/dispatch/MessageDispatcher.java
View File

@ -33,7 +33,6 @@ public class MessageDispatcher extends TelegramLongPollingBot {
Message message = update.getMessage();
MessageHandler handler = getMessageHandler(message);
log.info(message.getText());
handler.handle(message);
}

21
receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/HelloWorldMessageHandler.java Normal file
View File

@ -0,0 +1,21 @@
package com.myoa.engineering.crawl.ppomppu.receiver.handler.message;
import com.myoa.engineering.crawl.ppomppu.support.util.ObjectUtil;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import org.telegram.telegrambots.meta.api.objects.Message;
@Slf4j
@Component
public class HelloWorldMessageHandler implements MessageHandler {
@Override
public boolean isApplicable(Message message) {
return ObjectUtil.isEmpty(message);
}
@Override
public void handle(Message message) {
// skip empty event message.
}
}

5
receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/TextMessageHandler.java
View File

@ -1,17 +1,18 @@
package com.myoa.engineering.crawl.ppomppu.receiver.handler.message;
import com.myoa.engineering.crawl.ppomppu.support.util.ObjectUtil;
import org.telegram.telegrambots.meta.api.objects.Message;
/**
* TextMessageHandler
*
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-08-21
*
*/
public interface TextMessageHandler extends MessageHandler {
@Override
default boolean isApplicable(Message message) {
return message.isUserMessage() && message.hasText();
return ObjectUtil.isNotEmpty(message) && message.isUserMessage() && message.hasText();
}
}

4
receiver/src/main/resources/application-development.yml
View File

@ -3,5 +3,5 @@ spring:
activate:
on-profile: development
import:
- classpath:/webclient-development.yml
- classpath:/temppassword.yml
- classpath:/development/webclient.yml
- classpath:/development/temppassword.yml

0
receiver/src/main/resources/webclient-development.yml → receiver/src/main/resources/development/webclient.yml
View File

0
receiver/src/main/resources/webclient-production.yml → receiver/src/main/resources/production/production.yml
View File

54
support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/dto/code/PpomppuBoardName.java
View File

@ -12,34 +12,42 @@ import lombok.NoArgsConstructor;
@Getter
@NoArgsConstructor
public enum PpomppuBoardName {
PPOMPPU_DOMESTIC_ETC("zboard/zboard.php?id=ppomppu&category=1"),
PPOMPPU_DOMESTIC_COMPUTER("zboard/zboard.php?id=ppomppu&category=4"),
PPOMPPU_DOMESTIC_DIGITAL("zboard/zboard.php?id=ppomppu&category=5"),
PPOMPPU_DOMESTIC_FOOD("zboard/zboard.php?id=ppomppu&category=6"),
PPOMPPU_DOMESTIC_BOOK("zboard/zboard.php?id=ppomppu&category=8"),
PPOMPPU_DOMESTIC_APPLIANCES("zboard/zboard.php?id=ppomppu&category=9"),
PPOMPPU_DOMESTIC_PARENTING("zboard/zboard.php?id=ppomppu&category=10"),
PPOMPPU_DOMESTIC_GIFTCARD("zboard/zboard.php?id=ppomppu&category=11"),
PPOMPPU_DOMESTIC_CLOTHES("zboard/zboard.php?id=ppomppu&category=12"),
PPOMPPU_DOMESTIC_COSMETIC("zboard/zboard.php?id=ppomppu&category=13"),
PPOMPPU_DOMESTIC_OUTDOOR("zboard/zboard.php?id=ppomppu&category=15"),
PPOMPPU_OVERSEA_ETC("zboard/zboard.php?id=ppomppu4&category=1"),
PPOMPPU_OVERSEA_APPLIANCES("zboard/zboard.php?id=ppomppu4&category=7"),
PPOMPPU_OVERSEA_TVAV("zboard/zboard.php?id=ppomppu4&category=8"),
PPOMPPU_OVERSEA_COMPUTER("zboard/zboard.php?id=ppomppu4&category=3"),
PPOMPPU_OVERSEA_DIGITAL("zboard/zboard.php?id=ppomppu4&category=4"),
PPOMPPU_OVERSEA_MOBILEACCESSORY("zboard/zboard.php?id=ppomppu4&category=9"),
PPOMPPU_OVERSEA_CLOTHES("zboard/zboard.php?id=ppomppu4&category=5"),
PPOMPPU_OVERSEA_WATCH("zboard/zboard.php?id=ppomppu4&category=2"),
PPOMPPU_OVERSEA_SHOES("zboard/zboard.php?id=ppomppu4&category=11"),
PPOMPPU_OVERSEA_FOOD("zboard/zboard.php?id=ppomppu4&category=10"),
PPOMPPU_OVERSEA_PARENTING("zboard/zboard.php?id=ppomppu4&category=6"),
PPOMPPU_DOMESTIC_ETC("zboard/zboard.php?id=ppomppu&category=1", true),
PPOMPPU_DOMESTIC_COMPUTER("zboard/zboard.php?id=ppomppu&category=4", true),
PPOMPPU_DOMESTIC_DIGITAL("zboard/zboard.php?id=ppomppu&category=5", true),
PPOMPPU_DOMESTIC_FOOD("zboard/zboard.php?id=ppomppu&category=6", true),
PPOMPPU_DOMESTIC_BOOK("zboard/zboard.php?id=ppomppu&category=8", true),
PPOMPPU_DOMESTIC_APPLIANCES("zboard/zboard.php?id=ppomppu&category=9", true),
PPOMPPU_DOMESTIC_PARENTING("zboard/zboard.php?id=ppomppu&category=10", true),
PPOMPPU_DOMESTIC_GIFTCARD("zboard/zboard.php?id=ppomppu&category=11", true),
PPOMPPU_DOMESTIC_CLOTHES("zboard/zboard.php?id=ppomppu&category=12", true),
PPOMPPU_DOMESTIC_COSMETIC("zboard/zboard.php?id=ppomppu&category=13", true),
PPOMPPU_DOMESTIC_OUTDOOR("zboard/zboard.php?id=ppomppu&category=15", true),
PPOMPPU_OVERSEA_ETC("zboard/zboard.php?id=ppomppu4&category=1", true),
PPOMPPU_OVERSEA_APPLIANCES("zboard/zboard.php?id=ppomppu4&category=7", true),
PPOMPPU_OVERSEA_TVAV("zboard/zboard.php?id=ppomppu4&category=8", true),
PPOMPPU_OVERSEA_COMPUTER("zboard/zboard.php?id=ppomppu4&category=3", true),
PPOMPPU_OVERSEA_DIGITAL("zboard/zboard.php?id=ppomppu4&category=4", true),
PPOMPPU_OVERSEA_MOBILEACCESSORY("zboard/zboard.php?id=ppomppu4&category=9", true),
PPOMPPU_OVERSEA_CLOTHES("zboard/zboard.php?id=ppomppu4&category=5", true),
PPOMPPU_OVERSEA_WATCH("zboard/zboard.php?id=ppomppu4&category=2", true),
PPOMPPU_OVERSEA_SHOES("zboard/zboard.php?id=ppomppu4&category=11", true),
PPOMPPU_OVERSEA_FOOD("zboard/zboard.php?id=ppomppu4&category=10", true),
PPOMPPU_OVERSEA_PARENTING("zboard/zboard.php?id=ppomppu4&category=6", true),
;
private String resourcePath;
private boolean crawlWithDefaultTimer;
PpomppuBoardName(String boardPath) {
PpomppuBoardName(String boardPath, boolean crawlWithDefaultTimer) {
this.resourcePath = boardPath;
this.crawlWithDefaultTimer = crawlWithDefaultTimer;
}
public static final String PPOMPPU_URL = "https://www.ppomppu.co.kr/";
public static String ofViewPageUrl(String articleUrl) {
return PPOMPPU_URL + "zboard/" + articleUrl;
}
}

8
support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/util/ObjectUtil.java
View File

@ -4,4 +4,12 @@ public final class ObjectUtil {
private ObjectUtil() {}
public static boolean isEmpty(Object o) {
return o == null;
}
public static boolean isNotEmpty(Object o) {
return !isEmpty(o);
}
}

41
support/src/main/resources/development/database.yml
View File

@ -0,0 +1,41 @@
spring:
jpa:
open-in-view: false
hibernate:
ddl-auto: create
datasource:
# driver-class-name: com.mysql.cj.jdbc.Driver
driver-class-name: org.h2.Driver
url: jdbc:h2:mem:ppomppu-local;DB_CLOSE_DELAY=-1
hikari:
minimum:idle: 5
maximum-pool-size: 10
idle-timeout: 600000
validation-timeout: 5000
connection-timeout: 5000
max-lifetime: 1800000
auto-commit: false
h2:
console:
enabled: true
path: /h2
port: 20082
datasource:
init: true
units:
- unit-name: ppomppu
schema-name: ppomppu-local
db-connection-url: jdbc:h2:mem:ppomppu-local
simple-connection-url: true
hibernate:
units:
- unit-name: ppomppu
dialect: org.hibernate.dialect.H2Dialect
format-sql: true
show-sql: true
hbm2ddl-auto: create
disable-auto-commit: true