From ab4ab339f65c2701105ffc1a767ee6eddb317bda Mon Sep 17 00:00:00 2001 From: "woozu.shin" Date: Sun, 26 Sep 2021 21:20:16 +0900 Subject: [PATCH 1/2] [PPN-210926-5] Set-up database for development environment --- PpomppuNotifier_DB.sql | 46 +++++++ PpomppuNotifier_ERD.puml | 54 ++++++++ processor/build.gradle | 2 + .../configuration/H2ConsoleConfiguration.java | 35 +++++ .../PpomppuDatasourceConfiguration.java | 124 ++++++++++++++++++ .../properties/DatasourceProperties.java | 44 +++++++ .../properties/HibernateProperties.java | 38 ++++++ .../properties/HikariProperties.java | 22 ++++ .../processor/domain/PpomppuArticle.java | 33 ++--- .../domain/PpomppuBoardFeedStatus.java | 33 +++++ .../processor/domain/PublishedHistory.java | 32 +++++ .../processor/domain/SubscribedBoard.java | 29 ++++ .../processor/domain/SubscribedUser.java | 29 ++++ .../repository/BaseScanRepository.java | 4 + .../repository/PpomppuArticleRepository.java | 10 ++ .../PpomppuBoardFeedStatusRepository.java | 10 ++ .../service/PpomppuArticleService.java | 30 +++++ .../resources/application-development.yml | 1 + processor/src/main/resources/application.yml | 2 +- .../main/resources/development/webclient.yml | 0 .../main/resources/production/webclient.yml | 0 .../receiver/dispatch/MessageDispatcher.java | 1 - .../message/HelloWorldMessageHandler.java | 21 +++ .../handler/message/TextMessageHandler.java | 5 +- .../resources/application-development.yml | 4 +- .../webclient.yml} | 0 .../production.yml} | 0 .../ppomppu/support/util/ObjectUtil.java | 8 ++ .../main/resources/development/database.yml | 41 ++++++ 29 files changed, 632 insertions(+), 26 deletions(-) create mode 100644 PpomppuNotifier_DB.sql create mode 100644 PpomppuNotifier_ERD.puml create mode 100644 processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/H2ConsoleConfiguration.java create mode 100644 processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/PpomppuDatasourceConfiguration.java create mode 100644 processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/properties/DatasourceProperties.java create mode 100644 processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/properties/HibernateProperties.java create mode 100644 processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/properties/HikariProperties.java create mode 100644 processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/PpomppuBoardFeedStatus.java create mode 100644 processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/PublishedHistory.java create mode 100644 processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/SubscribedBoard.java create mode 100644 processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/SubscribedUser.java create mode 100644 processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/repository/BaseScanRepository.java create mode 100644 processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/repository/PpomppuArticleRepository.java create mode 100644 processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/repository/PpomppuBoardFeedStatusRepository.java create mode 100644 processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/PpomppuArticleService.java rename {support => processor}/src/main/resources/development/webclient.yml (100%) rename {support => processor}/src/main/resources/production/webclient.yml (100%) create mode 100644 receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/HelloWorldMessageHandler.java rename receiver/src/main/resources/{webclient-development.yml => development/webclient.yml} (100%) rename receiver/src/main/resources/{webclient-production.yml => production/production.yml} (100%) diff --git a/PpomppuNotifier_DB.sql b/PpomppuNotifier_DB.sql new file mode 100644 index 0000000..6b06b1e --- /dev/null +++ b/PpomppuNotifier_DB.sql @@ -0,0 +1,46 @@ +create table "ppomppu_article" +( + "id" bigint generated by default as identity, + "article_id" bigint, + "article_url" varchar(255), + "board_name" integer, + "hit" integer, + "recommended" integer, + "registered_at" timestamp, + "title" varchar(255), + primary key ("id") +) + +create table "ppomppu_board_feed_status" +( + "id" bigint generated by default as identity, + "board_name" integer, + "latest_parsed_article_id" bigint, + "updated_at" timestamp, + primary key ("id") +) + +create table "published_history" +( + "id" bigint generated by default as identity, + "board_name_list" varchar(255), + "published_at" timestamp, + "user_id" bigint, + primary key ("id") +) + +create table "subscribed_board" +( + "id" bigint generated by default as identity, + "board_name" integer, + "user_id" bigint, + primary key ("id") +) + +create table "subscribed_user" +( + "id" bigint generated by default as identity, + "registered_at" timestamp, + "user_id" bigint, + primary key ("id") +) \ No newline at end of file diff --git a/PpomppuNotifier_ERD.puml b/PpomppuNotifier_ERD.puml new file mode 100644 index 0000000..c8f4e8b --- /dev/null +++ b/PpomppuNotifier_ERD.puml @@ -0,0 +1,54 @@ +@startuml +'https://plantuml.com/sequence-diagram + +class SubscribedUser { +- id ++ user_id ++ registered_at ++ created_at ++ modified_at +} + +class SubscribedBoard { +- id +- user_id ++ board_name ++ created_at ++ modified_at +} + +class PublishedHisotry { +- id ++ user_id ++ board_name_list ++ published_at ++ created_at ++ modified_at +} + +class PpomppuArticle { +- id ++ article_id ++ board_name ++ article_url ++ title ++ recommended ++ hit ++ registered_at ++ created_at ++ modified_at +} + +class PpomppuBoardFeedStatus { +- id ++ board_name ++ latest_parsed_article_id ++ updated_at ++ created_at ++ modified_at +} + +SubscribedUser --o{ SubscribedBoard +SubscribedUser --o{ PublishedHisotry + +@enduml \ No newline at end of file diff --git a/processor/build.gradle b/processor/build.gradle index b38b0cd..ffc9e1d 100644 --- a/processor/build.gradle +++ b/processor/build.gradle @@ -8,8 +8,10 @@ dependencies { // https://projectreactor.io/docs/core/release/reference/#debug-activate implementation 'org.springframework.boot:spring-boot-starter-webflux' implementation 'org.springframework.boot:spring-boot-starter-data-jpa' + implementation 'org.springframework.boot:spring-boot-starter-jdbc' implementation 'com.rometools:rome:1.16.0' implementation 'org.jsoup:jsoup:1.14.2' + implementation 'com.h2database:h2:1.4.200' annotationProcessor 'org.springframework.boot:spring-boot-configuration-processor' annotationProcessor 'org.projectlombok:lombok' diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/H2ConsoleConfiguration.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/H2ConsoleConfiguration.java new file mode 100644 index 0000000..d0d1a9b --- /dev/null +++ b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/H2ConsoleConfiguration.java @@ -0,0 +1,35 @@ +package com.myoa.engineering.crawl.ppomppu.processor.configuration; + +import java.sql.SQLException; +import lombok.extern.slf4j.Slf4j; +import org.h2.tools.Server; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Profile; +import org.springframework.context.event.ContextClosedEvent; +import org.springframework.context.event.ContextRefreshedEvent; +import org.springframework.context.event.EventListener; + +@Slf4j +@Profile("development") +@Configuration +public class H2ConsoleConfiguration { + + private Server webServer; + + @Value("${spring.h2.console.port}") + private String port; + + @EventListener(ContextRefreshedEvent.class) + public void start() throws SQLException { + log.info("starting h2 console"); + this.webServer = Server.createWebServer("-webPort", port, "-tcpAllowOthers").start(); + } + + @EventListener(ContextClosedEvent.class) + public void stop() { + log.info("stopping h2 console"); + this.webServer.stop(); ; + } + +} diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/PpomppuDatasourceConfiguration.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/PpomppuDatasourceConfiguration.java new file mode 100644 index 0000000..64c0a52 --- /dev/null +++ b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/PpomppuDatasourceConfiguration.java @@ -0,0 +1,124 @@ +package com.myoa.engineering.crawl.ppomppu.processor.configuration; + +import com.myoa.engineering.crawl.ppomppu.processor.configuration.properties.DatasourceProperties; +import com.myoa.engineering.crawl.ppomppu.processor.configuration.properties.DatasourceProperties.DataSourcePropertiesUnit; +import com.myoa.engineering.crawl.ppomppu.processor.configuration.properties.HibernateProperties; +import com.myoa.engineering.crawl.ppomppu.processor.configuration.properties.HikariProperties; +import com.myoa.engineering.crawl.ppomppu.processor.domain.BaseScanDomain; +import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository.BaseScanRepository; +import com.zaxxer.hikari.HikariConfig; +import com.zaxxer.hikari.HikariDataSource; +import java.util.Enumeration; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; +import javax.persistence.EntityManagerFactory; +import javax.sql.DataSource; +import lombok.NonNull; +import org.hibernate.cfg.AvailableSettings; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.orm.jpa.EntityManagerFactoryBuilder; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.data.jpa.repository.config.EnableJpaRepositories; +import org.springframework.orm.jpa.JpaTransactionManager; +import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean; +import org.springframework.transaction.PlatformTransactionManager; + +@Configuration +@EnableJpaRepositories(basePackageClasses = BaseScanRepository.class, + entityManagerFactoryRef = "ppomppuNotifierProcessorEntityManagerFactory", + transactionManagerRef = "ppomppuNotifierProcessorTransactionManager" +) +public class PpomppuDatasourceConfiguration { + + private static final String DATA_SOURCE_UNIT_NAME = "ppomppu"; + + private final DatasourceProperties dataSourceProeprties; + private final HikariProperties hikariProperties; + private final HibernateProperties hibernateProperties; + + public PpomppuDatasourceConfiguration(DatasourceProperties dataSourceProeprties, + HikariProperties hikariProperties, + HibernateProperties hibernateProperties) { + this.dataSourceProeprties = dataSourceProeprties; + this.hikariProperties = hikariProperties; + this.hibernateProperties = hibernateProperties; + } + + @Bean(name = "ppomppuNotifierProcessorDataSource") + public DataSource dataSource() { + DataSourcePropertiesUnit dataSourcePropertiesUnit = dataSourceProeprties.find(DATA_SOURCE_UNIT_NAME); + + final HikariConfig hikariConfig = new HikariConfig(); + hikariConfig.setJdbcUrl(dataSourcePropertiesUnit.toCompletedJdbcUrl()); + hikariConfig.setUsername("sa"); + hikariConfig.setPassword("sa"); + hikariConfig.setAutoCommit(hikariProperties.getAutoCommit()); + hikariConfig.setMaximumPoolSize(hikariProperties.getMaximumPoolSize()); + hikariConfig.setMinimumIdle(hikariProperties.getMinimumIdle()); + if (hikariProperties.getMaximumPoolSize() > hikariProperties.getMinimumIdle()) { + hikariConfig.setIdleTimeout(hikariProperties.getIdleTimeout()); + } + hikariConfig.setValidationTimeout(hikariProperties.getValidationTimeout()); + hikariConfig.setConnectionTimeout(hikariProperties.getConnectionTimeout()); + hikariConfig.setMaxLifetime(hikariProperties.getMaxLifetime()); + + final DataSource dataSource = new HikariDataSource(hikariConfig); + return dataSource; + } + + @Bean("ppomppuNotifierProcessorEntityManagerFactory") + public LocalContainerEntityManagerFactoryBean entityManagerFactory( + EntityManagerFactoryBuilder builder, + @Qualifier("ppomppuNotifierProcessorDataSource") DataSource dataSource) { + return builder.dataSource(dataSource) + .packages(BaseScanDomain.class) + .properties(getPropsMap(DATA_SOURCE_UNIT_NAME)) + .build(); + } + + @Bean("ppomppuNotifierProcessorTransactionManager") + public PlatformTransactionManager transactionManager( + @Qualifier("ppomppuNotifierProcessorEntityManagerFactory") EntityManagerFactory entityManagerFactory) { + return new JpaTransactionManager(entityManagerFactory); + } + + public static Properties getProps(@NonNull HibernateProperties.HibernatePropertiesUnit hibernateProperties) { + Properties properties = new Properties(); + properties.put(AvailableSettings.DIALECT, hibernateProperties.getDialect()); + properties.put(AvailableSettings.FORMAT_SQL, hibernateProperties.getFormatSql()); + properties.put(AvailableSettings.SHOW_SQL, hibernateProperties.getShowSql()); + properties.put(AvailableSettings.HBM2DDL_AUTO, hibernateProperties.getHbm2ddlAuto()); + properties.put(AvailableSettings.CONNECTION_PROVIDER_DISABLES_AUTOCOMMIT, + hibernateProperties.getDisableAutoCommit()); + properties.put(AvailableSettings.IMPLICIT_NAMING_STRATEGY, + "org.springframework.boot.orm.jpa.hibernate.SpringImplicitNamingStrategy"); + properties.put(AvailableSettings.PHYSICAL_NAMING_STRATEGY, + "org.springframework.boot.orm.jpa.hibernate.SpringPhysicalNamingStrategy"); + properties.put(AvailableSettings.GENERATE_STATISTICS, "false"); + properties.put(AvailableSettings.GLOBALLY_QUOTED_IDENTIFIERS, "true"); + properties.put(AvailableSettings.GLOBALLY_QUOTED_IDENTIFIERS_SKIP_COLUMN_DEFINITIONS, "true"); + properties.put(AvailableSettings.STATEMENT_BATCH_SIZE, "20"); + properties.put(AvailableSettings.ORDER_INSERTS, "true"); + properties.put(AvailableSettings.ORDER_UPDATES, "true"); + properties.put(AvailableSettings.BATCH_VERSIONED_DATA, "true"); + properties.put(AvailableSettings.USE_NEW_ID_GENERATOR_MAPPINGS, "false"); + return properties; + } + + public Map getPropsMap(@NonNull String unitName) { + return convertPropertiestoMaps(getProps(hibernateProperties.find(unitName))); + } + + public Map convertPropertiestoMaps(Properties properties) { + Map propertiesMap = new HashMap<>(); + + for (Enumeration e = properties.propertyNames(); e.hasMoreElements(); ) { + String key = (String) e.nextElement(); + propertiesMap.put(key, properties.getProperty(key)); + } + return propertiesMap; + } + +} diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/properties/DatasourceProperties.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/properties/DatasourceProperties.java new file mode 100644 index 0000000..34c0ac5 --- /dev/null +++ b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/properties/DatasourceProperties.java @@ -0,0 +1,44 @@ +package com.myoa.engineering.crawl.ppomppu.processor.configuration.properties; + +import com.myoa.engineering.crawl.ppomppu.support.util.ObjectUtil; +import java.util.List; +import lombok.Getter; +import lombok.Setter; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +@Component +@Setter +@Getter +@ConfigurationProperties(prefix = "datasource") +public class DatasourceProperties { + + private List units; + + @Getter + @Setter + public static class DataSourcePropertiesUnit { + + private String unitName; + private String schemaName; + private String connectionParameters; + private String dbConnectionUrl; + private Boolean simpleConnectionUrl; + + public String toCompletedJdbcUrl() { + if (ObjectUtil.isEmpty(simpleConnectionUrl) || simpleConnectionUrl == false) { + return String.format("%s/%s?%s", dbConnectionUrl, schemaName, connectionParameters); + } + return dbConnectionUrl; + } + } + + public DataSourcePropertiesUnit find(String unitName) { + return units.stream() + .filter(e -> e.getUnitName().equals(unitName)) + .findFirst() + .orElseThrow( + () -> new IllegalArgumentException(this.getClass().getName() + ": unitName Not found. " + unitName)); + } + +} diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/properties/HibernateProperties.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/properties/HibernateProperties.java new file mode 100644 index 0000000..8de1649 --- /dev/null +++ b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/properties/HibernateProperties.java @@ -0,0 +1,38 @@ +package com.myoa.engineering.crawl.ppomppu.processor.configuration.properties; + +import java.util.List; +import lombok.Getter; +import lombok.Setter; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +@Component +@Setter +@Getter +@ConfigurationProperties(prefix = "hibernate") +public class HibernateProperties { + + private List units; + + @Getter + @Setter + public static class HibernatePropertiesUnit { + + private String unitName; + private String dialect; + private String formatSql; + private String showSql; + private String hbm2ddlAuto; + private String disableAutoCommit; + + } + + public HibernatePropertiesUnit find(String unitName) { + return units.stream() + .filter(x -> x.getUnitName().equals(unitName)) + .findFirst() + .orElseThrow( + () -> new IllegalArgumentException(this.getClass().getName() + ": unitName Not found. " + unitName)); + } + +} diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/properties/HikariProperties.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/properties/HikariProperties.java new file mode 100644 index 0000000..91a52bf --- /dev/null +++ b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/properties/HikariProperties.java @@ -0,0 +1,22 @@ +package com.myoa.engineering.crawl.ppomppu.processor.configuration.properties; + +import lombok.Getter; +import lombok.Setter; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +@Component +@Setter +@Getter +@ConfigurationProperties(prefix = "spring.datasource.hikari") +public class HikariProperties { + + private Integer minimumIdle; + private Integer maximumPoolSize; + private Integer idleTimeout; + private Integer validationTimeout; + private Integer connectionTimeout; + private Integer maxLifetime; + private Boolean autoCommit; + +} diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/PpomppuArticle.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/PpomppuArticle.java index 5c389a1..e42c085 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/PpomppuArticle.java +++ b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/PpomppuArticle.java @@ -1,22 +1,22 @@ package com.myoa.engineering.crawl.ppomppu.processor.domain; +import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName; import java.time.Instant; import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.Enumerated; import javax.persistence.GeneratedValue; import javax.persistence.GenerationType; import javax.persistence.Id; +import javax.persistence.Table; import lombok.Builder; import lombok.Getter; import lombok.NoArgsConstructor; -/** - * PpomppuArticle - * - * @author Shin Woo-jin (woozu.shin@kakaoent.com) - * @since 2021-09-08 - */ @Getter @NoArgsConstructor +@Entity +@Table(name = "ppomppu_article") public class PpomppuArticle extends Auditable { @Id @@ -26,6 +26,10 @@ public class PpomppuArticle extends Auditable { @Column private Long articleId; + @Column + @Enumerated + private PpomppuBoardName boardName; + @Column private String articleUrl; @@ -42,10 +46,11 @@ public class PpomppuArticle extends Auditable { private Instant registeredAt; @Builder - public PpomppuArticle(Long id, Long articleId, String articleUrl, String title, - Integer recommended, Integer hit, Instant registeredAt) { + public PpomppuArticle(Long id, Long articleId, PpomppuBoardName boardName, String articleUrl, + String title, Integer recommended, Integer hit, Instant registeredAt) { this.id = id; this.articleId = articleId; + this.boardName = boardName; this.articleUrl = articleUrl; this.title = title; this.recommended = recommended; @@ -53,16 +58,4 @@ public class PpomppuArticle extends Auditable { this.registeredAt = registeredAt; } - @Override - public String toString() { - return "PpomppuArticle{" + - "id=" + id + - ", articleId=" + articleId + - ", articleUrl='" + articleUrl + '\'' + - ", title='" + title + '\'' + - ", hit=" + hit + - ", recommended=" + recommended + - ", registeredAt=" + registeredAt + - '}'; - } } diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/PpomppuBoardFeedStatus.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/PpomppuBoardFeedStatus.java new file mode 100644 index 0000000..ceda0da --- /dev/null +++ b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/PpomppuBoardFeedStatus.java @@ -0,0 +1,33 @@ +package com.myoa.engineering.crawl.ppomppu.processor.domain; + +import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName; +import java.time.Instant; +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Table; +import lombok.Getter; +import lombok.NoArgsConstructor; + +@Getter +@NoArgsConstructor +@Entity +@Table(name = "ppomppu_board_feed_status") +public class PpomppuBoardFeedStatus extends Auditable { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + @Column + private Long latestParsedArticleId; + + @Column + private PpomppuBoardName boardName; + + @Column + private Instant updatedAt; + +} diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/PublishedHistory.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/PublishedHistory.java new file mode 100644 index 0000000..e52c069 --- /dev/null +++ b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/PublishedHistory.java @@ -0,0 +1,32 @@ +package com.myoa.engineering.crawl.ppomppu.processor.domain; + +import java.time.Instant; +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Table; +import lombok.Getter; +import lombok.NoArgsConstructor; + +@Getter +@NoArgsConstructor +@Entity +@Table(name = "published_history") +public class PublishedHistory extends Auditable { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + @Column + private Long userId; + + @Column + private String boardNameList; + + @Column + private Instant publishedAt; + +} diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/SubscribedBoard.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/SubscribedBoard.java new file mode 100644 index 0000000..5aa7032 --- /dev/null +++ b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/SubscribedBoard.java @@ -0,0 +1,29 @@ +package com.myoa.engineering.crawl.ppomppu.processor.domain; + +import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName; +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Table; +import lombok.Getter; +import lombok.NoArgsConstructor; + +@Getter +@NoArgsConstructor +@Entity +@Table(name = "subscribed_board") +public class SubscribedBoard extends Auditable { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + @Column + private Long userId; + + @Column + private PpomppuBoardName boardName; + +} diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/SubscribedUser.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/SubscribedUser.java new file mode 100644 index 0000000..fd89bde --- /dev/null +++ b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/SubscribedUser.java @@ -0,0 +1,29 @@ +package com.myoa.engineering.crawl.ppomppu.processor.domain; + +import java.time.Instant; +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Table; +import lombok.Getter; +import lombok.NoArgsConstructor; + +@Getter +@NoArgsConstructor +@Entity +@Table(name = "subscribed_user") +public class SubscribedUser extends Auditable{ + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + @Column + private Long userId; + + @Column + private Instant registeredAt; + +} diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/repository/BaseScanRepository.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/repository/BaseScanRepository.java new file mode 100644 index 0000000..ebbfedc --- /dev/null +++ b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/repository/BaseScanRepository.java @@ -0,0 +1,4 @@ +package com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository; + +public interface BaseScanRepository { +} diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/repository/PpomppuArticleRepository.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/repository/PpomppuArticleRepository.java new file mode 100644 index 0000000..0b727d9 --- /dev/null +++ b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/repository/PpomppuArticleRepository.java @@ -0,0 +1,10 @@ +package com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository; + +import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.stereotype.Repository; + +@Repository +public interface PpomppuArticleRepository extends JpaRepository { + +} diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/repository/PpomppuBoardFeedStatusRepository.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/repository/PpomppuBoardFeedStatusRepository.java new file mode 100644 index 0000000..cf1d84c --- /dev/null +++ b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/repository/PpomppuBoardFeedStatusRepository.java @@ -0,0 +1,10 @@ +package com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository; + +import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuBoardFeedStatus; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.stereotype.Repository; + +@Repository +public interface PpomppuBoardFeedStatusRepository extends JpaRepository { + +} diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/PpomppuArticleService.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/PpomppuArticleService.java new file mode 100644 index 0000000..4b68630 --- /dev/null +++ b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/PpomppuArticleService.java @@ -0,0 +1,30 @@ +package com.myoa.engineering.crawl.ppomppu.processor.service; + +import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle; +import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository.PpomppuArticleRepository; +import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository.PpomppuBoardFeedStatusRepository; +import java.util.List; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; + +@Slf4j +@Service +public class PpomppuArticleService { + + private final PpomppuArticleRepository ppomppuArticleRepository; + + private final PpomppuBoardFeedStatusRepository ppomppuBoardFeedStatusRepository; + + public PpomppuArticleService(PpomppuArticleRepository ppomppuArticleRepository, + PpomppuBoardFeedStatusRepository ppomppuBoardFeedStatusRepository) { + this.ppomppuArticleRepository = ppomppuArticleRepository; + this.ppomppuBoardFeedStatusRepository = ppomppuBoardFeedStatusRepository; + } + + public void save(List articles) { + // TODO get latest parsed article id + // TODO filter articles + + ppomppuArticleRepository.saveAll(articles); + } +} diff --git a/processor/src/main/resources/application-development.yml b/processor/src/main/resources/application-development.yml index 92c9127..6d98894 100644 --- a/processor/src/main/resources/application-development.yml +++ b/processor/src/main/resources/application-development.yml @@ -4,4 +4,5 @@ spring: on-profile: development import: - classpath:/development/webclient.yml + - classpath:/development/temppassword.yml - classpath:/development/database.yml \ No newline at end of file diff --git a/processor/src/main/resources/application.yml b/processor/src/main/resources/application.yml index 2ffa7a1..aa61bfb 100644 --- a/processor/src/main/resources/application.yml +++ b/processor/src/main/resources/application.yml @@ -11,4 +11,4 @@ server: port: 20081 error: whitelabel: - enabled: false \ No newline at end of file + enabled: false diff --git a/support/src/main/resources/development/webclient.yml b/processor/src/main/resources/development/webclient.yml similarity index 100% rename from support/src/main/resources/development/webclient.yml rename to processor/src/main/resources/development/webclient.yml diff --git a/support/src/main/resources/production/webclient.yml b/processor/src/main/resources/production/webclient.yml similarity index 100% rename from support/src/main/resources/production/webclient.yml rename to processor/src/main/resources/production/webclient.yml diff --git a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/dispatch/MessageDispatcher.java b/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/dispatch/MessageDispatcher.java index 30254db..22ea270 100644 --- a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/dispatch/MessageDispatcher.java +++ b/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/dispatch/MessageDispatcher.java @@ -33,7 +33,6 @@ public class MessageDispatcher extends TelegramLongPollingBot { Message message = update.getMessage(); MessageHandler handler = getMessageHandler(message); - log.info(message.getText()); handler.handle(message); } diff --git a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/HelloWorldMessageHandler.java b/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/HelloWorldMessageHandler.java new file mode 100644 index 0000000..ee5f83e --- /dev/null +++ b/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/HelloWorldMessageHandler.java @@ -0,0 +1,21 @@ +package com.myoa.engineering.crawl.ppomppu.receiver.handler.message; + +import com.myoa.engineering.crawl.ppomppu.support.util.ObjectUtil; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import org.telegram.telegrambots.meta.api.objects.Message; + +@Slf4j +@Component +public class HelloWorldMessageHandler implements MessageHandler { + + @Override + public boolean isApplicable(Message message) { + return ObjectUtil.isEmpty(message); + } + + @Override + public void handle(Message message) { + // skip empty event message. + } +} diff --git a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/TextMessageHandler.java b/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/TextMessageHandler.java index 7514838..07c185e 100644 --- a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/TextMessageHandler.java +++ b/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/TextMessageHandler.java @@ -1,17 +1,18 @@ package com.myoa.engineering.crawl.ppomppu.receiver.handler.message; +import com.myoa.engineering.crawl.ppomppu.support.util.ObjectUtil; import org.telegram.telegrambots.meta.api.objects.Message; /** * TextMessageHandler + * * @author Shin Woo-jin (woo-jin.shin@linecorp.com) * @since 2021-08-21 - * */ public interface TextMessageHandler extends MessageHandler { @Override default boolean isApplicable(Message message) { - return message.isUserMessage() && message.hasText(); + return ObjectUtil.isNotEmpty(message) && message.isUserMessage() && message.hasText(); } } diff --git a/receiver/src/main/resources/application-development.yml b/receiver/src/main/resources/application-development.yml index 79adfa1..34f30c9 100644 --- a/receiver/src/main/resources/application-development.yml +++ b/receiver/src/main/resources/application-development.yml @@ -3,5 +3,5 @@ spring: activate: on-profile: development import: - - classpath:/webclient-development.yml - - classpath:/temppassword.yml + - classpath:/development/webclient.yml + - classpath:/development/temppassword.yml diff --git a/receiver/src/main/resources/webclient-development.yml b/receiver/src/main/resources/development/webclient.yml similarity index 100% rename from receiver/src/main/resources/webclient-development.yml rename to receiver/src/main/resources/development/webclient.yml diff --git a/receiver/src/main/resources/webclient-production.yml b/receiver/src/main/resources/production/production.yml similarity index 100% rename from receiver/src/main/resources/webclient-production.yml rename to receiver/src/main/resources/production/production.yml diff --git a/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/util/ObjectUtil.java b/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/util/ObjectUtil.java index 0358a7c..e2d9863 100644 --- a/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/util/ObjectUtil.java +++ b/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/util/ObjectUtil.java @@ -4,4 +4,12 @@ public final class ObjectUtil { private ObjectUtil() {} + public static boolean isEmpty(Object o) { + return o == null; + } + + public static boolean isNotEmpty(Object o) { + return !isEmpty(o); + } + } diff --git a/support/src/main/resources/development/database.yml b/support/src/main/resources/development/database.yml index e69de29..a4af932 100644 --- a/support/src/main/resources/development/database.yml +++ b/support/src/main/resources/development/database.yml @@ -0,0 +1,41 @@ +spring: + jpa: + open-in-view: false + hibernate: + ddl-auto: create + datasource: + # driver-class-name: com.mysql.cj.jdbc.Driver + driver-class-name: org.h2.Driver + url: jdbc:h2:mem:ppomppu-local;DB_CLOSE_DELAY=-1 + hikari: + minimum:idle: 5 + maximum-pool-size: 10 + idle-timeout: 600000 + validation-timeout: 5000 + connection-timeout: 5000 + max-lifetime: 1800000 + auto-commit: false + h2: + console: + enabled: true + path: /h2 + port: 20082 + +datasource: + init: true + units: + - unit-name: ppomppu + schema-name: ppomppu-local + db-connection-url: jdbc:h2:mem:ppomppu-local + simple-connection-url: true + + +hibernate: + units: + - unit-name: ppomppu + dialect: org.hibernate.dialect.H2Dialect + format-sql: true + show-sql: true + hbm2ddl-auto: create + disable-auto-commit: true + From 86fa1cbe09093d14b364dfa2cf92265103c33ab0 Mon Sep 17 00:00:00 2001 From: "woozu.shin" Date: Sun, 26 Sep 2021 22:22:30 +0900 Subject: [PATCH 2/2] [PPN-210926-6] Persist feed articles --- .../controller/CrawlAPIController.java | 14 +++-- .../processor/domain/PpomppuArticle.java | 7 ++- .../domain/PpomppuBoardFeedStatus.java | 24 +++++++++ .../processor/domain/SubscribedBoard.java | 3 ++ .../dto/PpomppuArticleTransformer.java | 3 +- .../client/PpomppuBoardFeedRetriever.java | 7 ++- .../PpomppuBoardFeedStatusRepository.java | 4 ++ .../service/PpomppuArticleService.java | 35 ++++++++++-- .../processor/service/PpomppuFeedService.java | 7 ++- .../support/dto/code/PpomppuBoardName.java | 54 +++++++++++-------- 10 files changed, 120 insertions(+), 38 deletions(-) diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/controller/CrawlAPIController.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/controller/CrawlAPIController.java index 99a2ff3..3592278 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/controller/CrawlAPIController.java +++ b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/controller/CrawlAPIController.java @@ -1,16 +1,17 @@ package com.myoa.engineering.crawl.ppomppu.processor.controller; import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle; -import com.myoa.engineering.crawl.ppomppu.support.dto.APIResponse; import com.myoa.engineering.crawl.ppomppu.processor.dto.FeedParsedResult; +import com.myoa.engineering.crawl.ppomppu.processor.service.PpomppuArticleService; import com.myoa.engineering.crawl.ppomppu.processor.service.PpomppuFeedService; +import com.myoa.engineering.crawl.ppomppu.support.dto.APIResponse; import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName; +import java.util.List; import lombok.extern.slf4j.Slf4j; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; -import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; /** @@ -25,16 +26,21 @@ import reactor.core.publisher.Mono; public class CrawlAPIController { private final PpomppuFeedService ppomppuRSSFeedService; + private final PpomppuArticleService ppomppuArticleService; - public CrawlAPIController(PpomppuFeedService ppomppuRSSFeedService) { + public CrawlAPIController(PpomppuFeedService ppomppuRSSFeedService, + PpomppuArticleService ppomppuArticleService) { this.ppomppuRSSFeedService = ppomppuRSSFeedService; + this.ppomppuArticleService = ppomppuArticleService; } @PostMapping("/boards/{boardName}") public Mono> crawlBoard(@PathVariable("boardName") PpomppuBoardName boardName) { log.info("got request... {}", boardName); FeedParsedResult result = FeedParsedResult.of(boardName); - Flux articles = ppomppuRSSFeedService.getArticles(boardName); + Mono> articles = ppomppuRSSFeedService.getArticles(boardName) + .doOnNext(e -> ppomppuArticleService.filterOnlyNewArticles(boardName, e)) + .doOnNext(e -> ppomppuArticleService.save(boardName, e)); return articles.then(Mono.just(APIResponse.success(result.done()))); } diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/PpomppuArticle.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/PpomppuArticle.java index e42c085..9d88870 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/PpomppuArticle.java +++ b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/PpomppuArticle.java @@ -4,6 +4,7 @@ import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName; import java.time.Instant; import javax.persistence.Column; import javax.persistence.Entity; +import javax.persistence.EnumType; import javax.persistence.Enumerated; import javax.persistence.GeneratedValue; import javax.persistence.GenerationType; @@ -27,7 +28,7 @@ public class PpomppuArticle extends Auditable { private Long articleId; @Column - @Enumerated + @Enumerated(EnumType.STRING) private PpomppuBoardName boardName; @Column @@ -58,4 +59,8 @@ public class PpomppuArticle extends Auditable { this.registeredAt = registeredAt; } + public PpomppuArticle updateBoardName(PpomppuBoardName boardName) { + this.boardName = boardName; + return this; + } } diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/PpomppuBoardFeedStatus.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/PpomppuBoardFeedStatus.java index ceda0da..b751c13 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/PpomppuBoardFeedStatus.java +++ b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/PpomppuBoardFeedStatus.java @@ -4,10 +4,13 @@ import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName; import java.time.Instant; import javax.persistence.Column; import javax.persistence.Entity; +import javax.persistence.EnumType; +import javax.persistence.Enumerated; import javax.persistence.GeneratedValue; import javax.persistence.GenerationType; import javax.persistence.Id; import javax.persistence.Table; +import lombok.Builder; import lombok.Getter; import lombok.NoArgsConstructor; @@ -25,9 +28,30 @@ public class PpomppuBoardFeedStatus extends Auditable { private Long latestParsedArticleId; @Column + @Enumerated(EnumType.STRING) private PpomppuBoardName boardName; @Column private Instant updatedAt; + public static PpomppuBoardFeedStatus of(PpomppuBoardName boardName, Long latestArticleId) { + return PpomppuBoardFeedStatus.builder() + .boardName(boardName) + .latestParsedArticleId(latestArticleId) + .updatedAt(Instant.now()) + .build(); + } + + public void updateArticleId(Long latestArticleId) { + this.updatedAt = Instant.now(); + this.latestParsedArticleId = latestArticleId; + } + + @Builder + public PpomppuBoardFeedStatus(Long id, Long latestParsedArticleId, PpomppuBoardName boardName, Instant updatedAt) { + this.id = id; + this.latestParsedArticleId = latestParsedArticleId; + this.boardName = boardName; + this.updatedAt = updatedAt; + } } diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/SubscribedBoard.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/SubscribedBoard.java index 5aa7032..2eb7565 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/SubscribedBoard.java +++ b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/SubscribedBoard.java @@ -3,6 +3,8 @@ package com.myoa.engineering.crawl.ppomppu.processor.domain; import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName; import javax.persistence.Column; import javax.persistence.Entity; +import javax.persistence.EnumType; +import javax.persistence.Enumerated; import javax.persistence.GeneratedValue; import javax.persistence.GenerationType; import javax.persistence.Id; @@ -24,6 +26,7 @@ public class SubscribedBoard extends Auditable { private Long userId; @Column + @Enumerated(EnumType.STRING) private PpomppuBoardName boardName; } diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/PpomppuArticleTransformer.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/PpomppuArticleTransformer.java index ea00ee8..18bd30f 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/PpomppuArticleTransformer.java +++ b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/PpomppuArticleTransformer.java @@ -1,6 +1,7 @@ package com.myoa.engineering.crawl.ppomppu.processor.dto; import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle; +import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName; import java.time.Instant; import java.time.ZoneId; import java.time.format.DateTimeFormatter; @@ -47,7 +48,7 @@ public final class PpomppuArticleTransformer { } public static String toArticleUrl(Element td) { - return td.getElementsByTag("a").attr("href"); + return PpomppuBoardName.ofViewPageUrl(td.getElementsByTag("a").attr("href")); } public static Integer toRecommended(Element td) { diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/client/PpomppuBoardFeedRetriever.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/client/PpomppuBoardFeedRetriever.java index 075adf8..a635288 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/client/PpomppuBoardFeedRetriever.java +++ b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/client/PpomppuBoardFeedRetriever.java @@ -2,6 +2,7 @@ package com.myoa.engineering.crawl.ppomppu.processor.infrastructure.client; import com.myoa.engineering.crawl.ppomppu.processor.configuration.factory.WebClientFilterFactory; import com.myoa.engineering.crawl.ppomppu.processor.configuration.factory.WebFluxExchangeStragiesFactory; +import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; import org.springframework.web.reactive.function.client.WebClient; @@ -19,12 +20,10 @@ import reactor.core.scheduler.Schedulers; @Component public class PpomppuBoardFeedRetriever { - private static final String PPOMPPU_URL = "https://www.ppomppu.co.kr/"; - private final WebClient webClient; public PpomppuBoardFeedRetriever(WebClient.Builder webClientBuilder) { - this.webClient = webClientBuilder.baseUrl(PPOMPPU_URL) + this.webClient = webClientBuilder.baseUrl(PpomppuBoardName.PPOMPPU_URL) .exchangeStrategies(WebFluxExchangeStragiesFactory.ofTextHtml()) .filter(WebClientFilterFactory.logRequest()) .filter(WebClientFilterFactory.logResponse()) @@ -33,7 +32,7 @@ public class PpomppuBoardFeedRetriever { public Mono getHtml(String uri) { return webClient.get() - .uri("/zboard/zboard.php?id=ppomppu") + .uri(uri) .exchangeToMono(e -> e.bodyToMono(String.class)) .publishOn(Schedulers.boundedElastic()) .onErrorResume(WebClientRequestException.class, t -> { diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/repository/PpomppuBoardFeedStatusRepository.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/repository/PpomppuBoardFeedStatusRepository.java index cf1d84c..e84a077 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/repository/PpomppuBoardFeedStatusRepository.java +++ b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/repository/PpomppuBoardFeedStatusRepository.java @@ -1,10 +1,14 @@ package com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository; import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuBoardFeedStatus; +import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName; +import java.util.Optional; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Repository; @Repository public interface PpomppuBoardFeedStatusRepository extends JpaRepository { + Optional findByBoardName(PpomppuBoardName boardName); + } diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/PpomppuArticleService.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/PpomppuArticleService.java index 4b68630..632e3c3 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/PpomppuArticleService.java +++ b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/PpomppuArticleService.java @@ -1,11 +1,16 @@ package com.myoa.engineering.crawl.ppomppu.processor.service; import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle; +import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuBoardFeedStatus; import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository.PpomppuArticleRepository; import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository.PpomppuBoardFeedStatusRepository; +import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName; import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; @Slf4j @Service @@ -21,10 +26,34 @@ public class PpomppuArticleService { this.ppomppuBoardFeedStatusRepository = ppomppuBoardFeedStatusRepository; } - public void save(List articles) { - // TODO get latest parsed article id - // TODO filter articles + @Transactional(readOnly = true) + public List filterOnlyNewArticles(PpomppuBoardName boardName, List articles) { + Optional boardFeedStatus = ppomppuBoardFeedStatusRepository.findByBoardName(boardName); + Long latestArticleId = boardFeedStatus.map(PpomppuBoardFeedStatus::getLatestParsedArticleId) + .orElse(0L); + return articles.stream() + .filter(e -> e.getArticleId().compareTo(latestArticleId) > 0) + .collect(Collectors.toList()); + } + + @Transactional + public void save(PpomppuBoardName boardName, List articles) { + Long latestArticleId = articles.stream() + .map(PpomppuArticle::getArticleId) + .max(Long::compareTo) + .orElse(0L); + + // save PpomppuBoardFeedStatus + Optional boardFeedStatus = ppomppuBoardFeedStatusRepository.findByBoardName(boardName); + boardFeedStatus.ifPresentOrElse(e -> { + e.updateArticleId(latestArticleId); + ppomppuBoardFeedStatusRepository.save(e); + }, + () -> ppomppuBoardFeedStatusRepository.save(PpomppuBoardFeedStatus.of(boardName, + latestArticleId))); + + // save real articles. ppomppuArticleRepository.saveAll(articles); } } diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/PpomppuFeedService.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/PpomppuFeedService.java index 0f64392..461bd2f 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/PpomppuFeedService.java +++ b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/PpomppuFeedService.java @@ -4,6 +4,7 @@ import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle; import com.myoa.engineering.crawl.ppomppu.processor.dto.PpomppuArticleTransformer; import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.client.PpomppuBoardFeedRetriever; import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName; +import java.util.List; import lombok.extern.slf4j.Slf4j; import org.jsoup.Jsoup; import org.jsoup.nodes.Element; @@ -27,12 +28,14 @@ public class PpomppuFeedService { this.ppomppuBoardFeedRetriever = ppomppuBoardFeedRetriever; } - public Flux getArticles(PpomppuBoardName boardName) { + public Mono> getArticles(PpomppuBoardName boardName) { final Mono html = ppomppuBoardFeedRetriever.getHtml(boardName.getResourcePath()); final Mono tbody = extractTbodyFromHtml(html) .doOnNext(e -> log.info("pre tbody - {}", e.html())); return extractArticlesFromTbody(tbody).map(this::convertFromElement) - .doOnNext(e -> log.info("parsed Result: {}", e)); + .map(e -> e.updateBoardName(boardName)) + .doOnNext(e -> log.info("parsed Result: {}", e)) + .collectList(); } private Mono extractTbodyFromHtml(Mono html) { diff --git a/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/dto/code/PpomppuBoardName.java b/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/dto/code/PpomppuBoardName.java index e254e84..eca19fe 100644 --- a/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/dto/code/PpomppuBoardName.java +++ b/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/dto/code/PpomppuBoardName.java @@ -12,34 +12,42 @@ import lombok.NoArgsConstructor; @Getter @NoArgsConstructor public enum PpomppuBoardName { - PPOMPPU_DOMESTIC_ETC("zboard/zboard.php?id=ppomppu&category=1"), - PPOMPPU_DOMESTIC_COMPUTER("zboard/zboard.php?id=ppomppu&category=4"), - PPOMPPU_DOMESTIC_DIGITAL("zboard/zboard.php?id=ppomppu&category=5"), - PPOMPPU_DOMESTIC_FOOD("zboard/zboard.php?id=ppomppu&category=6"), - PPOMPPU_DOMESTIC_BOOK("zboard/zboard.php?id=ppomppu&category=8"), - PPOMPPU_DOMESTIC_APPLIANCES("zboard/zboard.php?id=ppomppu&category=9"), - PPOMPPU_DOMESTIC_PARENTING("zboard/zboard.php?id=ppomppu&category=10"), - PPOMPPU_DOMESTIC_GIFTCARD("zboard/zboard.php?id=ppomppu&category=11"), - PPOMPPU_DOMESTIC_CLOTHES("zboard/zboard.php?id=ppomppu&category=12"), - PPOMPPU_DOMESTIC_COSMETIC("zboard/zboard.php?id=ppomppu&category=13"), - PPOMPPU_DOMESTIC_OUTDOOR("zboard/zboard.php?id=ppomppu&category=15"), - PPOMPPU_OVERSEA_ETC("zboard/zboard.php?id=ppomppu4&category=1"), - PPOMPPU_OVERSEA_APPLIANCES("zboard/zboard.php?id=ppomppu4&category=7"), - PPOMPPU_OVERSEA_TVAV("zboard/zboard.php?id=ppomppu4&category=8"), - PPOMPPU_OVERSEA_COMPUTER("zboard/zboard.php?id=ppomppu4&category=3"), - PPOMPPU_OVERSEA_DIGITAL("zboard/zboard.php?id=ppomppu4&category=4"), - PPOMPPU_OVERSEA_MOBILEACCESSORY("zboard/zboard.php?id=ppomppu4&category=9"), - PPOMPPU_OVERSEA_CLOTHES("zboard/zboard.php?id=ppomppu4&category=5"), - PPOMPPU_OVERSEA_WATCH("zboard/zboard.php?id=ppomppu4&category=2"), - PPOMPPU_OVERSEA_SHOES("zboard/zboard.php?id=ppomppu4&category=11"), - PPOMPPU_OVERSEA_FOOD("zboard/zboard.php?id=ppomppu4&category=10"), - PPOMPPU_OVERSEA_PARENTING("zboard/zboard.php?id=ppomppu4&category=6"), + PPOMPPU_DOMESTIC_ETC("zboard/zboard.php?id=ppomppu&category=1", true), + PPOMPPU_DOMESTIC_COMPUTER("zboard/zboard.php?id=ppomppu&category=4", true), + PPOMPPU_DOMESTIC_DIGITAL("zboard/zboard.php?id=ppomppu&category=5", true), + PPOMPPU_DOMESTIC_FOOD("zboard/zboard.php?id=ppomppu&category=6", true), + PPOMPPU_DOMESTIC_BOOK("zboard/zboard.php?id=ppomppu&category=8", true), + PPOMPPU_DOMESTIC_APPLIANCES("zboard/zboard.php?id=ppomppu&category=9", true), + PPOMPPU_DOMESTIC_PARENTING("zboard/zboard.php?id=ppomppu&category=10", true), + PPOMPPU_DOMESTIC_GIFTCARD("zboard/zboard.php?id=ppomppu&category=11", true), + PPOMPPU_DOMESTIC_CLOTHES("zboard/zboard.php?id=ppomppu&category=12", true), + PPOMPPU_DOMESTIC_COSMETIC("zboard/zboard.php?id=ppomppu&category=13", true), + PPOMPPU_DOMESTIC_OUTDOOR("zboard/zboard.php?id=ppomppu&category=15", true), + PPOMPPU_OVERSEA_ETC("zboard/zboard.php?id=ppomppu4&category=1", true), + PPOMPPU_OVERSEA_APPLIANCES("zboard/zboard.php?id=ppomppu4&category=7", true), + PPOMPPU_OVERSEA_TVAV("zboard/zboard.php?id=ppomppu4&category=8", true), + PPOMPPU_OVERSEA_COMPUTER("zboard/zboard.php?id=ppomppu4&category=3", true), + PPOMPPU_OVERSEA_DIGITAL("zboard/zboard.php?id=ppomppu4&category=4", true), + PPOMPPU_OVERSEA_MOBILEACCESSORY("zboard/zboard.php?id=ppomppu4&category=9", true), + PPOMPPU_OVERSEA_CLOTHES("zboard/zboard.php?id=ppomppu4&category=5", true), + PPOMPPU_OVERSEA_WATCH("zboard/zboard.php?id=ppomppu4&category=2", true), + PPOMPPU_OVERSEA_SHOES("zboard/zboard.php?id=ppomppu4&category=11", true), + PPOMPPU_OVERSEA_FOOD("zboard/zboard.php?id=ppomppu4&category=10", true), + PPOMPPU_OVERSEA_PARENTING("zboard/zboard.php?id=ppomppu4&category=6", true), ; private String resourcePath; + private boolean crawlWithDefaultTimer; - PpomppuBoardName(String boardPath) { + PpomppuBoardName(String boardPath, boolean crawlWithDefaultTimer) { this.resourcePath = boardPath; + this.crawlWithDefaultTimer = crawlWithDefaultTimer; + } + + public static final String PPOMPPU_URL = "https://www.ppomppu.co.kr/"; + + public static String ofViewPageUrl(String articleUrl) { + return PPOMPPU_URL + "zboard/" + articleUrl; } }