[PPN-210926] Persist feed articles #8

Merged
nthfuncx merged 3 commits from feature/PPN-210926-5 into develop 2021-09-26 13:25:36 +00:00
29 changed files with 632 additions and 26 deletions
Showing only changes of commit ab4ab339f6 - Show all commits

46
PpomppuNotifier_DB.sql Normal file
View File

@ -0,0 +1,46 @@
create table "ppomppu_article"
(
"id" bigint generated by default as identity,
"article_id" bigint,
"article_url" varchar(255),
"board_name" integer,
"hit" integer,
"recommended" integer,
"registered_at" timestamp,
"title" varchar(255),
primary key ("id")
)
create table "ppomppu_board_feed_status"
(
"id" bigint generated by default as identity,
"board_name" integer,
"latest_parsed_article_id" bigint,
"updated_at" timestamp,
primary key ("id")
)
create table "published_history"
(
"id" bigint generated by default as identity,
"board_name_list" varchar(255),
"published_at" timestamp,
"user_id" bigint,
primary key ("id")
)
create table "subscribed_board"
(
"id" bigint generated by default as identity,
"board_name" integer,
"user_id" bigint,
primary key ("id")
)
create table "subscribed_user"
(
"id" bigint generated by default as identity,
"registered_at" timestamp,
"user_id" bigint,
primary key ("id")
)

54
PpomppuNotifier_ERD.puml Normal file
View File

@ -0,0 +1,54 @@
@startuml
'https://plantuml.com/sequence-diagram
class SubscribedUser {
- id
+ user_id
+ registered_at
+ created_at
+ modified_at
}
class SubscribedBoard {
- id
- user_id
+ board_name
+ created_at
+ modified_at
}
class PublishedHisotry {
- id
+ user_id
+ board_name_list
+ published_at
+ created_at
+ modified_at
}
class PpomppuArticle {
- id
+ article_id
+ board_name
+ article_url
+ title
+ recommended
+ hit
+ registered_at
+ created_at
+ modified_at
}
class PpomppuBoardFeedStatus {
- id
+ board_name
+ latest_parsed_article_id
+ updated_at
+ created_at
+ modified_at
}
SubscribedUser --o{ SubscribedBoard
SubscribedUser --o{ PublishedHisotry
@enduml

View File

@ -8,8 +8,10 @@ dependencies {
// https://projectreactor.io/docs/core/release/reference/#debug-activate
implementation 'org.springframework.boot:spring-boot-starter-webflux'
implementation 'org.springframework.boot:spring-boot-starter-data-jpa'
implementation 'org.springframework.boot:spring-boot-starter-jdbc'
implementation 'com.rometools:rome:1.16.0'
implementation 'org.jsoup:jsoup:1.14.2'
implementation 'com.h2database:h2:1.4.200'
annotationProcessor 'org.springframework.boot:spring-boot-configuration-processor'
annotationProcessor 'org.projectlombok:lombok'

View File

@ -0,0 +1,35 @@
package com.myoa.engineering.crawl.ppomppu.processor.configuration;
import java.sql.SQLException;
import lombok.extern.slf4j.Slf4j;
import org.h2.tools.Server;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Profile;
import org.springframework.context.event.ContextClosedEvent;
import org.springframework.context.event.ContextRefreshedEvent;
import org.springframework.context.event.EventListener;
@Slf4j
@Profile("development")
@Configuration
public class H2ConsoleConfiguration {
private Server webServer;
@Value("${spring.h2.console.port}")
private String port;
@EventListener(ContextRefreshedEvent.class)
public void start() throws SQLException {
log.info("starting h2 console");
this.webServer = Server.createWebServer("-webPort", port, "-tcpAllowOthers").start();
}
@EventListener(ContextClosedEvent.class)
public void stop() {
log.info("stopping h2 console");
this.webServer.stop(); ;
}
}

View File

@ -0,0 +1,124 @@
package com.myoa.engineering.crawl.ppomppu.processor.configuration;
import com.myoa.engineering.crawl.ppomppu.processor.configuration.properties.DatasourceProperties;
import com.myoa.engineering.crawl.ppomppu.processor.configuration.properties.DatasourceProperties.DataSourcePropertiesUnit;
import com.myoa.engineering.crawl.ppomppu.processor.configuration.properties.HibernateProperties;
import com.myoa.engineering.crawl.ppomppu.processor.configuration.properties.HikariProperties;
import com.myoa.engineering.crawl.ppomppu.processor.domain.BaseScanDomain;
import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository.BaseScanRepository;
import com.zaxxer.hikari.HikariConfig;
import com.zaxxer.hikari.HikariDataSource;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
import javax.persistence.EntityManagerFactory;
import javax.sql.DataSource;
import lombok.NonNull;
import org.hibernate.cfg.AvailableSettings;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.boot.orm.jpa.EntityManagerFactoryBuilder;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.data.jpa.repository.config.EnableJpaRepositories;
import org.springframework.orm.jpa.JpaTransactionManager;
import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean;
import org.springframework.transaction.PlatformTransactionManager;
@Configuration
@EnableJpaRepositories(basePackageClasses = BaseScanRepository.class,
entityManagerFactoryRef = "ppomppuNotifierProcessorEntityManagerFactory",
transactionManagerRef = "ppomppuNotifierProcessorTransactionManager"
)
public class PpomppuDatasourceConfiguration {
private static final String DATA_SOURCE_UNIT_NAME = "ppomppu";
private final DatasourceProperties dataSourceProeprties;
private final HikariProperties hikariProperties;
private final HibernateProperties hibernateProperties;
public PpomppuDatasourceConfiguration(DatasourceProperties dataSourceProeprties,
HikariProperties hikariProperties,
HibernateProperties hibernateProperties) {
this.dataSourceProeprties = dataSourceProeprties;
this.hikariProperties = hikariProperties;
this.hibernateProperties = hibernateProperties;
}
@Bean(name = "ppomppuNotifierProcessorDataSource")
public DataSource dataSource() {
DataSourcePropertiesUnit dataSourcePropertiesUnit = dataSourceProeprties.find(DATA_SOURCE_UNIT_NAME);
final HikariConfig hikariConfig = new HikariConfig();
hikariConfig.setJdbcUrl(dataSourcePropertiesUnit.toCompletedJdbcUrl());
hikariConfig.setUsername("sa");
hikariConfig.setPassword("sa");
hikariConfig.setAutoCommit(hikariProperties.getAutoCommit());
hikariConfig.setMaximumPoolSize(hikariProperties.getMaximumPoolSize());
hikariConfig.setMinimumIdle(hikariProperties.getMinimumIdle());
if (hikariProperties.getMaximumPoolSize() > hikariProperties.getMinimumIdle()) {
hikariConfig.setIdleTimeout(hikariProperties.getIdleTimeout());
}
hikariConfig.setValidationTimeout(hikariProperties.getValidationTimeout());
hikariConfig.setConnectionTimeout(hikariProperties.getConnectionTimeout());
hikariConfig.setMaxLifetime(hikariProperties.getMaxLifetime());
final DataSource dataSource = new HikariDataSource(hikariConfig);
return dataSource;
}
@Bean("ppomppuNotifierProcessorEntityManagerFactory")
public LocalContainerEntityManagerFactoryBean entityManagerFactory(
EntityManagerFactoryBuilder builder,
@Qualifier("ppomppuNotifierProcessorDataSource") DataSource dataSource) {
return builder.dataSource(dataSource)
.packages(BaseScanDomain.class)
.properties(getPropsMap(DATA_SOURCE_UNIT_NAME))
.build();
}
@Bean("ppomppuNotifierProcessorTransactionManager")
public PlatformTransactionManager transactionManager(
@Qualifier("ppomppuNotifierProcessorEntityManagerFactory") EntityManagerFactory entityManagerFactory) {
return new JpaTransactionManager(entityManagerFactory);
}
public static Properties getProps(@NonNull HibernateProperties.HibernatePropertiesUnit hibernateProperties) {
Properties properties = new Properties();
properties.put(AvailableSettings.DIALECT, hibernateProperties.getDialect());
properties.put(AvailableSettings.FORMAT_SQL, hibernateProperties.getFormatSql());
properties.put(AvailableSettings.SHOW_SQL, hibernateProperties.getShowSql());
properties.put(AvailableSettings.HBM2DDL_AUTO, hibernateProperties.getHbm2ddlAuto());
properties.put(AvailableSettings.CONNECTION_PROVIDER_DISABLES_AUTOCOMMIT,
hibernateProperties.getDisableAutoCommit());
properties.put(AvailableSettings.IMPLICIT_NAMING_STRATEGY,
"org.springframework.boot.orm.jpa.hibernate.SpringImplicitNamingStrategy");
properties.put(AvailableSettings.PHYSICAL_NAMING_STRATEGY,
"org.springframework.boot.orm.jpa.hibernate.SpringPhysicalNamingStrategy");
properties.put(AvailableSettings.GENERATE_STATISTICS, "false");
properties.put(AvailableSettings.GLOBALLY_QUOTED_IDENTIFIERS, "true");
properties.put(AvailableSettings.GLOBALLY_QUOTED_IDENTIFIERS_SKIP_COLUMN_DEFINITIONS, "true");
properties.put(AvailableSettings.STATEMENT_BATCH_SIZE, "20");
properties.put(AvailableSettings.ORDER_INSERTS, "true");
properties.put(AvailableSettings.ORDER_UPDATES, "true");
properties.put(AvailableSettings.BATCH_VERSIONED_DATA, "true");
properties.put(AvailableSettings.USE_NEW_ID_GENERATOR_MAPPINGS, "false");
return properties;
}
public Map<String, String> getPropsMap(@NonNull String unitName) {
return convertPropertiestoMaps(getProps(hibernateProperties.find(unitName)));
}
public Map<String, String> convertPropertiestoMaps(Properties properties) {
Map<String, String> propertiesMap = new HashMap<>();
for (Enumeration<?> e = properties.propertyNames(); e.hasMoreElements(); ) {
String key = (String) e.nextElement();
propertiesMap.put(key, properties.getProperty(key));
}
return propertiesMap;
}
}

View File

@ -0,0 +1,44 @@
package com.myoa.engineering.crawl.ppomppu.processor.configuration.properties;
import com.myoa.engineering.crawl.ppomppu.support.util.ObjectUtil;
import java.util.List;
import lombok.Getter;
import lombok.Setter;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;
@Component
@Setter
@Getter
@ConfigurationProperties(prefix = "datasource")
public class DatasourceProperties {
private List<DataSourcePropertiesUnit> units;
@Getter
@Setter
public static class DataSourcePropertiesUnit {
private String unitName;
private String schemaName;
private String connectionParameters;
private String dbConnectionUrl;
private Boolean simpleConnectionUrl;
public String toCompletedJdbcUrl() {
if (ObjectUtil.isEmpty(simpleConnectionUrl) || simpleConnectionUrl == false) {
return String.format("%s/%s?%s", dbConnectionUrl, schemaName, connectionParameters);
}
return dbConnectionUrl;
}
}
public DataSourcePropertiesUnit find(String unitName) {
return units.stream()
.filter(e -> e.getUnitName().equals(unitName))
.findFirst()
.orElseThrow(
() -> new IllegalArgumentException(this.getClass().getName() + ": unitName Not found. " + unitName));
}
}

View File

@ -0,0 +1,38 @@
package com.myoa.engineering.crawl.ppomppu.processor.configuration.properties;
import java.util.List;
import lombok.Getter;
import lombok.Setter;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;
@Component
@Setter
@Getter
@ConfigurationProperties(prefix = "hibernate")
public class HibernateProperties {
private List<HibernatePropertiesUnit> units;
@Getter
@Setter
public static class HibernatePropertiesUnit {
private String unitName;
private String dialect;
private String formatSql;
private String showSql;
private String hbm2ddlAuto;
private String disableAutoCommit;
}
public HibernatePropertiesUnit find(String unitName) {
return units.stream()
.filter(x -> x.getUnitName().equals(unitName))
.findFirst()
.orElseThrow(
() -> new IllegalArgumentException(this.getClass().getName() + ": unitName Not found. " + unitName));
}
}

View File

@ -0,0 +1,22 @@
package com.myoa.engineering.crawl.ppomppu.processor.configuration.properties;
import lombok.Getter;
import lombok.Setter;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;
@Component
@Setter
@Getter
@ConfigurationProperties(prefix = "spring.datasource.hikari")
public class HikariProperties {
private Integer minimumIdle;
private Integer maximumPoolSize;
private Integer idleTimeout;
private Integer validationTimeout;
private Integer connectionTimeout;
private Integer maxLifetime;
private Boolean autoCommit;
}

View File

@ -1,22 +1,22 @@
package com.myoa.engineering.crawl.ppomppu.processor.domain;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import java.time.Instant;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.Enumerated;
import javax.persistence.GeneratedValue;
import javax.persistence.GenerationType;
import javax.persistence.Id;
import javax.persistence.Table;
import lombok.Builder;
import lombok.Getter;
import lombok.NoArgsConstructor;
/**
* PpomppuArticle
*
* @author Shin Woo-jin (woozu.shin@kakaoent.com)
* @since 2021-09-08
*/
@Getter
@NoArgsConstructor
@Entity
@Table(name = "ppomppu_article")
public class PpomppuArticle extends Auditable {
@Id
@ -26,6 +26,10 @@ public class PpomppuArticle extends Auditable {
@Column
private Long articleId;
@Column
@Enumerated
private PpomppuBoardName boardName;
@Column
private String articleUrl;
@ -42,10 +46,11 @@ public class PpomppuArticle extends Auditable {
private Instant registeredAt;
@Builder
public PpomppuArticle(Long id, Long articleId, String articleUrl, String title,
Integer recommended, Integer hit, Instant registeredAt) {
public PpomppuArticle(Long id, Long articleId, PpomppuBoardName boardName, String articleUrl,
String title, Integer recommended, Integer hit, Instant registeredAt) {
this.id = id;
this.articleId = articleId;
this.boardName = boardName;
this.articleUrl = articleUrl;
this.title = title;
this.recommended = recommended;
@ -53,16 +58,4 @@ public class PpomppuArticle extends Auditable {
this.registeredAt = registeredAt;
}
@Override
public String toString() {
return "PpomppuArticle{" +
"id=" + id +
", articleId=" + articleId +
", articleUrl='" + articleUrl + '\'' +
", title='" + title + '\'' +
", hit=" + hit +
", recommended=" + recommended +
", registeredAt=" + registeredAt +
'}';
}
}

View File

@ -0,0 +1,33 @@
package com.myoa.engineering.crawl.ppomppu.processor.domain;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import java.time.Instant;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.GeneratedValue;
import javax.persistence.GenerationType;
import javax.persistence.Id;
import javax.persistence.Table;
import lombok.Getter;
import lombok.NoArgsConstructor;
@Getter
@NoArgsConstructor
@Entity
@Table(name = "ppomppu_board_feed_status")
public class PpomppuBoardFeedStatus extends Auditable {
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
private Long id;
@Column
private Long latestParsedArticleId;
@Column
private PpomppuBoardName boardName;
@Column
private Instant updatedAt;
}

View File

@ -0,0 +1,32 @@
package com.myoa.engineering.crawl.ppomppu.processor.domain;
import java.time.Instant;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.GeneratedValue;
import javax.persistence.GenerationType;
import javax.persistence.Id;
import javax.persistence.Table;
import lombok.Getter;
import lombok.NoArgsConstructor;
@Getter
@NoArgsConstructor
@Entity
@Table(name = "published_history")
public class PublishedHistory extends Auditable {
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
private Long id;
@Column
private Long userId;
@Column
private String boardNameList;
@Column
private Instant publishedAt;
}

View File

@ -0,0 +1,29 @@
package com.myoa.engineering.crawl.ppomppu.processor.domain;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.GeneratedValue;
import javax.persistence.GenerationType;
import javax.persistence.Id;
import javax.persistence.Table;
import lombok.Getter;
import lombok.NoArgsConstructor;
@Getter
@NoArgsConstructor
@Entity
@Table(name = "subscribed_board")
public class SubscribedBoard extends Auditable {
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
private Long id;
@Column
private Long userId;
@Column
private PpomppuBoardName boardName;
}

View File

@ -0,0 +1,29 @@
package com.myoa.engineering.crawl.ppomppu.processor.domain;
import java.time.Instant;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.GeneratedValue;
import javax.persistence.GenerationType;
import javax.persistence.Id;
import javax.persistence.Table;
import lombok.Getter;
import lombok.NoArgsConstructor;
@Getter
@NoArgsConstructor
@Entity
@Table(name = "subscribed_user")
public class SubscribedUser extends Auditable{
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
private Long id;
@Column
private Long userId;
@Column
private Instant registeredAt;
}

View File

@ -0,0 +1,4 @@
package com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository;
public interface BaseScanRepository {
}

View File

@ -0,0 +1,10 @@
package com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository;
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.stereotype.Repository;
@Repository
public interface PpomppuArticleRepository extends JpaRepository<PpomppuArticle, Long> {
}

View File

@ -0,0 +1,10 @@
package com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository;
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuBoardFeedStatus;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.stereotype.Repository;
@Repository
public interface PpomppuBoardFeedStatusRepository extends JpaRepository<PpomppuBoardFeedStatus, Long> {
}

View File

@ -0,0 +1,30 @@
package com.myoa.engineering.crawl.ppomppu.processor.service;
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository.PpomppuArticleRepository;
import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository.PpomppuBoardFeedStatusRepository;
import java.util.List;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
@Slf4j
@Service
public class PpomppuArticleService {
private final PpomppuArticleRepository ppomppuArticleRepository;
private final PpomppuBoardFeedStatusRepository ppomppuBoardFeedStatusRepository;
public PpomppuArticleService(PpomppuArticleRepository ppomppuArticleRepository,
PpomppuBoardFeedStatusRepository ppomppuBoardFeedStatusRepository) {
this.ppomppuArticleRepository = ppomppuArticleRepository;
this.ppomppuBoardFeedStatusRepository = ppomppuBoardFeedStatusRepository;
}
public void save(List<PpomppuArticle> articles) {
// TODO get latest parsed article id
// TODO filter articles
ppomppuArticleRepository.saveAll(articles);
}
}

View File

@ -4,4 +4,5 @@ spring:
on-profile: development
import:
- classpath:/development/webclient.yml
- classpath:/development/temppassword.yml
- classpath:/development/database.yml

View File

@ -11,4 +11,4 @@ server:
port: 20081
error:
whitelabel:
enabled: false
enabled: false

View File

@ -33,7 +33,6 @@ public class MessageDispatcher extends TelegramLongPollingBot {
Message message = update.getMessage();
MessageHandler handler = getMessageHandler(message);
log.info(message.getText());
handler.handle(message);
}

View File

@ -0,0 +1,21 @@
package com.myoa.engineering.crawl.ppomppu.receiver.handler.message;
import com.myoa.engineering.crawl.ppomppu.support.util.ObjectUtil;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import org.telegram.telegrambots.meta.api.objects.Message;
@Slf4j
@Component
public class HelloWorldMessageHandler implements MessageHandler {
@Override
public boolean isApplicable(Message message) {
return ObjectUtil.isEmpty(message);
}
@Override
public void handle(Message message) {
// skip empty event message.
}
}

View File

@ -1,17 +1,18 @@
package com.myoa.engineering.crawl.ppomppu.receiver.handler.message;
import com.myoa.engineering.crawl.ppomppu.support.util.ObjectUtil;
import org.telegram.telegrambots.meta.api.objects.Message;
/**
* TextMessageHandler
*
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-08-21
*
*/
public interface TextMessageHandler extends MessageHandler {
@Override
default boolean isApplicable(Message message) {
return message.isUserMessage() && message.hasText();
return ObjectUtil.isNotEmpty(message) && message.isUserMessage() && message.hasText();
}
}

View File

@ -3,5 +3,5 @@ spring:
activate:
on-profile: development
import:
- classpath:/webclient-development.yml
- classpath:/temppassword.yml
- classpath:/development/webclient.yml
- classpath:/development/temppassword.yml

View File

@ -4,4 +4,12 @@ public final class ObjectUtil {
private ObjectUtil() {}
public static boolean isEmpty(Object o) {
return o == null;
}
public static boolean isNotEmpty(Object o) {
return !isEmpty(o);
}
}

View File

@ -0,0 +1,41 @@
spring:
jpa:
open-in-view: false
hibernate:
ddl-auto: create
datasource:
# driver-class-name: com.mysql.cj.jdbc.Driver
driver-class-name: org.h2.Driver
url: jdbc:h2:mem:ppomppu-local;DB_CLOSE_DELAY=-1
hikari:
minimum:idle: 5
maximum-pool-size: 10
idle-timeout: 600000
validation-timeout: 5000
connection-timeout: 5000
max-lifetime: 1800000
auto-commit: false
h2:
console:
enabled: true
path: /h2
port: 20082
datasource:
init: true
units:
- unit-name: ppomppu
schema-name: ppomppu-local
db-connection-url: jdbc:h2:mem:ppomppu-local
simple-connection-url: true
hibernate:
units:
- unit-name: ppomppu
dialect: org.hibernate.dialect.H2Dialect
format-sql: true
show-sql: true
hbm2ddl-auto: create
disable-auto-commit: true