MyoaEngineering/ShoppingCrawler
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

[PPN-210926-5] Set-up database for development environment

Browse Source
This commit is contained in:
woozu.shin
2021-09-26 21:20:16 +09:00
parent 398a94ce3f
commit ab4ab339f6
29 changed files with 632 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/H2ConsoleConfiguration.java Normal file
View File

@@ -0,0 +1,35 @@
package com.myoa.engineering.crawl.ppomppu.processor.configuration;
import java.sql.SQLException;
import lombok.extern.slf4j.Slf4j;
import org.h2.tools.Server;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Profile;
import org.springframework.context.event.ContextClosedEvent;
import org.springframework.context.event.ContextRefreshedEvent;
import org.springframework.context.event.EventListener;
@Slf4j
@Profile("development")
@Configuration
public class H2ConsoleConfiguration {
private Server webServer;
@Value("${spring.h2.console.port}")
private String port;
@EventListener(ContextRefreshedEvent.class)
public void start() throws SQLException {
log.info("starting h2 console");
this.webServer = Server.createWebServer("-webPort", port, "-tcpAllowOthers").start();
}
@EventListener(ContextClosedEvent.class)
public void stop() {
log.info("stopping h2 console");
this.webServer.stop(); ;
}
}

124
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/PpomppuDatasourceConfiguration.java Normal file
View File

@@ -0,0 +1,124 @@
package com.myoa.engineering.crawl.ppomppu.processor.configuration;
import com.myoa.engineering.crawl.ppomppu.processor.configuration.properties.DatasourceProperties;
import com.myoa.engineering.crawl.ppomppu.processor.configuration.properties.DatasourceProperties.DataSourcePropertiesUnit;
import com.myoa.engineering.crawl.ppomppu.processor.configuration.properties.HibernateProperties;
import com.myoa.engineering.crawl.ppomppu.processor.configuration.properties.HikariProperties;
import com.myoa.engineering.crawl.ppomppu.processor.domain.BaseScanDomain;
import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository.BaseScanRepository;
import com.zaxxer.hikari.HikariConfig;
import com.zaxxer.hikari.HikariDataSource;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
import javax.persistence.EntityManagerFactory;
import javax.sql.DataSource;
import lombok.NonNull;
import org.hibernate.cfg.AvailableSettings;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.boot.orm.jpa.EntityManagerFactoryBuilder;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.data.jpa.repository.config.EnableJpaRepositories;
import org.springframework.orm.jpa.JpaTransactionManager;
import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean;
import org.springframework.transaction.PlatformTransactionManager;
@Configuration
@EnableJpaRepositories(basePackageClasses = BaseScanRepository.class,
entityManagerFactoryRef = "ppomppuNotifierProcessorEntityManagerFactory",
transactionManagerRef = "ppomppuNotifierProcessorTransactionManager"
)
public class PpomppuDatasourceConfiguration {
private static final String DATA_SOURCE_UNIT_NAME = "ppomppu";
private final DatasourceProperties dataSourceProeprties;
private final HikariProperties hikariProperties;
private final HibernateProperties hibernateProperties;
public PpomppuDatasourceConfiguration(DatasourceProperties dataSourceProeprties,
HikariProperties hikariProperties,
HibernateProperties hibernateProperties) {
this.dataSourceProeprties = dataSourceProeprties;
this.hikariProperties = hikariProperties;
this.hibernateProperties = hibernateProperties;
}
@Bean(name = "ppomppuNotifierProcessorDataSource")
public DataSource dataSource() {
DataSourcePropertiesUnit dataSourcePropertiesUnit = dataSourceProeprties.find(DATA_SOURCE_UNIT_NAME);
final HikariConfig hikariConfig = new HikariConfig();
hikariConfig.setJdbcUrl(dataSourcePropertiesUnit.toCompletedJdbcUrl());
hikariConfig.setUsername("sa");
hikariConfig.setPassword("sa");
hikariConfig.setAutoCommit(hikariProperties.getAutoCommit());
hikariConfig.setMaximumPoolSize(hikariProperties.getMaximumPoolSize());
hikariConfig.setMinimumIdle(hikariProperties.getMinimumIdle());
if (hikariProperties.getMaximumPoolSize() > hikariProperties.getMinimumIdle()) {
hikariConfig.setIdleTimeout(hikariProperties.getIdleTimeout());
}
hikariConfig.setValidationTimeout(hikariProperties.getValidationTimeout());
hikariConfig.setConnectionTimeout(hikariProperties.getConnectionTimeout());
hikariConfig.setMaxLifetime(hikariProperties.getMaxLifetime());
final DataSource dataSource = new HikariDataSource(hikariConfig);
return dataSource;
}
@Bean("ppomppuNotifierProcessorEntityManagerFactory")
public LocalContainerEntityManagerFactoryBean entityManagerFactory(
EntityManagerFactoryBuilder builder,
@Qualifier("ppomppuNotifierProcessorDataSource") DataSource dataSource) {
return builder.dataSource(dataSource)
.packages(BaseScanDomain.class)
.properties(getPropsMap(DATA_SOURCE_UNIT_NAME))
.build();
}
@Bean("ppomppuNotifierProcessorTransactionManager")
public PlatformTransactionManager transactionManager(
@Qualifier("ppomppuNotifierProcessorEntityManagerFactory") EntityManagerFactory entityManagerFactory) {
return new JpaTransactionManager(entityManagerFactory);
}
public static Properties getProps(@NonNull HibernateProperties.HibernatePropertiesUnit hibernateProperties) {
Properties properties = new Properties();
properties.put(AvailableSettings.DIALECT, hibernateProperties.getDialect());
properties.put(AvailableSettings.FORMAT_SQL, hibernateProperties.getFormatSql());
properties.put(AvailableSettings.SHOW_SQL, hibernateProperties.getShowSql());
properties.put(AvailableSettings.HBM2DDL_AUTO, hibernateProperties.getHbm2ddlAuto());
properties.put(AvailableSettings.CONNECTION_PROVIDER_DISABLES_AUTOCOMMIT,
hibernateProperties.getDisableAutoCommit());
properties.put(AvailableSettings.IMPLICIT_NAMING_STRATEGY,
"org.springframework.boot.orm.jpa.hibernate.SpringImplicitNamingStrategy");
properties.put(AvailableSettings.PHYSICAL_NAMING_STRATEGY,
"org.springframework.boot.orm.jpa.hibernate.SpringPhysicalNamingStrategy");
properties.put(AvailableSettings.GENERATE_STATISTICS, "false");
properties.put(AvailableSettings.GLOBALLY_QUOTED_IDENTIFIERS, "true");
properties.put(AvailableSettings.GLOBALLY_QUOTED_IDENTIFIERS_SKIP_COLUMN_DEFINITIONS, "true");
properties.put(AvailableSettings.STATEMENT_BATCH_SIZE, "20");
properties.put(AvailableSettings.ORDER_INSERTS, "true");
properties.put(AvailableSettings.ORDER_UPDATES, "true");
properties.put(AvailableSettings.BATCH_VERSIONED_DATA, "true");
properties.put(AvailableSettings.USE_NEW_ID_GENERATOR_MAPPINGS, "false");
return properties;
}
public Map<String, String> getPropsMap(@NonNull String unitName) {
return convertPropertiestoMaps(getProps(hibernateProperties.find(unitName)));
}
public Map<String, String> convertPropertiestoMaps(Properties properties) {
Map<String, String> propertiesMap = new HashMap<>();
for (Enumeration<?> e = properties.propertyNames(); e.hasMoreElements(); ) {
String key = (String) e.nextElement();
propertiesMap.put(key, properties.getProperty(key));
}
return propertiesMap;
}
}

44
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/properties/DatasourceProperties.java Normal file
View File

@@ -0,0 +1,44 @@
package com.myoa.engineering.crawl.ppomppu.processor.configuration.properties;
import com.myoa.engineering.crawl.ppomppu.support.util.ObjectUtil;
import java.util.List;
import lombok.Getter;
import lombok.Setter;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;
@Component
@Setter
@Getter
@ConfigurationProperties(prefix = "datasource")
public class DatasourceProperties {
private List<DataSourcePropertiesUnit> units;
@Getter
@Setter
public static class DataSourcePropertiesUnit {
private String unitName;
private String schemaName;
private String connectionParameters;
private String dbConnectionUrl;
private Boolean simpleConnectionUrl;
public String toCompletedJdbcUrl() {
if (ObjectUtil.isEmpty(simpleConnectionUrl) || simpleConnectionUrl == false) {
return String.format("%s/%s?%s", dbConnectionUrl, schemaName, connectionParameters);
}
return dbConnectionUrl;
}
}
public DataSourcePropertiesUnit find(String unitName) {
return units.stream()
.filter(e -> e.getUnitName().equals(unitName))
.findFirst()
.orElseThrow(
() -> new IllegalArgumentException(this.getClass().getName() + ": unitName Not found. " + unitName));
}
}

38
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/properties/HibernateProperties.java Normal file
View File

@@ -0,0 +1,38 @@
package com.myoa.engineering.crawl.ppomppu.processor.configuration.properties;
import java.util.List;
import lombok.Getter;
import lombok.Setter;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;
@Component
@Setter
@Getter
@ConfigurationProperties(prefix = "hibernate")
public class HibernateProperties {
private List<HibernatePropertiesUnit> units;
@Getter
@Setter
public static class HibernatePropertiesUnit {
private String unitName;
private String dialect;
private String formatSql;
private String showSql;
private String hbm2ddlAuto;
private String disableAutoCommit;
}
public HibernatePropertiesUnit find(String unitName) {
return units.stream()
.filter(x -> x.getUnitName().equals(unitName))
.findFirst()
.orElseThrow(
() -> new IllegalArgumentException(this.getClass().getName() + ": unitName Not found. " + unitName));
}
}

22
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/properties/HikariProperties.java Normal file
View File

@@ -0,0 +1,22 @@
package com.myoa.engineering.crawl.ppomppu.processor.configuration.properties;
import lombok.Getter;
import lombok.Setter;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;
@Component
@Setter
@Getter
@ConfigurationProperties(prefix = "spring.datasource.hikari")
public class HikariProperties {
private Integer minimumIdle;
private Integer maximumPoolSize;
private Integer idleTimeout;
private Integer validationTimeout;
private Integer connectionTimeout;
private Integer maxLifetime;
private Boolean autoCommit;
}

33
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/PpomppuArticle.java
View File

@@ -1,22 +1,22 @@
package com.myoa.engineering.crawl.ppomppu.processor.domain;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import java.time.Instant;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.Enumerated;
import javax.persistence.GeneratedValue;
import javax.persistence.GenerationType;
import javax.persistence.Id;
import javax.persistence.Table;
import lombok.Builder;
import lombok.Getter;
import lombok.NoArgsConstructor;
/**
* PpomppuArticle
*
* @author Shin Woo-jin (woozu.shin@kakaoent.com)
* @since 2021-09-08
*/
@Getter
@NoArgsConstructor
@Entity
@Table(name = "ppomppu_article")
public class PpomppuArticle extends Auditable {
@Id
@@ -26,6 +26,10 @@ public class PpomppuArticle extends Auditable {
@Column
private Long articleId;
@Column
@Enumerated
private PpomppuBoardName boardName;
@Column
private String articleUrl;
@@ -42,10 +46,11 @@ public class PpomppuArticle extends Auditable {
private Instant registeredAt;
@Builder
public PpomppuArticle(Long id, Long articleId, String articleUrl, String title,
Integer recommended, Integer hit, Instant registeredAt) {
public PpomppuArticle(Long id, Long articleId, PpomppuBoardName boardName, String articleUrl,
String title, Integer recommended, Integer hit, Instant registeredAt) {
this.id = id;
this.articleId = articleId;
this.boardName = boardName;
this.articleUrl = articleUrl;
this.title = title;
this.recommended = recommended;
@@ -53,16 +58,4 @@ public class PpomppuArticle extends Auditable {
this.registeredAt = registeredAt;
}
@Override
public String toString() {
return "PpomppuArticle{" +
"id=" + id +
", articleId=" + articleId +
", articleUrl='" + articleUrl + '\'' +
", title='" + title + '\'' +
", hit=" + hit +
", recommended=" + recommended +
", registeredAt=" + registeredAt +
'}';
}
}

33
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/PpomppuBoardFeedStatus.java Normal file
View File

@@ -0,0 +1,33 @@
package com.myoa.engineering.crawl.ppomppu.processor.domain;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import java.time.Instant;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.GeneratedValue;
import javax.persistence.GenerationType;
import javax.persistence.Id;
import javax.persistence.Table;
import lombok.Getter;
import lombok.NoArgsConstructor;
@Getter
@NoArgsConstructor
@Entity
@Table(name = "ppomppu_board_feed_status")
public class PpomppuBoardFeedStatus extends Auditable {
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
private Long id;
@Column
private Long latestParsedArticleId;
@Column
private PpomppuBoardName boardName;
@Column
private Instant updatedAt;
}

32
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/PublishedHistory.java Normal file
View File

@@ -0,0 +1,32 @@
package com.myoa.engineering.crawl.ppomppu.processor.domain;
import java.time.Instant;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.GeneratedValue;
import javax.persistence.GenerationType;
import javax.persistence.Id;
import javax.persistence.Table;
import lombok.Getter;
import lombok.NoArgsConstructor;
@Getter
@NoArgsConstructor
@Entity
@Table(name = "published_history")
public class PublishedHistory extends Auditable {
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
private Long id;
@Column
private Long userId;
@Column
private String boardNameList;
@Column
private Instant publishedAt;
}

29
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/SubscribedBoard.java Normal file
View File

@@ -0,0 +1,29 @@
package com.myoa.engineering.crawl.ppomppu.processor.domain;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.GeneratedValue;
import javax.persistence.GenerationType;
import javax.persistence.Id;
import javax.persistence.Table;
import lombok.Getter;
import lombok.NoArgsConstructor;
@Getter
@NoArgsConstructor
@Entity
@Table(name = "subscribed_board")
public class SubscribedBoard extends Auditable {
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
private Long id;
@Column
private Long userId;
@Column
private PpomppuBoardName boardName;
}

29
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/SubscribedUser.java Normal file
View File

@@ -0,0 +1,29 @@
package com.myoa.engineering.crawl.ppomppu.processor.domain;
import java.time.Instant;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.GeneratedValue;
import javax.persistence.GenerationType;
import javax.persistence.Id;
import javax.persistence.Table;
import lombok.Getter;
import lombok.NoArgsConstructor;
@Getter
@NoArgsConstructor
@Entity
@Table(name = "subscribed_user")
public class SubscribedUser extends Auditable{
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
private Long id;
@Column
private Long userId;
@Column
private Instant registeredAt;
}

4
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/repository/BaseScanRepository.java Normal file
View File

@@ -0,0 +1,4 @@
package com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository;
public interface BaseScanRepository {
}

10
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/repository/PpomppuArticleRepository.java Normal file
View File

@@ -0,0 +1,10 @@
package com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository;
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.stereotype.Repository;
@Repository
public interface PpomppuArticleRepository extends JpaRepository<PpomppuArticle, Long> {
}

10
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/repository/PpomppuBoardFeedStatusRepository.java Normal file
View File

@@ -0,0 +1,10 @@
package com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository;
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuBoardFeedStatus;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.stereotype.Repository;
@Repository
public interface PpomppuBoardFeedStatusRepository extends JpaRepository<PpomppuBoardFeedStatus, Long> {
}

30
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/PpomppuArticleService.java Normal file
View File

@@ -0,0 +1,30 @@
package com.myoa.engineering.crawl.ppomppu.processor.service;
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository.PpomppuArticleRepository;
import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository.PpomppuBoardFeedStatusRepository;
import java.util.List;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
@Slf4j
@Service
public class PpomppuArticleService {
private final PpomppuArticleRepository ppomppuArticleRepository;
private final PpomppuBoardFeedStatusRepository ppomppuBoardFeedStatusRepository;
public PpomppuArticleService(PpomppuArticleRepository ppomppuArticleRepository,
PpomppuBoardFeedStatusRepository ppomppuBoardFeedStatusRepository) {
this.ppomppuArticleRepository = ppomppuArticleRepository;
this.ppomppuBoardFeedStatusRepository = ppomppuBoardFeedStatusRepository;
}
public void save(List<PpomppuArticle> articles) {
// TODO get latest parsed article id
// TODO filter articles
ppomppuArticleRepository.saveAll(articles);
}
}

1
processor/src/main/resources/application-development.yml
View File

@@ -4,4 +4,5 @@ spring:
on-profile: development
import:
- classpath:/development/webclient.yml
- classpath:/development/temppassword.yml
- classpath:/development/database.yml

2
processor/src/main/resources/application.yml
View File

@@ -11,4 +11,4 @@ server:
port: 20081
error:
whitelabel:
enabled: false
enabled: false

5
processor/src/main/resources/development/webclient.yml Normal file
View File

@@ -0,0 +1,5 @@
webclient:
some: test
units:
- unit-name: processor-api
base-url: http://localhost:20081

4
processor/src/main/resources/production/webclient.yml Normal file
View File

@@ -0,0 +1,4 @@
webclient:
units:
- unit-name: processor-api
base-url: http://soundhoundfound-processor:20080