MyoaEngineering/ShoppingCrawler
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

[NO-ISSUE] Implement v2

Browse Source
This commit is contained in:
woozu-shin
2024-04-30 22:34:49 +09:00
parent 0524a18ee5
commit 0c4be3cc05
147 changed files with 3873 additions and 546 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
processor/build.gradle
View File

@@ -12,14 +12,15 @@ dependencies {
implementation 'org.springframework.boot:spring-boot-starter-jdbc'
implementation 'org.springframework.cloud:spring-cloud-starter-config'
implementation 'org.springframework.boot:spring-boot-starter-actuator'
implementation 'com.rometools:rome:1.16.0'
implementation 'org.jsoup:jsoup:1.14.2'
implementation 'com.h2database:h2:1.4.200'
implementation 'com.rometools:rome:2.1.0'
implementation 'org.jsoup:jsoup:1.15.3'
implementation 'com.h2database:h2:2.2.220'
annotationProcessor 'org.springframework.boot:spring-boot-configuration-processor'
annotationProcessor 'org.projectlombok:lombok'
testImplementation 'org.springframework.boot:spring-boot-starter-test'
testImplementation("org.assertj:assertj-core:3.24.2")
}
test {

2
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/ProcessorApplication.java
View File

@@ -4,7 +4,7 @@ import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.context.annotation.Import;
import com.myoa.engineering.crawl.ppomppu.support.webclient.PpomppuNotifierWebClientConfiguration;
import com.myoa.engineering.crawl.shopping.support.webclient.PpomppuNotifierWebClientConfiguration;
/**
* ProcessorApplication

35
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/H2ConsoleConfiguration.java
View File

@@ -1,35 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.processor.configuration;
import java.sql.SQLException;
import lombok.extern.slf4j.Slf4j;
import org.h2.tools.Server;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Profile;
import org.springframework.context.event.ContextClosedEvent;
import org.springframework.context.event.ContextRefreshedEvent;
import org.springframework.context.event.EventListener;
@Slf4j
@Profile({"datasource-local", "datasource-development"})
@Configuration
public class H2ConsoleConfiguration {
private Server webServer;
@Value("${spring.h2.console.port}")
private String port;
@EventListener(ContextRefreshedEvent.class)
public void start() throws SQLException {
log.info("starting h2 console");
this.webServer = Server.createWebServer("-webPort", port, "-tcpAllowOthers").start();
}
@EventListener(ContextClosedEvent.class)
public void stop() {
log.info("stopping h2 console");
this.webServer.stop(); ;
}
}

125
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/PpomppuDatasourceConfiguration.java
View File

@@ -1,125 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.processor.configuration;
import com.myoa.engineering.crawl.ppomppu.processor.configuration.properties.DatasourceProperties;
import com.myoa.engineering.crawl.ppomppu.processor.configuration.properties.DatasourceProperties.DataSourcePropertiesUnit;
import com.myoa.engineering.crawl.ppomppu.processor.configuration.properties.HibernateProperties;
import com.myoa.engineering.crawl.ppomppu.processor.configuration.properties.HikariProperties;
import com.myoa.engineering.crawl.ppomppu.processor.domain.BaseScanDomain;
import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository.BaseScanRepository;
import com.zaxxer.hikari.HikariConfig;
import com.zaxxer.hikari.HikariDataSource;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
import javax.persistence.EntityManagerFactory;
import javax.sql.DataSource;
import lombok.NonNull;
import org.hibernate.cfg.AvailableSettings;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.boot.orm.jpa.EntityManagerFactoryBuilder;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.data.jpa.repository.config.EnableJpaRepositories;
import org.springframework.orm.jpa.JpaTransactionManager;
import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean;
import org.springframework.transaction.PlatformTransactionManager;
@Configuration
@EnableJpaRepositories(basePackageClasses = BaseScanRepository.class,
entityManagerFactoryRef = "ppomppuNotifierProcessorEntityManagerFactory",
transactionManagerRef = "ppomppuNotifierProcessorTransactionManager"
)
public class PpomppuDatasourceConfiguration {
private static final String DATA_SOURCE_UNIT_NAME = "ppn_mysql";
private final DatasourceProperties dataSourceProeprties;
private final HikariProperties hikariProperties;
private final HibernateProperties hibernateProperties;
public PpomppuDatasourceConfiguration(DatasourceProperties dataSourceProeprties,
HikariProperties hikariProperties,
HibernateProperties hibernateProperties) {
this.dataSourceProeprties = dataSourceProeprties;
this.hikariProperties = hikariProperties;
this.hibernateProperties = hibernateProperties;
}
@Bean(name = "ppomppuNotifierProcessorDataSource")
public DataSource dataSource() {
DataSourcePropertiesUnit dataSourcePropertiesUnit = dataSourceProeprties.find(DATA_SOURCE_UNIT_NAME);
final HikariConfig hikariConfig = new HikariConfig();
hikariConfig.setJdbcUrl(dataSourcePropertiesUnit.toCompletedJdbcUrl());
hikariConfig.setDriverClassName(dataSourcePropertiesUnit.getDriverClassName());
hikariConfig.setUsername(dataSourcePropertiesUnit.getUsername());
hikariConfig.setPassword(dataSourcePropertiesUnit.getPassword());
hikariConfig.setAutoCommit(hikariProperties.getAutoCommit());
hikariConfig.setMaximumPoolSize(hikariProperties.getMaximumPoolSize());
hikariConfig.setMinimumIdle(hikariProperties.getMinimumIdle());
if (hikariProperties.getMaximumPoolSize() > hikariProperties.getMinimumIdle()) {
hikariConfig.setIdleTimeout(hikariProperties.getIdleTimeout());
}
hikariConfig.setValidationTimeout(hikariProperties.getValidationTimeout());
hikariConfig.setConnectionTimeout(hikariProperties.getConnectionTimeout());
hikariConfig.setMaxLifetime(hikariProperties.getMaxLifetime());
final DataSource dataSource = new HikariDataSource(hikariConfig);
return dataSource;
}
@Bean("ppomppuNotifierProcessorEntityManagerFactory")
public LocalContainerEntityManagerFactoryBean entityManagerFactory(
EntityManagerFactoryBuilder builder,
@Qualifier("ppomppuNotifierProcessorDataSource") DataSource dataSource) {
return builder.dataSource(dataSource)
.packages(BaseScanDomain.class)
.properties(getPropsMap(DATA_SOURCE_UNIT_NAME))
.build();
}
@Bean("ppomppuNotifierProcessorTransactionManager")
public PlatformTransactionManager transactionManager(
@Qualifier("ppomppuNotifierProcessorEntityManagerFactory") EntityManagerFactory entityManagerFactory) {
return new JpaTransactionManager(entityManagerFactory);
}
public static Properties getProps(@NonNull HibernateProperties.HibernatePropertiesUnit hibernateProperties) {
Properties properties = new Properties();
properties.put(AvailableSettings.DIALECT, hibernateProperties.getDialect());
properties.put(AvailableSettings.FORMAT_SQL, hibernateProperties.getFormatSql());
properties.put(AvailableSettings.SHOW_SQL, hibernateProperties.getShowSql());
properties.put(AvailableSettings.HBM2DDL_AUTO, hibernateProperties.getHbm2ddlAuto());
properties.put(AvailableSettings.CONNECTION_PROVIDER_DISABLES_AUTOCOMMIT,
hibernateProperties.getDisableAutoCommit());
properties.put(AvailableSettings.IMPLICIT_NAMING_STRATEGY,
"org.springframework.boot.orm.jpa.hibernate.SpringImplicitNamingStrategy");
properties.put(AvailableSettings.PHYSICAL_NAMING_STRATEGY,
"org.springframework.boot.orm.jpa.hibernate.SpringPhysicalNamingStrategy");
properties.put(AvailableSettings.GENERATE_STATISTICS, "false");
properties.put(AvailableSettings.GLOBALLY_QUOTED_IDENTIFIERS, "true");
properties.put(AvailableSettings.GLOBALLY_QUOTED_IDENTIFIERS_SKIP_COLUMN_DEFINITIONS, "true");
properties.put(AvailableSettings.STATEMENT_BATCH_SIZE, "20");
properties.put(AvailableSettings.ORDER_INSERTS, "true");
properties.put(AvailableSettings.ORDER_UPDATES, "true");
properties.put(AvailableSettings.BATCH_VERSIONED_DATA, "true");
properties.put(AvailableSettings.USE_NEW_ID_GENERATOR_MAPPINGS, "false");
return properties;
}
public Map<String, String> getPropsMap(@NonNull String unitName) {
return convertPropertiestoMaps(getProps(hibernateProperties.find(unitName)));
}
public Map<String, String> convertPropertiestoMaps(Properties properties) {
Map<String, String> propertiesMap = new HashMap<>();
for (Enumeration<?> e = properties.propertyNames(); e.hasMoreElements(); ) {
String key = (String) e.nextElement();
propertiesMap.put(key, properties.getProperty(key));
}
return propertiesMap;
}
}

48
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/properties/DatasourceProperties.java
View File

@@ -1,48 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.processor.configuration.properties;
import com.myoa.engineering.crawl.ppomppu.support.util.ObjectUtil;
import java.util.List;
import lombok.Getter;
import lombok.Setter;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;
@Component
@Setter
@Getter
@ConfigurationProperties(prefix = "infra.database")
public class DatasourceProperties {
private List<DataSourcePropertiesUnit> units;
@Getter
@Setter
public static class DataSourcePropertiesUnit {
private String unitName;
private String schemaName;
private String connectionParameters;
private String datasourceUrl;
private Boolean isSimpleConnectionUrl;
private String username;
private String password;
private String driverClassName;
public String toCompletedJdbcUrl() {
if (ObjectUtil.isEmpty(isSimpleConnectionUrl) || isSimpleConnectionUrl == false) {
return String.format("%s/%s?%s", datasourceUrl, schemaName, connectionParameters);
}
return datasourceUrl;
}
}
public DataSourcePropertiesUnit find(String unitName) {
return units.stream()
.filter(e -> e.getUnitName().equals(unitName))
.findFirst()
.orElseThrow(
() -> new IllegalArgumentException(this.getClass().getName() + ": unitName Not found. " + unitName));
}
}

41
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/properties/HibernateProperties.java
View File

@@ -1,41 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.processor.configuration.properties;
import java.util.List;
import lombok.Getter;
import lombok.Setter;
import org.hibernate.dialect.MySQL8Dialect;
import org.hibernate.dialect.MySQLDialect;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;
@Component
@Setter
@Getter
@ConfigurationProperties(prefix = "hibernate")
public class HibernateProperties {
private List<HibernatePropertiesUnit> units;
@Getter
@Setter
public static class HibernatePropertiesUnit {
private String unitName;
private String dialect;
private String formatSql;
private String showSql;
private String hbm2ddlAuto;
private String disableAutoCommit;
}
public HibernatePropertiesUnit find(String unitName) {
return units.stream()
.filter(x -> x.getUnitName().equals(unitName))
.findFirst()
.orElseThrow(
() -> new IllegalArgumentException(this.getClass().getName() + ": unitName Not found. " + unitName));
}
}

22
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/properties/HikariProperties.java
View File

@@ -1,22 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.processor.configuration.properties;
import lombok.Getter;
import lombok.Setter;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;
@Component
@Setter
@Getter
@ConfigurationProperties(prefix = "spring.datasource.hikari")
public class HikariProperties {
private Integer minimumIdle;
private Integer maximumPoolSize;
private Integer idleTimeout;
private Integer validationTimeout;
private Integer connectionTimeout;
private Integer maxLifetime;
private Boolean autoCommit;
}

5
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/controller/CrawlAPIController.java
View File

@@ -1,6 +1,5 @@
package com.myoa.engineering.crawl.ppomppu.processor.controller;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
@@ -10,8 +9,8 @@ import com.myoa.engineering.crawl.ppomppu.processor.dto.FeedParsedResult;
import com.myoa.engineering.crawl.ppomppu.processor.service.MessageSenderService;
import com.myoa.engineering.crawl.ppomppu.processor.service.PpomppuArticleService;
import com.myoa.engineering.crawl.ppomppu.processor.service.PpomppuFeedService;
import com.myoa.engineering.crawl.ppomppu.support.dto.APIResponse;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import com.myoa.engineering.crawl.shopping.support.dto.APIResponse;
import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName;
import lombok.extern.slf4j.Slf4j;
import reactor.core.publisher.Mono;

26
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/Auditable.java
View File

@@ -1,26 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.processor.domain;
import java.io.Serializable;
import java.time.Instant;
import javax.persistence.Column;
import org.springframework.data.annotation.CreatedDate;
import org.springframework.data.annotation.LastModifiedDate;
/**
* Auditable
*
* @author Shin Woo-jin (woozu.shin@kakaoent.com)
* @since 2021-09-08
*/
public abstract class Auditable implements Serializable {
private static final long serialVersionUID = -7105030870015828551L;
@Column
@CreatedDate
private Instant createdAt;
@Column
@LastModifiedDate
private Instant modifiedAt;
}

10
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/BaseScanDomain.java
View File

@@ -1,10 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.processor.domain;
/**
* BaseScanDomain
*
* @author Shin Woo-jin (woozu.shin@kakaoent.com)
* @since 2021-09-08
*/
public interface BaseScanDomain {
}

66
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/PpomppuArticle.java
View File

@@ -1,66 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.processor.domain;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import java.time.Instant;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.EnumType;
import javax.persistence.Enumerated;
import javax.persistence.GeneratedValue;
import javax.persistence.GenerationType;
import javax.persistence.Id;
import javax.persistence.Table;
import lombok.Builder;
import lombok.Getter;
import lombok.NoArgsConstructor;
@Getter
@NoArgsConstructor
@Entity
@Table(name = "ppomppu_article")
public class PpomppuArticle extends Auditable {
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
private Long id;
@Column
private Long articleId;
@Column
@Enumerated(EnumType.STRING)
private PpomppuBoardName boardName;
@Column
private String articleUrl;
@Column
private String title;
@Column
private Integer hit;
@Column
private Integer recommended;
@Column
private Instant registeredAt;
@Builder
public PpomppuArticle(Long id, Long articleId, PpomppuBoardName boardName, String articleUrl,
String title, Integer recommended, Integer hit, Instant registeredAt) {
this.id = id;
this.articleId = articleId;
this.boardName = boardName;
this.articleUrl = articleUrl;
this.title = title;
this.recommended = recommended;
this.hit = hit;
this.registeredAt = registeredAt;
}
public PpomppuArticle updateBoardName(PpomppuBoardName boardName) {
this.boardName = boardName;
return this;
}
}

57
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/PpomppuBoardFeedStatus.java
View File

@@ -1,57 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.processor.domain;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import java.time.Instant;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.EnumType;
import javax.persistence.Enumerated;
import javax.persistence.GeneratedValue;
import javax.persistence.GenerationType;
import javax.persistence.Id;
import javax.persistence.Table;
import lombok.Builder;
import lombok.Getter;
import lombok.NoArgsConstructor;
@Getter
@NoArgsConstructor
@Entity
@Table(name = "ppomppu_board_feed_status")
public class PpomppuBoardFeedStatus extends Auditable {
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
private Long id;
@Column
private Long latestParsedArticleId;
@Column
@Enumerated(EnumType.STRING)
private PpomppuBoardName boardName;
@Column
private Instant updatedAt;
public static PpomppuBoardFeedStatus of(PpomppuBoardName boardName, Long latestArticleId) {
return PpomppuBoardFeedStatus.builder()
.boardName(boardName)
.latestParsedArticleId(latestArticleId)
.updatedAt(Instant.now())
.build();
}
public void updateArticleId(Long latestArticleId) {
this.updatedAt = Instant.now();
this.latestParsedArticleId = latestArticleId;
}
@Builder
public PpomppuBoardFeedStatus(Long id, Long latestParsedArticleId, PpomppuBoardName boardName, Instant updatedAt) {
this.id = id;
this.latestParsedArticleId = latestParsedArticleId;
this.boardName = boardName;
this.updatedAt = updatedAt;
}
}

32
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/PublishedHistory.java
View File

@@ -1,32 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.processor.domain;
import java.time.Instant;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.GeneratedValue;
import javax.persistence.GenerationType;
import javax.persistence.Id;
import javax.persistence.Table;
import lombok.Getter;
import lombok.NoArgsConstructor;
@Getter
@NoArgsConstructor
@Entity
@Table(name = "published_history")
public class PublishedHistory extends Auditable {
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
private Long id;
@Column
private Long userId;
@Column
private String boardNameList;
@Column
private Instant publishedAt;
}

32
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/SubscribedBoard.java
View File

@@ -1,32 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.processor.domain;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.EnumType;
import javax.persistence.Enumerated;
import javax.persistence.GeneratedValue;
import javax.persistence.GenerationType;
import javax.persistence.Id;
import javax.persistence.Table;
import lombok.Getter;
import lombok.NoArgsConstructor;
@Getter
@NoArgsConstructor
@Entity
@Table(name = "subscribed_board")
public class SubscribedBoard extends Auditable {
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
private Long id;
@Column
private Long userId;
@Column
@Enumerated(EnumType.STRING)
private PpomppuBoardName boardName;
}

29
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/SubscribedUser.java
View File

@@ -1,29 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.processor.domain;
import java.time.Instant;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.GeneratedValue;
import javax.persistence.GenerationType;
import javax.persistence.Id;
import javax.persistence.Table;
import lombok.Getter;
import lombok.NoArgsConstructor;
@Getter
@NoArgsConstructor
@Entity
@Table(name = "subscribed_user")
public class SubscribedUser extends Auditable{
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
private Long id;
@Column
private Long userId;
@Column
private Instant registeredAt;
}

45
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/FeedParsedResult.java
View File

@@ -1,45 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.processor.dto;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import java.io.Serializable;
import java.time.Instant;
import lombok.Builder;
import lombok.Getter;
import lombok.NoArgsConstructor;
/**
* FeedParsedResult
*
* @author Shin Woo-jin (woozu.shin@kakaoent.com)
* @since 2021-09-08
*/
@Getter
@NoArgsConstructor
public class FeedParsedResult implements Serializable {
private static final long serialVersionUID = -3771310078623481348L;
private PpomppuBoardName boardName;
private Instant requestedAt;
private Instant processedAt;
@Builder
public FeedParsedResult(PpomppuBoardName boardName, Instant requestedAt, Instant processedAt) {
this.boardName = boardName;
this.requestedAt = requestedAt;
this.processedAt = processedAt;
}
public static FeedParsedResult of(PpomppuBoardName boardName) {
return FeedParsedResult.builder()
.boardName(boardName)
.requestedAt(Instant.now())
.build();
}
public FeedParsedResult done() {
this.processedAt = Instant.now();
return this;
}
}

10
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/PpomppuArticleDTO.java
View File

@@ -1,10 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.processor.dto;
/**
* PpomppuArticle
*
* @author Shin Woo-jin (woozu.shin@kakaoent.com)
* @since 2021-09-08
*/
public class PpomppuArticleDTO {
}

53
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/PpomppuArticleTransformer.java
View File

@@ -1,53 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.processor.dto;
import java.time.Instant;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.util.List;
import java.util.function.Function;
import java.util.stream.Collectors;
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
import com.myoa.engineering.crawl.ppomppu.support.dto.SimpleMessageDTO;
/**
* PpomppuArticleTransformer
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-11-21
*
*/
public final class PpomppuArticleTransformer {
private PpomppuArticleTransformer() {}
private static final String MESSAGE_FORMAT_V1 = "%s)) `%s` <%s:LINK>";
private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")
.withZone(ZoneId.of("Asia/Seoul"));
public static final Function<PpomppuArticle, SimpleMessageDTO> TRANSFORM_TO_MESSAGE_DTO = entity ->
SimpleMessageDTO.builder()
.requestedAt(Instant.now())
.publishedAt(entity.getRegisteredAt())
.title(String.format(MESSAGE_FORMAT_V1, entity.getBoardName().getMenuName(), entity.getTitle()))
.body(entity.getArticleUrl())
.build();
// https://stackoverflow.com/questions/24882927/using-streams-to-convert-a-list-of-objects-into-a-string-obtained-from-the-tostr
public static SimpleMessageDTO transform(List<PpomppuArticle> articles) {
Instant requestedAt = Instant.now();
String body = articles.stream()
.map(PpomppuArticleTransformer::convertToInlineMessage)
.collect(Collectors.joining("\n\n"));
return SimpleMessageDTO.builder()
.requestedAt(requestedAt)
.title(DATE_TIME_FORMATTER.format(requestedAt))
.body(body)
.build();
}
public static String convertToInlineMessage(PpomppuArticle article) {
return String.format(MESSAGE_FORMAT_V1,
article.getBoardName().getMenuName(), article.getTitle(), article.getArticleUrl());
}
}

19
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/constant/WebClientPropertiesUnitName.java
View File

@@ -1,19 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.processor.dto.constant;
import lombok.AllArgsConstructor;
import lombok.Getter;
/**
* WebClientPropertiesUnitName
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-11-18
*
*/
@Getter
@AllArgsConstructor
public enum WebClientPropertiesUnitName {
PPOMPPU_NOTIFIER_SENDER_API("ppn-sender-api"),
;
private String unitName;
}

56
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/client/MessageSenderAPIClient.java
View File

@@ -1,56 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.processor.infrastructure.client;
import org.springframework.core.ParameterizedTypeReference;
import org.springframework.http.HttpHeaders;
import org.springframework.http.MediaType;
import org.springframework.stereotype.Component;
import org.springframework.web.reactive.function.client.WebClient;
import org.springframework.web.reactive.function.client.WebClientRequestException;
import com.myoa.engineering.crawl.ppomppu.processor.dto.constant.WebClientPropertiesUnitName;
import com.myoa.engineering.crawl.ppomppu.support.dto.SimpleMessageDTO;
import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebClientFilterFactory;
import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebFluxExchangeStragiesFactory;
import com.myoa.engineering.crawl.ppomppu.support.webclient.properties.WebClientProperties;
import com.myoa.engineering.crawl.ppomppu.support.webclient.properties.WebClientProperties.WebClientPropertiesUnit;
import lombok.extern.slf4j.Slf4j;
import reactor.core.publisher.Mono;
import reactor.core.scheduler.Schedulers;
/**
* PpomppuNotifierSenderAPIClient
*
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-11-17
*/
@Slf4j
@Component
public class MessageSenderAPIClient {
private final WebClient webClient;
public MessageSenderAPIClient(WebClientProperties webClientProperties) {
WebClientPropertiesUnit webClientPropertiesUnit =
webClientProperties.find(WebClientPropertiesUnitName.PPOMPPU_NOTIFIER_SENDER_API.getUnitName());
this.webClient = WebClient.builder()
.baseUrl(webClientPropertiesUnit.getBaseUrl())
.exchangeStrategies(WebFluxExchangeStragiesFactory.ofDefault())
.defaultHeader(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_VALUE)
// .filter(WebClientFilterFactory.logRequest())
// .filter(WebClientFilterFactory.logResponse())
.build();
}
public Mono<String> sendMessageToSlack(SimpleMessageDTO dto) {
return webClient.post()
.uri("/api/v1/messages/sendMessage/messengers/slack")
.bodyValue(dto)
.exchangeToMono(e -> e.bodyToMono(new ParameterizedTypeReference<String>() {}))
.publishOn(Schedulers.boundedElastic())
.onErrorResume(WebClientRequestException.class, t -> {
log.info("Exception occured, ignoring. : {}", t.getClass().getSimpleName());
return Mono.empty();
});
}
}

47
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/client/PpomppuBoardFeedRetriever.java
View File

@@ -1,47 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.processor.infrastructure.client;
import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebClientFilterFactory;
import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebFluxExchangeStragiesFactory;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import lombok.extern.slf4j.Slf4j;
import org.springframework.core.ParameterizedTypeReference;
import org.springframework.stereotype.Component;
import org.springframework.web.reactive.function.client.WebClient;
import org.springframework.web.reactive.function.client.WebClientRequestException;
import reactor.core.publisher.Mono;
import reactor.core.scheduler.Schedulers;
/**
* PpomppuBoardFeedRetriever
*
* @author Shin Woo-jin (woozu.shin@kakaoent.com)
* @since 2021-09-08
*/
@Slf4j
@Component
public class PpomppuBoardFeedRetriever {
private final WebClient webClient;
public PpomppuBoardFeedRetriever(WebClient.Builder webClientBuilder) {
this.webClient = webClientBuilder.baseUrl(PpomppuBoardName.PPOMPPU_URL)
.exchangeStrategies(WebFluxExchangeStragiesFactory.ofTextHtml())
.filter(WebClientFilterFactory.logRequest())
.filter(WebClientFilterFactory.logResponse())
.build();
}
public Mono<String> getHtml(String uri) {
return webClient.get()
.uri(uri)
.exchangeToMono(e -> e.bodyToMono(String.class))
.publishOn(Schedulers.boundedElastic())
.onErrorResume(WebClientRequestException.class, t -> {
log.info("Exception occured, ignoring. : {}", t.getClass().getSimpleName());
return Mono.empty();
});
// .doOnNext(e -> log.info("[getHtml] {}", e));
}
}

4
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/repository/BaseScanRepository.java
View File

@@ -1,4 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository;
public interface BaseScanRepository {
}

10
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/repository/PpomppuArticleRepository.java
View File

@@ -1,10 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository;
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.stereotype.Repository;
@Repository
public interface PpomppuArticleRepository extends JpaRepository<PpomppuArticle, Long> {
}

14
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/repository/PpomppuBoardFeedStatusRepository.java
View File

@@ -1,14 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository;
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuBoardFeedStatus;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import java.util.Optional;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.stereotype.Repository;
@Repository
public interface PpomppuBoardFeedStatusRepository extends JpaRepository<PpomppuBoardFeedStatus, Long> {
Optional<PpomppuBoardFeedStatus> findByBoardName(PpomppuBoardName boardName);
}

4
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/PpomppuArticleParser.java → processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/parser/PpomppuArticleParser.java
View File

@@ -1,4 +1,4 @@
package com.myoa.engineering.crawl.ppomppu.processor.dto;
package com.myoa.engineering.crawl.ppomppu.processor.parser;
import java.time.Instant;
import java.time.ZoneId;
@@ -8,7 +8,7 @@ import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName;
/**
* PpomppuArticleTransformer

38
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/MessageSenderService.java
View File

@@ -1,38 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.processor.service;
import java.util.List;
import org.springframework.stereotype.Service;
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
import com.myoa.engineering.crawl.ppomppu.processor.dto.PpomppuArticleTransformer;
import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.client.MessageSenderAPIClient;
import lombok.extern.slf4j.Slf4j;
import reactor.core.publisher.Mono;
/**
* MessageSenderService
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-11-21
*
*/
@Slf4j
@Service
public class MessageSenderService {
private final MessageSenderAPIClient messageSenderAPIClient;
public MessageSenderService(MessageSenderAPIClient messageSenderAPIClient) {
this.messageSenderAPIClient = messageSenderAPIClient;
}
public Mono<String> sendMessageToSlack(PpomppuArticle article) {
return messageSenderAPIClient.sendMessageToSlack(PpomppuArticleTransformer.TRANSFORM_TO_MESSAGE_DTO.apply(article));
}
public Mono<String> sendMessageToSlack(List<PpomppuArticle> articles) {
return messageSenderAPIClient.sendMessageToSlack(PpomppuArticleTransformer.transform(articles));
}
}

66
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/PpomppuArticleService.java
View File

@@ -1,66 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.processor.service;
import java.util.List;
import java.util.Optional;
import java.util.stream.Collectors;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuBoardFeedStatus;
import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository.PpomppuArticleRepository;
import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository.PpomppuBoardFeedStatusRepository;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Service
public class PpomppuArticleService {
private final PpomppuArticleRepository ppomppuArticleRepository;
private final PpomppuBoardFeedStatusRepository ppomppuBoardFeedStatusRepository;
public PpomppuArticleService(PpomppuArticleRepository ppomppuArticleRepository,
PpomppuBoardFeedStatusRepository ppomppuBoardFeedStatusRepository) {
this.ppomppuArticleRepository = ppomppuArticleRepository;
this.ppomppuBoardFeedStatusRepository = ppomppuBoardFeedStatusRepository;
}
@Transactional(readOnly = true)
public List<PpomppuArticle> filterOnlyNewArticles(PpomppuBoardName boardName, List<PpomppuArticle> articles) {
Optional<PpomppuBoardFeedStatus> boardFeedStatus = ppomppuBoardFeedStatusRepository.findByBoardName(boardName);
Long latestArticleId = boardFeedStatus.map(PpomppuBoardFeedStatus::getLatestParsedArticleId)
.orElse(0L);
log.info("latestArticleId : {}", latestArticleId);
return articles.stream()
.filter(e -> e.getArticleId().compareTo(latestArticleId) > 0)
.collect(Collectors.toList());
}
@Transactional
public List<PpomppuArticle> save(PpomppuBoardName boardName, List<PpomppuArticle> articles) {
Long latestArticleId = articles.stream()
.map(PpomppuArticle::getArticleId)
.max(Long::compareTo)
.orElse(0L);
// save PpomppuBoardFeedStatus
Optional<PpomppuBoardFeedStatus> boardFeedStatus = ppomppuBoardFeedStatusRepository.findByBoardName(boardName);
log.info("boardName: {}, isPresent?: {}", boardName, boardFeedStatus.isPresent());
boardFeedStatus.ifPresentOrElse(e -> {
if (latestArticleId.longValue() > 0L) {
e.updateArticleId(latestArticleId);
ppomppuBoardFeedStatusRepository.save(e);
}
},
() -> ppomppuBoardFeedStatusRepository.save(PpomppuBoardFeedStatus.of(boardName,
latestArticleId)));
// save real articles.
return ppomppuArticleRepository.saveAll(articles);
}
}

59
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/PpomppuFeedService.java
View File

@@ -1,59 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.processor.service;
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
import com.myoa.engineering.crawl.ppomppu.processor.dto.PpomppuArticleParser;
import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.client.PpomppuBoardFeedRetriever;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import java.util.List;
import lombok.extern.slf4j.Slf4j;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Element;
import org.springframework.stereotype.Component;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
/**
* PpomppuFeedService
*
* @author Shin Woo-jin (woozu.shin@kakaoent.com)
* @since 2021-09-08
*/
@Slf4j
@Component
public class PpomppuFeedService {
private final PpomppuBoardFeedRetriever ppomppuBoardFeedRetriever;
public PpomppuFeedService(PpomppuBoardFeedRetriever ppomppuBoardFeedRetriever) {
this.ppomppuBoardFeedRetriever = ppomppuBoardFeedRetriever;
}
public Mono<List<PpomppuArticle>> getArticles(PpomppuBoardName boardName) {
final Mono<String> html = ppomppuBoardFeedRetriever.getHtml(boardName.getResourcePath());
final Mono<Element> tbody = extractTbodyFromHtml(html);
// .doOnNext(e -> log.info("pre tbody - {}", e.html()));
return extractArticlesFromTbody(tbody).map(this::convertFromElement)
.map(e -> e.updateBoardName(boardName))
// .doOnNext(e -> log.info("parsed Result: {}", e))
.collectList();
}
private Mono<Element> extractTbodyFromHtml(Mono<String> html) {
return html.map(Jsoup::parse)
.mapNotNull(e -> e.getElementById("revolution_main_table"))
.map(e -> e.getElementsByTag("tbody"))
// .doOnNext(e -> log.info("tbody - {}", e.html()))
.map(e -> e.stream()
.findFirst()
.orElseThrow(() -> new IndexOutOfBoundsException("no tbody")));
}
private Flux<Element> extractArticlesFromTbody(Mono<Element> tbody) {
return Flux.concat(tbody.flatMapMany(e -> Flux.fromArray(e.select("tr.list0").toArray(Element[]::new))),
tbody.flatMapMany(e -> Flux.fromArray(e.select("tr.list1").toArray(Element[]::new))));
}
private PpomppuArticle convertFromElement(Element element) {
return PpomppuArticleParser.toArticle(element.getElementsByTag("td"));
}
}

121
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/util/ObjectUtil.java
View File

@@ -1,121 +0,0 @@
/*
* Copyright (c) 2019 LINE Corporation. All rights reserved.
* LINE Corporation PROPRIETARY/CONFIDENTIAL. Use is subject to license terms.
*/
package com.myoa.engineering.crawl.ppomppu.processor.util;
import java.util.Arrays;
import java.util.Collection;
/**
* NumberUtils
*
* @author Shin Woo-jin (lp12254@linecorp.com)
* @since 2019-10-28
*/
public final class ObjectUtil {
private ObjectUtil() {
}
/**
* Check if given object is null.
* <code>
* e == object == &gt; false e == null == &gt; true
* </code>
*
* @param e Target object
* @param <E> Unfixed specific type. If you want restrict specific interface, Copy and extend qualifier.
* @return Is null given object?
*/
public static <E> boolean isNullObject(final E e) {
return e == null;
}
/**
* Check if given object is not null.
* <code>
* e == object == &gt; false e == null == &gt; true
* </code>
*
* @param e Target object
* @param <E> Unfixed specific type. If you want restrict specific interface, Copy and extend qualifier.
* @return Is not null given object?
*/
public static <E> boolean isNotEmpty(final E e) {
return !isNullObject(e);
}
/**
* Check if there are any null object in given objects.
* <code>
* args == object = &gt; false args == object, object = &gt; false args == null, null, object = &gt; true args
* == null = &gt; true args == null, null = &gt; true
* </code>
*
* @param args Want to check objects that have null.
* @return Is there objects array has null?
*/
public static boolean hasNullObject(Object... args) {
return Arrays.stream(args).anyMatch(ObjectUtil::isNullObject);
}
/**
* Check given objects are not empty.
* <code>
* args == object = &gt; true args == object, object = &gt; true args == null, null, object = &gt; false args
* == null = &gt; false args == null, null = &gt; false
* </code>
*
* @param args Want to check objects that have null.
* @return Is there objects array has null?
*/
public static boolean hasAllObject(Object... args) {
return Arrays.stream(args).noneMatch(ObjectUtil::isNullObject);
}
/**
* Check if there are all null object in given objects.
* <code>
* args == object = &gt; false args == object, object = &gt; false args == null, null, object = &gt; false args
* == null = &gt; true args == null, null = &gt; true
* </code>
*
* @param args Want to check objects that have null.
* @return Is there null all of given objects?
*/
public static boolean hasAllNullObjects(final Object... args) {
return Arrays.stream(args).allMatch(ObjectUtil::isNullObject);
}
/**
* Check if given collection object is null or empty collecton.
* <code>
* e == null = &gt; true e == emptyCollection = &gt; true e == hasElement = &gt; false
* </code>
*
* @param e e is must be Collection object
* @param <E> E is must be extended Collection Class
* @return boolean. given collection is null or empty?
*/
public static <E extends Collection<?>> boolean isNullOrEmptyCollection(final E e) {
return e == null || e.isEmpty();
}
/**
* Get collection's size. Even it pointed null
*
* @param e e is must be Collection object
* @param <E> E is must be extended Collection Class
* @return integer value. given collection's size.
*/
public static <E extends Collection<?>> int getCollectionSize(final E e) {
if (isNullOrEmptyCollection(e)) {
return 0;
}
return e.size();
}
}

1100
processor/src/test/resources/testdata/zboard/file1.html vendored Normal file
View File

File diff suppressed because it is too large Load Diff