From 378e6ef68e5c0e0fe4aa245264d3281426c10fed Mon Sep 17 00:00:00 2001 From: woozu-shin Date: Thu, 9 May 2024 09:32:39 +0900 Subject: [PATCH] [NO-ISSUE] Change references --- build.gradle | 10 +- .../controller/CrawlAPIController.java | 16 +-- .../processor/dto/PpomppuArticleParser.java | 6 +- .../receiver/scheduler/ParseEventEmitter.java | 6 +- .../MessageSenderAPIController.java | 16 +-- shopping-crawler/build.gradle | 3 +- ...hoppingCrawlerDatasourceConfiguration.java | 10 +- .../shopping/domain/entity/Auditable.java | 5 +- .../domain/entity/v1/PpomppuArticle.java | 17 +-- .../entity/v1/PpomppuBoardFeedStatus.java | 2 +- .../domain/entity/v1/PublishedHistory.java | 2 +- .../domain/entity/v1/SubscribedBoard.java | 2 +- .../domain/entity/v1/SubscribedUser.java | 2 +- .../shopping/domain/entity/v2/AppUser.java | 2 +- .../shopping/domain/entity/v2/Article.java | 2 +- .../domain/entity/v2/SubscribedKeyword.java | 2 +- .../shopping/dto/PpomppuArticleParseDTO.java | 13 +- .../dto/PpomppuArticleTransformer.java | 35 +++--- .../dto/slack/SlackBaseMessageBlock.java | 2 +- .../dto/slack/SlackImageMessageBlock.java | 2 +- .../shopping/dto/slack/SlackMessageBlock.java | 2 +- .../dto/slack/SlackMessageBlockType.java | 2 +- .../shopping/dto/slack/SlackMessageDTO.java | 11 +- .../dto/slack/SlackSectionMessageBlock.java | 5 +- .../infra/client/MessageSenderAPIClient.java | 70 ----------- .../MongeShoppingBotSlackMessageSender.java | 52 -------- .../client/slack/SlackMessageSender.java | 47 -------- .../service/ArticleCommandService.java | 2 +- .../service/MessageSenderService.java | 44 ------- .../shopping/service/PpomppuFeedService.java | 66 ----------- .../parser/PpomppuArticleParserV2Test.java | 112 ------------------ .../support/dto/code/PpomppuBoardName.java | 51 -------- .../shopping/support/dto/BlockMessageDTO.java | 15 ++- .../crawl/shopping/support/util/DateUtil.java | 2 +- 34 files changed, 93 insertions(+), 543 deletions(-) delete mode 100644 shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/MessageSenderAPIClient.java delete mode 100644 shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/slack/MongeShoppingBotSlackMessageSender.java delete mode 100644 shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/slack/SlackMessageSender.java delete mode 100644 shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/MessageSenderService.java delete mode 100644 shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/PpomppuFeedService.java delete mode 100644 support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/dto/code/PpomppuBoardName.java diff --git a/build.gradle b/build.gradle index f51210c..118203c 100644 --- a/build.gradle +++ b/build.gradle @@ -1,13 +1,13 @@ plugins { - id 'org.springframework.boot' version '2.5.4' - id 'io.spring.dependency-management' version '1.0.11.RELEASE' id 'java' id 'idea' + id 'org.springframework.boot' version '2.5.4' + id 'io.spring.dependency-management' version '1.0.11.RELEASE' } -group = 'com.myoa.engineering.crawl.ppomppu' +group = 'com.myoa.engineering.crawl.shopping' version = '1.1.1' -sourceCompatibility = '11' +sourceCompatibility = JavaVersion.VERSION_17 configurations { compileOnly { @@ -20,7 +20,7 @@ repositories { } allprojects { - group = 'com.myoa.engineering.crawl.ppomppu' + group = 'com.myoa.engineering.crawl.shopping' version = '1.1.1' apply plugin: 'java' diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/controller/CrawlAPIController.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/controller/CrawlAPIController.java index 6a0ac18..ce48628 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/controller/CrawlAPIController.java +++ b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/controller/CrawlAPIController.java @@ -1,4 +1,4 @@ -package com.myoa.engineering.crawl.ppomppu.processor.controller; +package com.myoa.engineering.crawl.shopping.controller; import java.util.List; @@ -7,13 +7,13 @@ import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; -import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle; -import com.myoa.engineering.crawl.ppomppu.processor.dto.FeedParsedResult; -import com.myoa.engineering.crawl.ppomppu.processor.service.MessageSenderService; -import com.myoa.engineering.crawl.ppomppu.processor.service.PpomppuArticleService; -import com.myoa.engineering.crawl.ppomppu.processor.service.PpomppuFeedService; -import com.myoa.engineering.crawl.ppomppu.support.dto.APIResponse; -import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName; +import com.myoa.engineering.crawl.shopping.processor.domain.PpomppuArticle; +import com.myoa.engineering.crawl.shopping.processor.dto.FeedParsedResult; +import com.myoa.engineering.crawl.shopping.processor.service.MessageSenderService; +import com.myoa.engineering.crawl.shopping.processor.service.PpomppuArticleService; +import com.myoa.engineering.crawl.shopping.processor.service.PpomppuFeedService; +import com.myoa.engineering.crawl.shopping.support.dto.APIResponse; +import com.myoa.engineering.crawl.shopping.support.dto.code.PpomppuBoardName; import lombok.extern.slf4j.Slf4j; import reactor.core.publisher.Mono; diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/PpomppuArticleParser.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/PpomppuArticleParser.java index 35214fb..55a9cef 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/PpomppuArticleParser.java +++ b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/PpomppuArticleParser.java @@ -1,10 +1,10 @@ -package com.myoa.engineering.crawl.ppomppu.processor.dto; +package com.myoa.engineering.crawl.shopping.dto; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; -import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle; -import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName; +import com.myoa.engineering.crawl.shopping.processor.domain.PpomppuArticle; +import com.myoa.engineering.crawl.shopping.support.dto.code.PpomppuBoardName; /** * PpomppuArticleTransformer diff --git a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/scheduler/ParseEventEmitter.java b/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/scheduler/ParseEventEmitter.java index e164cf5..f532934 100644 --- a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/scheduler/ParseEventEmitter.java +++ b/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/scheduler/ParseEventEmitter.java @@ -1,4 +1,4 @@ -package com.myoa.engineering.crawl.ppomppu.receiver.scheduler; +package com.myoa.engineering.crawl.shopping.receiver.scheduler; import java.util.Arrays; @@ -6,8 +6,8 @@ import org.springframework.scheduling.annotation.EnableScheduling; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; -import com.myoa.engineering.crawl.ppomppu.receiver.service.ProcessorAPIService; -import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName; +import com.myoa.engineering.crawl.shopping.receiver.service.ProcessorAPIService; +import com.myoa.engineering.crawl.shopping.support.dto.code.PpomppuBoardName; import lombok.extern.slf4j.Slf4j; diff --git a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/controller/MessageSenderAPIController.java b/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/controller/MessageSenderAPIController.java index 06010e6..8d6e18c 100644 --- a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/controller/MessageSenderAPIController.java +++ b/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/controller/MessageSenderAPIController.java @@ -1,17 +1,17 @@ -package com.myoa.engineering.crawl.ppomppu.sender.controller; +package com.myoa.engineering.crawl.shopping.controller; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; -import com.myoa.engineering.crawl.ppomppu.sender.dto.SlackBaseMessageBlock; -import com.myoa.engineering.crawl.ppomppu.sender.dto.SlackMessageDTO; -import com.myoa.engineering.crawl.ppomppu.sender.infrastructure.client.MongeShoppingBotSlackMessageSender; -import com.myoa.engineering.crawl.ppomppu.support.dto.APIResponse; -import com.myoa.engineering.crawl.ppomppu.support.dto.BlockMessageDTO; -import com.myoa.engineering.crawl.ppomppu.support.dto.SimpleMessageDTO; -import com.myoa.engineering.crawl.ppomppu.support.util.ObjectMapperFactory; +import com.myoa.engineering.crawl.shopping.sender.dto.SlackBaseMessageBlock; +import com.myoa.engineering.crawl.shopping.sender.dto.SlackMessageDTO; +import com.myoa.engineering.crawl.shopping.sender.infrastructure.client.MongeShoppingBotSlackMessageSender; +import com.myoa.engineering.crawl.shopping.support.dto.APIResponse; +import com.myoa.engineering.crawl.shopping.support.dto.BlockMessageDTO; +import com.myoa.engineering.crawl.shopping.support.dto.SimpleMessageDTO; +import com.myoa.engineering.crawl.shopping.support.util.ObjectMapperFactory; import lombok.extern.slf4j.Slf4j; import reactor.core.publisher.Mono; diff --git a/shopping-crawler/build.gradle b/shopping-crawler/build.gradle index 89c7625..1aa8ad3 100644 --- a/shopping-crawler/build.gradle +++ b/shopping-crawler/build.gradle @@ -1,7 +1,7 @@ dependencies { developmentOnly 'org.springframework.boot:spring-boot-devtools' runtimeOnly 'com.h2database:h2' - runtimeOnly 'com.mysql:mysql-connector-j' + runtimeOnly 'com.mysql:mysql-connector-j:8.4.0' compileOnly 'org.projectlombok:lombok' implementation project(':support') @@ -17,7 +17,6 @@ dependencies { implementation 'org.springframework.boot:spring-boot-configuration-processor' implementation 'org.springframework.cloud:spring-cloud-starter-config' implementation 'org.springframework.boot:spring-boot-starter-actuator' - implementation 'com.rometools:rome:2.1.0' implementation 'org.jsoup:jsoup:1.17.2' implementation 'com.h2database:h2:2.2.224' implementation "org.springframework.cloud:spring-cloud-starter-openfeign" diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/datasource/ShoppingCrawlerDatasourceConfiguration.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/datasource/ShoppingCrawlerDatasourceConfiguration.java index bd83f05..6c5afe3 100644 --- a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/datasource/ShoppingCrawlerDatasourceConfiguration.java +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/datasource/ShoppingCrawlerDatasourceConfiguration.java @@ -7,13 +7,12 @@ import com.myoa.engineering.crawl.shopping.domain.entity.BaseScanDomain; import com.myoa.engineering.crawl.shopping.infra.repository.BaseScanRepository; import com.zaxxer.hikari.HikariConfig; import com.zaxxer.hikari.HikariDataSource; -import jakarta.persistence.EntityManagerFactory; import lombok.NonNull; -import org.hibernate.boot.model.naming.CamelCaseToUnderscoresNamingStrategy; -import org.hibernate.boot.model.naming.ImplicitNamingStrategyJpaCompliantImpl; import org.hibernate.cfg.AvailableSettings; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.orm.jpa.EntityManagerFactoryBuilder; +import org.springframework.boot.orm.jpa.hibernate.SpringImplicitNamingStrategy; +import org.springframework.boot.orm.jpa.hibernate.SpringPhysicalNamingStrategy; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.data.jpa.repository.config.EnableJpaAuditing; @@ -22,6 +21,7 @@ import org.springframework.orm.jpa.JpaTransactionManager; import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean; import org.springframework.transaction.PlatformTransactionManager; +import javax.persistence.EntityManagerFactory; import javax.sql.DataSource; import java.util.Enumeration; import java.util.HashMap; @@ -96,8 +96,8 @@ public class ShoppingCrawlerDatasourceConfiguration { properties.put(AvailableSettings.SHOW_SQL, hibernateProperties.getShowSql()); properties.put(AvailableSettings.HBM2DDL_AUTO, hibernateProperties.getHbm2ddlAuto()); properties.put(AvailableSettings.CONNECTION_PROVIDER_DISABLES_AUTOCOMMIT, hibernateProperties.getDisableAutoCommit()); - properties.put(AvailableSettings.IMPLICIT_NAMING_STRATEGY, ImplicitNamingStrategyJpaCompliantImpl.class.getName()); - properties.put(AvailableSettings.PHYSICAL_NAMING_STRATEGY, CamelCaseToUnderscoresNamingStrategy.class.getName()); + properties.put(AvailableSettings.IMPLICIT_NAMING_STRATEGY, SpringImplicitNamingStrategy.class.getName()); + properties.put(AvailableSettings.PHYSICAL_NAMING_STRATEGY, SpringPhysicalNamingStrategy.class.getName()); properties.put(AvailableSettings.GENERATE_STATISTICS, "false"); // properties.put(AvailableSettings.GLOBALLY_QUOTED_IDENTIFIERS, "true"); // properties.put(AvailableSettings.GLOBALLY_QUOTED_IDENTIFIERS_SKIP_COLUMN_DEFINITIONS, "true"); diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/Auditable.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/Auditable.java index ce5c073..59b51a4 100644 --- a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/Auditable.java +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/Auditable.java @@ -3,10 +3,11 @@ package com.myoa.engineering.crawl.shopping.domain.entity; import lombok.Getter; import org.springframework.data.annotation.CreatedDate; import org.springframework.data.annotation.LastModifiedDate; - -import jakarta.persistence.*; import org.springframework.data.jpa.domain.support.AuditingEntityListener; +import javax.persistence.Column; +import javax.persistence.EntityListeners; +import javax.persistence.MappedSuperclass; import java.io.Serializable; import java.time.Instant; diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v1/PpomppuArticle.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v1/PpomppuArticle.java index ad68260..e7dee7c 100644 --- a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v1/PpomppuArticle.java +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v1/PpomppuArticle.java @@ -1,19 +1,14 @@ -package com.myoa.engineering.crawl.ppomppu.processor.domain; +package com.myoa.engineering.crawl.shopping.domain.entity.v1; -import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName; -import java.time.Instant; -import javax.persistence.Column; -import javax.persistence.Entity; -import javax.persistence.EnumType; -import javax.persistence.Enumerated; -import javax.persistence.GeneratedValue; -import javax.persistence.GenerationType; -import javax.persistence.Id; -import javax.persistence.Table; +import com.myoa.engineering.crawl.shopping.domain.entity.Auditable; +import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName; import lombok.Builder; import lombok.Getter; import lombok.NoArgsConstructor; +import javax.persistence.*; +import java.time.Instant; + @Getter @NoArgsConstructor @Entity diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v1/PpomppuBoardFeedStatus.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v1/PpomppuBoardFeedStatus.java index 1ff7d5e..18ec466 100644 --- a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v1/PpomppuBoardFeedStatus.java +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v1/PpomppuBoardFeedStatus.java @@ -2,11 +2,11 @@ package com.myoa.engineering.crawl.shopping.domain.entity.v1; import com.myoa.engineering.crawl.shopping.domain.entity.Auditable; import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName; -import jakarta.persistence.*; import lombok.Builder; import lombok.Getter; import lombok.NoArgsConstructor; +import javax.persistence.*; import java.time.Instant; @Getter diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v1/PublishedHistory.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v1/PublishedHistory.java index 46d39dd..d821fbf 100644 --- a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v1/PublishedHistory.java +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v1/PublishedHistory.java @@ -4,7 +4,7 @@ import com.myoa.engineering.crawl.shopping.domain.entity.Auditable; import lombok.Getter; import lombok.NoArgsConstructor; -import jakarta.persistence.*; +import javax.persistence.*; import java.time.Instant; @Getter diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v1/SubscribedBoard.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v1/SubscribedBoard.java index ebe0e10..0ed69da 100644 --- a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v1/SubscribedBoard.java +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v1/SubscribedBoard.java @@ -5,7 +5,7 @@ import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName import lombok.Getter; import lombok.NoArgsConstructor; -import jakarta.persistence.*; +import javax.persistence.*; @Getter @NoArgsConstructor diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v1/SubscribedUser.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v1/SubscribedUser.java index a8dc32e..12e7669 100644 --- a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v1/SubscribedUser.java +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v1/SubscribedUser.java @@ -4,7 +4,7 @@ import com.myoa.engineering.crawl.shopping.domain.entity.Auditable; import lombok.Getter; import lombok.NoArgsConstructor; -import jakarta.persistence.*; +import javax.persistence.*; import java.time.Instant; @Getter diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v2/AppUser.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v2/AppUser.java index fc31469..a5d0959 100644 --- a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v2/AppUser.java +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v2/AppUser.java @@ -1,7 +1,7 @@ package com.myoa.engineering.crawl.shopping.domain.entity.v2; import com.myoa.engineering.crawl.shopping.domain.entity.Auditable; -import jakarta.persistence.*; +import javax.persistence.*; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Getter; diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v2/Article.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v2/Article.java index a6e7a3e..4021435 100644 --- a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v2/Article.java +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v2/Article.java @@ -2,7 +2,7 @@ package com.myoa.engineering.crawl.shopping.domain.entity.v2; import com.myoa.engineering.crawl.shopping.domain.entity.Auditable; import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget; -import jakarta.persistence.*; +import javax.persistence.*; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Getter; diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v2/SubscribedKeyword.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v2/SubscribedKeyword.java index 1aa294c..b3b2969 100644 --- a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v2/SubscribedKeyword.java +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v2/SubscribedKeyword.java @@ -2,7 +2,7 @@ package com.myoa.engineering.crawl.shopping.domain.entity.v2; import com.myoa.engineering.crawl.shopping.domain.entity.Auditable; import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget; -import jakarta.persistence.*; +import javax.persistence.*; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Getter; diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/PpomppuArticleParseDTO.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/PpomppuArticleParseDTO.java index 51555a1..497d746 100644 --- a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/PpomppuArticleParseDTO.java +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/PpomppuArticleParseDTO.java @@ -1,14 +1,13 @@ -package com.myoa.engineering.crawl.ppomppu.processor.dto; - -import java.time.Instant; - -import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle; -import com.myoa.engineering.crawl.ppomppu.support.util.DateUtil; +package com.myoa.engineering.crawl.shopping.dto; +import com.myoa.engineering.crawl.shopping.domain.entity.v1.PpomppuArticle; +import com.myoa.engineering.crawl.shopping.support.util.DateUtil; import lombok.Builder; import lombok.Getter; import lombok.NoArgsConstructor; +import java.time.Instant; + /** * PpomppuArticleParseDTO * @@ -46,7 +45,7 @@ public class PpomppuArticleParseDTO { public boolean isInValidated() { return articleId == null || articleId.isEmpty() - || hit == null || hit.isEmpty(); + || hit == null || hit.isEmpty(); } public PpomppuArticle convert() { diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/PpomppuArticleTransformer.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/PpomppuArticleTransformer.java index b68bb49..504b036 100644 --- a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/PpomppuArticleTransformer.java +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/PpomppuArticleTransformer.java @@ -1,39 +1,40 @@ -package com.myoa.engineering.crawl.ppomppu.processor.dto; +package com.myoa.engineering.crawl.shopping.dto; + +import com.myoa.engineering.crawl.shopping.domain.entity.v1.PpomppuArticle; +import com.myoa.engineering.crawl.shopping.support.dto.BlockMessageDTO; +import com.myoa.engineering.crawl.shopping.support.dto.SimpleMessageDTO; +import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName; +import com.myoa.engineering.crawl.shopping.support.util.DateUtil; import java.time.Instant; import java.util.List; import java.util.function.Function; import java.util.stream.Collectors; -import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle; -import com.myoa.engineering.crawl.ppomppu.support.dto.BlockMessageDTO; -import com.myoa.engineering.crawl.ppomppu.support.dto.SimpleMessageDTO; -import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName; -import com.myoa.engineering.crawl.ppomppu.support.util.DateUtil; - /** * PpomppuArticleTransformer + * * @author Shin Woo-jin (woo-jin.shin@linecorp.com) * @since 2021-11-21 - * */ public final class PpomppuArticleTransformer { - private PpomppuArticleTransformer() {} + private PpomppuArticleTransformer() { + } private static final String MESSAGE_FORMAT_V1 = "%s)) <%s|LINK> `%s` "; private static final String MESSAGE_FORMAT_V2 = "%s *<%s|LINK>*\n%s"; private static final String TITLE_FORMAT_V1 = "_*:hearts: %s | %s*_"; public static final Function TRANSFORM_TO_MESSAGE_DTO = article -> - SimpleMessageDTO.builder() - .requestedAt(Instant.now()) - .publishedAt(article.getRegisteredAt()) - .title(String.format(MESSAGE_FORMAT_V1, - article.getBoardName().getMenuName(), article.getArticleUrl(), - article.getTitle())) - .body(article.getArticleUrl()) - .build(); + SimpleMessageDTO.builder() + .requestedAt(Instant.now()) + .publishedAt(article.getRegisteredAt()) + .title(String.format(MESSAGE_FORMAT_V1, + article.getBoardName().getMenuName(), article.getArticleUrl(), + article.getTitle())) + .body(article.getArticleUrl()) + .build(); // https://stackoverflow.com/questions/24882927/using-streams-to-convert-a-list-of-objects-into-a-string-obtained-from-the-tostr public static SimpleMessageDTO transformToSimpleMessage(List articles) { diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/slack/SlackBaseMessageBlock.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/slack/SlackBaseMessageBlock.java index f8968ee..4f41cba 100644 --- a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/slack/SlackBaseMessageBlock.java +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/slack/SlackBaseMessageBlock.java @@ -1,4 +1,4 @@ -package com.myoa.engineering.crawl.ppomppu.sender.dto; +package com.myoa.engineering.crawl.shopping.dto.slack; import com.fasterxml.jackson.annotation.JsonInclude; diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/slack/SlackImageMessageBlock.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/slack/SlackImageMessageBlock.java index 8377a2f..8ec1e51 100644 --- a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/slack/SlackImageMessageBlock.java +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/slack/SlackImageMessageBlock.java @@ -1,4 +1,4 @@ -package com.myoa.engineering.crawl.ppomppu.sender.dto; +package com.myoa.engineering.crawl.shopping.dto.slack; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/slack/SlackMessageBlock.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/slack/SlackMessageBlock.java index cd94027..59c9748 100644 --- a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/slack/SlackMessageBlock.java +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/slack/SlackMessageBlock.java @@ -1,4 +1,4 @@ -package com.myoa.engineering.crawl.ppomppu.sender.dto; +package com.myoa.engineering.crawl.shopping.dto.slack; import java.io.Serializable; diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/slack/SlackMessageBlockType.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/slack/SlackMessageBlockType.java index aef8d0d..a3cdc1b 100644 --- a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/slack/SlackMessageBlockType.java +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/slack/SlackMessageBlockType.java @@ -1,4 +1,4 @@ -package com.myoa.engineering.crawl.ppomppu.sender.dto; +package com.myoa.engineering.crawl.shopping.dto.slack; import lombok.AllArgsConstructor; import lombok.Getter; diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/slack/SlackMessageDTO.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/slack/SlackMessageDTO.java index 2f55024..ba79116 100644 --- a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/slack/SlackMessageDTO.java +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/slack/SlackMessageDTO.java @@ -1,21 +1,19 @@ package com.myoa.engineering.crawl.shopping.dto.slack; -import java.util.List; - import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; -import com.myoa.engineering.crawl.ppomppu.support.dto.BlockMessageDTO; -import com.myoa.engineering.crawl.ppomppu.support.dto.BlockMessageDTO.Block; - +import com.myoa.engineering.crawl.shopping.support.dto.BlockMessageDTO; import lombok.Builder; import lombok.Getter; import lombok.NoArgsConstructor; +import java.util.List; + /** * SlackMessageDTO + * * @author Shin Woo-jin (woo-jin.shin@linecorp.com) * @since 2021-11-14 - * */ @Getter @NoArgsConstructor @@ -51,6 +49,7 @@ public class SlackMessageDTO implements MessageDTO { slackSectionMessageBlock.applyImageaccessory(block.getImageUrl(), block.getAltText()); addBlock(slackSectionMessageBlock); } + public void addSectionBlock(String rawBlockMessage) { addBlock(SlackSectionMessageBlock.ofMarkDown(rawBlockMessage)); } diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/slack/SlackSectionMessageBlock.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/slack/SlackSectionMessageBlock.java index aa6fbbe..651d77f 100644 --- a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/slack/SlackSectionMessageBlock.java +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/slack/SlackSectionMessageBlock.java @@ -1,16 +1,15 @@ -package com.myoa.engineering.crawl.ppomppu.sender.dto; +package com.myoa.engineering.crawl.shopping.dto.slack; import com.fasterxml.jackson.annotation.JsonInclude; - import lombok.Builder; import lombok.Getter; import lombok.NoArgsConstructor; /** * SectionBlock + * * @author Shin Woo-jin (woo-jin.shin@linecorp.com) * @since 2021-11-30 - * */ @Getter @NoArgsConstructor diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/MessageSenderAPIClient.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/MessageSenderAPIClient.java deleted file mode 100644 index 4ba857d..0000000 --- a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/MessageSenderAPIClient.java +++ /dev/null @@ -1,70 +0,0 @@ -package com.myoa.engineering.crawl.ppomppu.processor.infrastructure.client; - -import org.springframework.core.ParameterizedTypeReference; -import org.springframework.http.HttpHeaders; -import org.springframework.http.MediaType; -import org.springframework.stereotype.Component; -import org.springframework.web.reactive.function.client.WebClient; -import org.springframework.web.reactive.function.client.WebClientRequestException; - -import com.myoa.engineering.crawl.ppomppu.processor.dto.constant.WebClientPropertiesUnitName; -import com.myoa.engineering.crawl.ppomppu.support.dto.BlockMessageDTO; -import com.myoa.engineering.crawl.ppomppu.support.dto.SimpleMessageDTO; -import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebClientFilterFactory; -import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebFluxExchangeStragiesFactory; -import com.myoa.engineering.crawl.ppomppu.support.webclient.properties.WebClientProperties; -import com.myoa.engineering.crawl.ppomppu.support.webclient.properties.WebClientProperties.WebClientPropertiesUnit; - -import lombok.extern.slf4j.Slf4j; -import reactor.core.publisher.Mono; -import reactor.core.scheduler.Schedulers; - -/** - * PpomppuNotifierSenderAPIClient - * - * @author Shin Woo-jin (woo-jin.shin@linecorp.com) - * @since 2021-11-17 - */ -@Slf4j -@Component -public class MessageSenderAPIClient { - - private final WebClient webClient; - - public MessageSenderAPIClient(WebClientProperties webClientProperties) { - WebClientPropertiesUnit webClientPropertiesUnit = - webClientProperties.find(WebClientPropertiesUnitName.PPOMPPU_NOTIFIER_SENDER_API.getUnitName()); - this.webClient = WebClient.builder() - .baseUrl(webClientPropertiesUnit.getBaseUrl()) - .exchangeStrategies(WebFluxExchangeStragiesFactory.ofDefault()) - .defaultHeader(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_VALUE) -// .filter(WebClientFilterFactory.logRequest()) -// .filter(WebClientFilterFactory.logResponse()) - .build(); - } - - public Mono sendSimpleMessageToSlack(SimpleMessageDTO dto) { - return webClient.post() - .uri("/api/v1/messages/sendSimpleMessage/messengers/slack") - .bodyValue(dto) - .exchangeToMono(e -> e.bodyToMono(new ParameterizedTypeReference() {})) - .publishOn(Schedulers.boundedElastic()) - .onErrorResume(WebClientRequestException.class, t -> { - log.info("Exception occured, ignoring. : {}", t.getClass().getSimpleName()); - return Mono.empty(); - }); - } - - - public Mono sendBlockMessageToSlack(BlockMessageDTO dto) { - return webClient.post() - .uri("/api/v1/messages/sendBlockMessage/messengers/slack") - .bodyValue(dto) - .exchangeToMono(e -> e.bodyToMono(new ParameterizedTypeReference() {})) - .publishOn(Schedulers.boundedElastic()) - .onErrorResume(WebClientRequestException.class, t -> { - log.info("Exception occured, ignoring. : {}", t.getClass().getSimpleName()); - return Mono.empty(); - }); - } -} \ No newline at end of file diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/slack/MongeShoppingBotSlackMessageSender.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/slack/MongeShoppingBotSlackMessageSender.java deleted file mode 100644 index 6bdb227..0000000 --- a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/slack/MongeShoppingBotSlackMessageSender.java +++ /dev/null @@ -1,52 +0,0 @@ -package com.myoa.engineering.crawl.ppomppu.sender.infrastructure.client; - -import java.util.ArrayList; - -import org.springframework.stereotype.Component; - -import com.myoa.engineering.crawl.ppomppu.sender.configuration.properties.SlackSecretProperties; -import com.myoa.engineering.crawl.ppomppu.sender.configuration.properties.SlackSecretProperties.SlackSecretPropertiesUnit; -import com.myoa.engineering.crawl.ppomppu.sender.dto.SlackMessageDTO; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -@Component -public class MongeShoppingBotSlackMessageSender extends SlackMessageSender { - - private static final String SLACK_SECRET_UNIT_NAME = "monge_shopping_bot"; - - private final SlackSecretPropertiesUnit slackProperties; - - public MongeShoppingBotSlackMessageSender(SlackSecretProperties slackSecretProperties) { - super(slackSecretProperties.find(SLACK_SECRET_UNIT_NAME).getToken()); - this.slackProperties = slackSecretProperties.find(SLACK_SECRET_UNIT_NAME); - } - - public SlackMessageDTO ofMessageTemplate() { - return SlackMessageDTO.builder() - .channel(slackProperties.getChannel()) - .iconEmoji(slackProperties.getIconEmoji()) - .username(slackProperties.getUsername()) - .build(); - } - - public SlackMessageDTO ofMessage(String text) { - return SlackMessageDTO.builder() - .channel(slackProperties.getChannel()) - .iconEmoji(slackProperties.getIconEmoji()) - .username(slackProperties.getUsername()) - .text(text) - .build(); - } - - public SlackMessageDTO ofBlockMessageBased() { - return SlackMessageDTO.builder() - .channel(slackProperties.getChannel()) - .iconEmoji(slackProperties.getIconEmoji()) - .username(slackProperties.getUsername()) - .blocks(new ArrayList<>()) - .build(); - } - -} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/slack/SlackMessageSender.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/slack/SlackMessageSender.java deleted file mode 100644 index 700b3c0..0000000 --- a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/slack/SlackMessageSender.java +++ /dev/null @@ -1,47 +0,0 @@ -package com.myoa.engineering.crawl.shopping.infra.client.slack; - -import lombok.extern.slf4j.Slf4j; - -/** - * SlackMessageSender - * - * @author Shin Woo-jin (woozu.shin@kakaoent.com) - * @since 2021-09-08 - */ -@Slf4j -public class SlackMessageSender { /* implements MessageSender {*/ - -/* - private static final String SLACK_API_URL = "https://slack.com/api"; - - private final WebClient webClient; - - public SlackMessageSender(String apiSecret) { - this.webClient = WebClient.builder() - .baseUrl(SLACK_API_URL) - .defaultHeader(HttpHeaders.AUTHORIZATION, "Bearer " + apiSecret) - .exchangeStrategies(WebFluxExchangeStragiesFactory.ofDefault()) - .defaultHeader(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_UTF8_VALUE) - .defaultHeader(HttpHeaders.ACCEPT_CHARSET, "UTF-8") - .filter(WebClientFilterFactory.logRequest()) - .filter(WebClientFilterFactory.logResponse()) - .build(); - } - - @Override - public Mono sendMessage(SlackMessageDTO message) { - return webClient.post() - .uri("/chat.postMessage") - .bodyValue(message) - .exchangeToMono(e -> e.bodyToMono(String.class)) - .publishOn(Schedulers.boundedElastic()) - .onErrorResume(WebClientRequestException.class, t -> { - log.info("Exception occured, ignoring. : {}", t.getClass().getSimpleName()); - return Mono.empty(); - }) - .doOnNext(e -> log.info("[sendMessage] {}", e)); - } - - */ - -} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/ArticleCommandService.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/ArticleCommandService.java index 20aa894..44af3db 100644 --- a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/ArticleCommandService.java +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/ArticleCommandService.java @@ -4,7 +4,7 @@ import com.myoa.engineering.crawl.shopping.domain.entity.v2.Article; import com.myoa.engineering.crawl.shopping.domain.model.v2.ArticleModel; import com.myoa.engineering.crawl.shopping.event.ArticleUpsertEvent; import com.myoa.engineering.crawl.shopping.infra.repository.v2.ArticleRepository; -import jakarta.transaction.Transactional; +import javax.transaction.Transactional; import org.springframework.context.ApplicationEventPublisher; import org.springframework.stereotype.Service; diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/MessageSenderService.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/MessageSenderService.java deleted file mode 100644 index c76d5f0..0000000 --- a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/MessageSenderService.java +++ /dev/null @@ -1,44 +0,0 @@ -package com.myoa.engineering.crawl.ppomppu.processor.service; - -import java.util.List; - -import org.springframework.stereotype.Service; - -import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle; -import com.myoa.engineering.crawl.ppomppu.processor.dto.PpomppuArticleTransformer; -import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.client.MessageSenderAPIClient; -import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName; - -import lombok.extern.slf4j.Slf4j; -import reactor.core.publisher.Mono; - -/** - * MessageSenderService - * @author Shin Woo-jin (woo-jin.shin@linecorp.com) - * @since 2021-11-21 - * - */ -@Slf4j -@Service -public class MessageSenderService { - - private final MessageSenderAPIClient messageSenderAPIClient; - - public MessageSenderService(MessageSenderAPIClient messageSenderAPIClient) { - this.messageSenderAPIClient = messageSenderAPIClient; - } - - public Mono sendSimpleMessageToSlack(PpomppuArticle article) { - return messageSenderAPIClient.sendSimpleMessageToSlack(PpomppuArticleTransformer.TRANSFORM_TO_MESSAGE_DTO.apply(article)); - } - - public Mono sendSimpleMessageToSlack(List articles) { - return messageSenderAPIClient.sendSimpleMessageToSlack(PpomppuArticleTransformer.transformToSimpleMessage(articles)); - } - - public Mono sendBlockMessageToSlack(PpomppuBoardName boardName, List articles) { - return messageSenderAPIClient.sendBlockMessageToSlack( - PpomppuArticleTransformer.transformToBlockMessage(boardName, articles)); - } - -} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/PpomppuFeedService.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/PpomppuFeedService.java deleted file mode 100644 index f9ad684..0000000 --- a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/PpomppuFeedService.java +++ /dev/null @@ -1,66 +0,0 @@ -package com.myoa.engineering.crawl.ppomppu.processor.service; - -import java.util.Comparator; -import java.util.List; - -import org.jsoup.Jsoup; -import org.jsoup.nodes.Element; -import org.springframework.stereotype.Component; - -import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle; -import com.myoa.engineering.crawl.ppomppu.processor.dto.PpomppuArticleParser; -import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.client.PpomppuBoardFeedRetriever; -import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName; - -import lombok.extern.slf4j.Slf4j; -import reactor.core.publisher.Flux; -import reactor.core.publisher.Mono; - -/** - * PpomppuFeedService - * - * @author Shin Woo-jin (woozu.shin@kakaoent.com) - * @since 2021-09-08 - */ -@Slf4j -@Component -public class PpomppuFeedService { - - private final PpomppuBoardFeedRetriever ppomppuBoardFeedRetriever; - - public PpomppuFeedService(PpomppuBoardFeedRetriever ppomppuBoardFeedRetriever) { - this.ppomppuBoardFeedRetriever = ppomppuBoardFeedRetriever; - } - - public Mono> getArticles(PpomppuBoardName boardName) { - final Mono html = ppomppuBoardFeedRetriever.getHtml(boardName.getResourcePath()); - final Mono tbody = extractTbodyFromHtml(html); - // .doOnNext(e -> log.info("pre tbody - {}", e.html())); - return extractArticlesFromTbody(tbody).map(this::convertFromElement) - .onErrorContinue((t, e) -> log.error("Error occured : {}, value: {}", - e, t.getLocalizedMessage())) - .map(e -> e.updateBoardName(boardName)) - .sort(Comparator.comparing(PpomppuArticle::getArticleId)) - // .doOnNext(e -> log.info("parsed Result: {}", e)) - .collectList(); - } - - private Mono extractTbodyFromHtml(Mono html) { - return html.map(Jsoup::parse) - .mapNotNull(e -> e.getElementById("revolution_main_table")) - .map(e -> e.getElementsByTag("tbody")) - // .doOnNext(e -> log.info("tbody - {}", e.html())) - .map(e -> e.stream() - .findFirst() - .orElseThrow(() -> new IndexOutOfBoundsException("no tbody"))); - } - - private Flux extractArticlesFromTbody(Mono tbody) { - return Flux.concat(tbody.flatMapMany(e -> Flux.fromArray(e.select("tr.list0").toArray(Element[]::new))), - tbody.flatMapMany(e -> Flux.fromArray(e.select("tr.list1").toArray(Element[]::new)))); - } - - private PpomppuArticle convertFromElement(Element element) { - return PpomppuArticleParser.toArticle(element.getElementsByTag("td")); - } -} diff --git a/shopping-crawler/src/test/java/com/myoa/engineering/crawl/shopping/crawlhandler/parser/PpomppuArticleParserV2Test.java b/shopping-crawler/src/test/java/com/myoa/engineering/crawl/shopping/crawlhandler/parser/PpomppuArticleParserV2Test.java index c39869e..eff04e6 100644 --- a/shopping-crawler/src/test/java/com/myoa/engineering/crawl/shopping/crawlhandler/parser/PpomppuArticleParserV2Test.java +++ b/shopping-crawler/src/test/java/com/myoa/engineering/crawl/shopping/crawlhandler/parser/PpomppuArticleParserV2Test.java @@ -1,118 +1,6 @@ package com.myoa.engineering.crawl.shopping.crawlhandler.parser; -import com.myoa.engineering.crawl.shopping.domain.entity.v1.PpomppuArticle; -import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName; -import com.myoa.engineering.crawl.shopping.util.DateTimeUtils; -import com.myoa.engineering.crawl.shopping.util.NumberUtils; -import com.myoa.engineering.crawl.shopping.util.TestDataUtils; -import io.micrometer.core.instrument.util.StringUtils; -import org.jsoup.Jsoup; -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; -import org.jsoup.select.Elements; -import org.junit.jupiter.api.Test; - -import java.time.ZonedDateTime; -import java.util.List; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - class PpomppuArticleParserV2Test { - @Test - void test1() { - String data = TestDataUtils.fileToString("testdata/zboard/file1.html"); - Document document = Jsoup.parse(data); - Elements trList = document.getElementById("revolution_main_table").getElementsByTag("tr"); - List articles = trList.stream() - .filter(this::isRealArticle) - .map(this::parse) - .toList(); - System.out.println(articles.size()); - System.out.println(articles.get(0)); - System.out.println(articles.get(1)); - } - - private boolean isRealArticle(Element tr) { - Elements tdList = tr.getElementsByTag("td"); - if (tdList.size() != 6) { - return false; - } - - if (!hasOnlyNumeric(tdList.get(0))) { - return false; - } - - return true; - } - - - Pattern pattern_numeric = Pattern.compile("\\d+"); - - private boolean hasOnlyNumeric(Element td) { - return pattern_numeric.matcher(td.text()).matches(); - } - - public PpomppuArticle parse(Element tr) { - Elements tdList = tr.getElementsByTag("td"); - Long articleId = Long.parseLong(tdList.get(0).text()); - - String title = tdList.get(2).text(); - String articleUrl = parseArticleUrl(tdList.get(2).getElementsByTag("a").attr("href")); - PpomppuBoardName boardName = parseBoardName(title); - Integer recommended = parseRecommended(tdList.get(4)); - Integer hit = NumberUtils.parseInt(tdList.get(5).text(), 0); - - - ZonedDateTime registeredAt = DateTimeUtils.parse(tdList.get(3).text()); - - return PpomppuArticle.builder() - .articleId(articleId) - .title(title) - .boardName(boardName) - .articleUrl(articleUrl) - .recommended(recommended) - .hit(hit) - .registeredAt(registeredAt) - .build(); - } - - public Long parseArticleId(Element td) { - return Long.parseLong(td.text().trim()); - } - - public static Integer parseRecommended(Element td) { - final String voteString = td.text(); - if (StringUtils.isEmpty(voteString)) { - return null; - } - - final int voteUp = Integer.parseInt(td.text().split(" - ")[0]); - final int voteDown = Integer.parseInt(td.text().split(" - ")[1]); - int recommended = voteUp - voteDown; - return recommended; - } - - public static String parseArticleUrl(String data) { - return PpomppuBoardName.ofViewPageUrl(data); - } - - - Pattern patternBoardName = Pattern.compile("\\[(.+?)\\]"); - - public PpomppuBoardName parseBoardName(String fullTitle) { - Matcher matcher = patternBoardName.matcher(fullTitle); - String lastMatched = null; - while (matcher.find()) { - lastMatched = matcher.group(1); - } - return PpomppuBoardName.ofBoardName(lastMatched, true); - } - - @Test - void test2() { - PpomppuBoardName boardName = parseBoardName("[자사몰]푸마 메쉬 폼스트라이프 [에디션] 5종 [세트] (18,220원/무료)6 [의류/잡화]"); - System.out.println(boardName); - } } \ No newline at end of file diff --git a/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/dto/code/PpomppuBoardName.java b/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/dto/code/PpomppuBoardName.java deleted file mode 100644 index 2e7ba95..0000000 --- a/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/dto/code/PpomppuBoardName.java +++ /dev/null @@ -1,51 +0,0 @@ -package com.myoa.engineering.crawl.ppomppu.support.dto.code; - -import lombok.AllArgsConstructor; -import lombok.Getter; - -/** - * PpomppuBoardName - * - * @author Shin Woo-jin (woo-jin.shin@linecorp.com) - * @since 2021-09-05 - */ -@Getter -@AllArgsConstructor -public enum PpomppuBoardName { - PPOMPPU_DOMESTIC_ALL("/zboard/zboard.php?id=ppomppu", "국내-전체", false), - PPOMPPU_DOMESTIC_ETC("/zboard/zboard.php?id=ppomppu&category=1", "국내-기타", true), - PPOMPPU_DOMESTIC_COMPUTER("/zboard/zboard.php?id=ppomppu&category=4", "국내-컴퓨터", true), - PPOMPPU_DOMESTIC_DIGITAL("/zboard/zboard.php?id=ppomppu&category=5", "국내-디지털", true), - PPOMPPU_DOMESTIC_FOOD("/zboard/zboard.php?id=ppomppu&category=6", "국내-식품/건강", true), - PPOMPPU_DOMESTIC_BOOK("/zboard/zboard.php?id=ppomppu&category=8", "국내-서적", true), - PPOMPPU_DOMESTIC_APPLIANCES("/zboard/zboard.php?id=ppomppu&category=9", "국내-가전/가구", true), - PPOMPPU_DOMESTIC_PARENTING("/zboard/zboard.php?id=ppomppu&category=10", "국내-육아", true), - PPOMPPU_DOMESTIC_GIFTCARD("/zboard/zboard.php?id=ppomppu&category=11", "국내-상품권", true), - PPOMPPU_DOMESTIC_CLOTHES("/zboard/zboard.php?id=ppomppu&category=12", "국내-의류/잡화", true), - PPOMPPU_DOMESTIC_COSMETIC("/zboard/zboard.php?id=ppomppu&category=13", "국내-화장품", true), - PPOMPPU_DOMESTIC_OUTDOOR("/zboard/zboard.php?id=ppomppu&category=15", "국내-등산/캠핑", true), - PPOMPPU_OVERSEA_ALL("/zboard/zboard.php?id=ppomppu4", "해외-전체", false), - PPOMPPU_OVERSEA_ETC("/zboard/zboard.php?id=ppomppu4&category=1", "해외-기타", true), - PPOMPPU_OVERSEA_APPLIANCES("/zboard/zboard.php?id=ppomppu4&category=7", "해외-가전", true), - PPOMPPU_OVERSEA_TVAV("/zboard/zboard.php?id=ppomppu4&category=8", "해외-TV/영상", true), - PPOMPPU_OVERSEA_COMPUTER("/zboard/zboard.php?id=ppomppu4&category=3", "해외-컴퓨터", true), - PPOMPPU_OVERSEA_DIGITAL("/zboard/zboard.php?id=ppomppu4&category=4", "해외-디지털", true), - PPOMPPU_OVERSEA_MOBILEACCESSORY("/zboard/zboard.php?id=ppomppu4&category=9", "해외-액세서리", true), - PPOMPPU_OVERSEA_CLOTHES("/zboard/zboard.php?id=ppomppu4&category=5", "해외-의류/잡화", true), - PPOMPPU_OVERSEA_WATCH("/zboard/zboard.php?id=ppomppu4&category=2", "해외-시계", true), - PPOMPPU_OVERSEA_SHOES("/zboard/zboard.php?id=ppomppu4&category=11", "해외-신발", true), - PPOMPPU_OVERSEA_FOOD("/zboard/zboard.php?id=ppomppu4&category=10", "해외-식품/건강", true), - PPOMPPU_OVERSEA_PARENTING("/zboard/zboard.php?id=ppomppu4&category=6", "해외-육아", true), - ; - - private String resourcePath; - private String menuName; - private boolean crawlWithDefaultTimer; - - public static final String PPOMPPU_URL = "https://www.ppomppu.co.kr"; - - public static String ofViewPageUrl(String articleUrl) { - return PPOMPPU_URL + "/zboard/" + articleUrl; - } - -} diff --git a/support/src/main/java/com/myoa/engineering/crawl/shopping/support/dto/BlockMessageDTO.java b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/dto/BlockMessageDTO.java index 9531210..7784d5a 100644 --- a/support/src/main/java/com/myoa/engineering/crawl/shopping/support/dto/BlockMessageDTO.java +++ b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/dto/BlockMessageDTO.java @@ -1,20 +1,19 @@ -package com.myoa.engineering.crawl.ppomppu.support.dto; +package com.myoa.engineering.crawl.shopping.support.dto; + +import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName; +import lombok.Builder; +import lombok.Getter; +import lombok.NoArgsConstructor; import java.io.Serializable; import java.time.Instant; import java.util.List; -import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName; - -import lombok.Builder; -import lombok.Getter; -import lombok.NoArgsConstructor; - /** * SimpleMessageDTO + * * @author Shin Woo-jin (woo-jin.shin@linecorp.com) * @since 2021-11-21 - * */ @Getter @NoArgsConstructor diff --git a/support/src/main/java/com/myoa/engineering/crawl/shopping/support/util/DateUtil.java b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/util/DateUtil.java index e444987..b33749a 100644 --- a/support/src/main/java/com/myoa/engineering/crawl/shopping/support/util/DateUtil.java +++ b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/util/DateUtil.java @@ -1,4 +1,4 @@ -package com.myoa.engineering.crawl.ppomppu.support.util; +package com.myoa.engineering.crawl.shopping.support.util; import java.time.ZoneId; import java.time.format.DateTimeFormatter;