build.gradle

@@ -1,13 +1,14 @@
 plugins {
     id 'java'
     id 'idea'
-    id 'org.springframework.boot' version '2.5.4'
+    id 'org.springframework.boot' version '3.2.5'
-    id 'io.spring.dependency-management' version '1.0.11.RELEASE'
+    id 'io.spring.dependency-management' version '1.1.4'
 }
 
-group = 'com.myoa.engineering.crawl.shopping'
+group = 'com.myoa.engineering.crawl.ppomppu'
-version = '1.1.1'
+version = '1.0.1'
-sourceCompatibility = JavaVersion.VERSION_17
+sourceCompatibility = JavaVersion.VERSION_21
+targetCompatibility = JavaVersion.VERSION_21
 
 configurations {
     compileOnly {
@@ -21,7 +22,7 @@ repositories {
 
 allprojects {
     group = 'com.myoa.engineering.crawl.shopping'
-    version = '1.1.1'
+    version = '2.0.0'
 
     apply plugin: 'java'
     apply plugin: 'idea'
@@ -36,7 +37,7 @@ allprojects {
     }
 
     ext {
-        set('springCloudVersion', "2020.0.4")
+        set('springCloudVersion', "2023.0.1")
     }
 
     dependencyManagement {

copy.bat

@@ -1,3 +0,0 @@
-xcopy /y .\processor\build\libs\*.jar .\
-xcopy /y .\receiver\build\libs\*.jar .\
-xcopy /y .\sender\build\libs\*.jar .\

processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/controller/CrawlAPIController.java

@@ -1,70 +0,0 @@
-package com.myoa.engineering.crawl.shopping.controller;
-
-import java.util.List;
-
-import org.springframework.web.bind.annotation.PathVariable;
-import org.springframework.web.bind.annotation.PostMapping;
-import org.springframework.web.bind.annotation.RequestMapping;
-import org.springframework.web.bind.annotation.RestController;
-
-import com.myoa.engineering.crawl.shopping.processor.domain.PpomppuArticle;
-import com.myoa.engineering.crawl.shopping.processor.dto.FeedParsedResult;
-import com.myoa.engineering.crawl.shopping.processor.service.MessageSenderService;
-import com.myoa.engineering.crawl.shopping.processor.service.PpomppuArticleService;
-import com.myoa.engineering.crawl.shopping.processor.service.PpomppuFeedService;
-import com.myoa.engineering.crawl.shopping.support.dto.APIResponse;
-import com.myoa.engineering.crawl.shopping.support.dto.code.PpomppuBoardName;
-
-import lombok.extern.slf4j.Slf4j;
-import reactor.core.publisher.Mono;
-
-/**
- * CrawlAPIController
- *
- * @author Shin Woo-jin (woo-jin.shin@linecorp.com)
- * @since 2021-09-05
- */
-@Slf4j
-@RestController
-@RequestMapping("/api/v1/crawl")
-public class CrawlAPIController {
-
-    private final PpomppuFeedService ppomppuRSSFeedService;
-    private final PpomppuArticleService ppomppuArticleService;
-    private final MessageSenderService messageSenderService;
-
-    public CrawlAPIController(PpomppuFeedService ppomppuRSSFeedService,
-                              PpomppuArticleService ppomppuArticleService,
-                              MessageSenderService messageSenderService) {
-        this.ppomppuRSSFeedService = ppomppuRSSFeedService;
-        this.ppomppuArticleService = ppomppuArticleService;
-        this.messageSenderService = messageSenderService;
-    }
-
-    @PostMapping("/boards/{boardName}")
-    public Mono<APIResponse<FeedParsedResult>> crawlBoard(@PathVariable("boardName") PpomppuBoardName boardName) {
-        log.info("got request... {}", boardName);
-        FeedParsedResult result = FeedParsedResult.of(boardName);
-
-        Mono<String> publishedMessages =
-            ppomppuRSSFeedService.getArticles(boardName)
-                                 .map(e -> ppomppuArticleService.filterOnlyNewArticles(boardName, e))
-                                 .map(e -> ppomppuArticleService.save(boardName, e))
-                                 .filter(e -> !e.isEmpty())
-                                 .flatMap(e -> messageSenderService.sendBlockMessageToSlack(boardName, e));
-
-        return publishedMessages.then(Mono.just(APIResponse.success(result.done())));
-    }
-
-    @PostMapping("/exploit/boards/{boardName}")
-    public Mono<APIResponse<String>> crawlBoardDryRun(
-        @PathVariable("boardName") PpomppuBoardName boardName) {
-        log.info("got request... {}", boardName);
-        Mono<String> publishedMessages =
-            ppomppuRSSFeedService.getArticles(boardName)
-                                 .flatMap(e -> messageSenderService.sendBlockMessageToSlack(boardName, e));
-
-        return publishedMessages.map(APIResponse::success);
-    }
-
-}

processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/PpomppuArticleParser.java

@@ -1,78 +0,0 @@
-package com.myoa.engineering.crawl.shopping.dto;
-
-import org.jsoup.nodes.Element;
-import org.jsoup.select.Elements;
-
-import com.myoa.engineering.crawl.shopping.processor.domain.PpomppuArticle;
-import com.myoa.engineering.crawl.shopping.support.dto.code.PpomppuBoardName;
-
-/**
- * PpomppuArticleTransformer
- *
- * @author Shin Woo-jin (woozu.shin@kakaoent.com)
- * @since 2021-09-08
- */
-public final class PpomppuArticleParser {
-
-    private PpomppuArticleParser() {}
-
-    public static PpomppuArticle toArticle(Elements articleElement) {
-        final String articleIdString = PpomppuArticleParser.parseArticleId(articleElement.get(0));
-        final String title = PpomppuArticleParser.parseTitle(articleElement.get(2));
-        final String articleUrl = PpomppuArticleParser.parseArticleUrl(articleElement.get(2));
-        final String thumbnailUrl = PpomppuArticleParser.parseThumbnailUrl(articleElement.get(3));
-        final Integer recommended = PpomppuArticleParser.parseRecommended(articleElement.get(6));
-        final String hitString = PpomppuArticleParser.parseHit(articleElement.get(7));
-        final String registeredAtString = PpomppuArticleParser.parseRegisteredAt(articleElement.get(5));
-
-        return PpomppuArticleParseDTO.builder()
-                                     .articleId(articleIdString)
-                                     .title(title)
-                                     .articleUrl(articleUrl)
-                                     .thumbnailUrl(thumbnailUrl)
-                                     .recommended(recommended)
-                                     .hit(hitString)
-                                     .registeredAt(registeredAtString)
-                                     .build()
-                                     .convert();
-    }
-
-    public static String parseArticleId(Element td) {
-        return td.text().trim();
-    }
-
-    public static String parseTitle(Element td) {
-        return td.getElementsByTag("a").text();
-    }
-
-    public static String parseArticleUrl(Element td) {
-        return PpomppuBoardName.ofViewPageUrl(td.getElementsByTag("a").attr("href"));
-    }
-
-    public static String parseThumbnailUrl(Element td) {
-        return "https:" + td.getElementsByTag("img").get(0).attr("src");
-    }
-
-    public static Integer parseRecommended(Element td) {
-        final String voteString = td.text();
-        final int recommended;
-
-        if (voteString.isEmpty()) {
-            recommended = 0;
-        } else {
-            final int voteUp = Integer.parseInt(td.text().split(" - ")[0]);
-            final int voteDown = Integer.parseInt(td.text().split(" - ")[1]);
-            recommended = voteUp - voteDown;
-        }
-        return recommended;
-    }
-
-    public static String parseHit(Element td) {
-        return td.text();
-    }
-
-    public static String parseRegisteredAt(Element td) {
-        return td.attr("title");
-    }
-
-}

receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/scheduler/ParseEventEmitter.java

@@ -1,38 +0,0 @@
-package com.myoa.engineering.crawl.shopping.receiver.scheduler;
-
-import java.util.Arrays;
-
-import org.springframework.scheduling.annotation.EnableScheduling;
-import org.springframework.scheduling.annotation.Scheduled;
-import org.springframework.stereotype.Component;
-
-import com.myoa.engineering.crawl.shopping.receiver.service.ProcessorAPIService;
-import com.myoa.engineering.crawl.shopping.support.dto.code.PpomppuBoardName;
-
-import lombok.extern.slf4j.Slf4j;
-
-/**
- * ParseEventEmitter
- * @author Shin Woo-jin (woo-jin.shin@linecorp.com)
- * @since 2021-09-05
- *
- */
-@Slf4j
-@Component
-@EnableScheduling
-public class ParseEventEmitter {
-
-    private final ProcessorAPIService processorAPIService;
-
-    public ParseEventEmitter(ProcessorAPIService processorAPIService) {
-        this.processorAPIService = processorAPIService;
-    }
-
-    @Scheduled(fixedRate = 600 * 1000L)
-    public void emitBoards() {
-        log.info("[emitDomesticBoard] trigger fired!");
-        Arrays.stream(PpomppuBoardName.values())
-              .filter(PpomppuBoardName::isCrawlWithDefaultTimer)
-              .forEach(boardName -> processorAPIService.emitParseEvent(boardName).block());
-    }
-}

receiver/src/main/resources/application-production.yml

@@ -1,6 +0,0 @@
-spring:
-  config:
-    activate:
-      on-profile: production
-    import:
-      - "configserver:http://ppn-config-server:20080"

receiver/src/main/resources/application.yml

@@ -1,25 +0,0 @@
-spring:
-  application:
-    name: ppn-receiver
-  main:
-    allow-bean-definition-overriding: true
-  profiles:
-    active: ${SPRING_ACTIVE_PROFILE:local}
-    group:
-      local: "local,webclient-local"
-      development: "development,webclient-development"
-      production: "production,webclient-production"
-  freemarker:
-    enabled: false
-
-server:
-  port: 20080
-  error:
-    whitelabel:
-      enabled: false
-
-management:
-  endpoints:
-    web:
-      exposure:
-        include: refresh,health

sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/controller/MessageSenderAPIController.java

@@ -1,62 +0,0 @@
-package com.myoa.engineering.crawl.shopping.controller;
-
-import org.springframework.web.bind.annotation.PostMapping;
-import org.springframework.web.bind.annotation.RequestBody;
-import org.springframework.web.bind.annotation.RequestMapping;
-import org.springframework.web.bind.annotation.RestController;
-
-import com.myoa.engineering.crawl.shopping.sender.dto.SlackBaseMessageBlock;
-import com.myoa.engineering.crawl.shopping.sender.dto.SlackMessageDTO;
-import com.myoa.engineering.crawl.shopping.sender.infrastructure.client.MongeShoppingBotSlackMessageSender;
-import com.myoa.engineering.crawl.shopping.support.dto.APIResponse;
-import com.myoa.engineering.crawl.shopping.support.dto.BlockMessageDTO;
-import com.myoa.engineering.crawl.shopping.support.dto.SimpleMessageDTO;
-import com.myoa.engineering.crawl.shopping.support.util.ObjectMapperFactory;
-
-import lombok.extern.slf4j.Slf4j;
-import reactor.core.publisher.Mono;
-
-/**
- * MessageSenderAPIController
- * @author Shin Woo-jin (woo-jin.shin@linecorp.com)
- * @since 2021-11-21
- *
- */
-@Slf4j
-@RestController
-@RequestMapping("/api/v1")
-public class MessageSenderAPIController {
-
-    private final MongeShoppingBotSlackMessageSender sender;
-
-    public MessageSenderAPIController(MongeShoppingBotSlackMessageSender sender) {
-        this.sender = sender;
-    }
-
-    @PostMapping("/messages/sendSimpleMessage/messengers/slack")
-    public Mono<APIResponse<SimpleMessageDTO>> sendSimpleMessageToSlack(@RequestBody SimpleMessageDTO dto) {
-        return sender.sendMessage(sender.ofMessage(dto.getBody()))
-                     .then(Mono.just(APIResponse.success(dto)));
-    }
-
-    @PostMapping("/messages/sendBlockMessage/messengers/slack")
-    public Mono<APIResponse<BlockMessageDTO>> sendBlockMessageToSlack(@RequestBody BlockMessageDTO dto) {
-        if (dto.getBlocks().isEmpty()) {
-            return Mono.just(APIResponse.fail(dto, "empty blocks"));
-        }
-
-
-        return sender.sendMessage(buildSlackMessageDTO(dto))
-//                     .doOnNext(e -> log.info("[sendBlockMessageToSlack] slackMessageDTO: {}",
-//                                             ObjectMapperFactory.writeAsString(buildSlackMessageDTO(dto))))
-                     .then(Mono.just(APIResponse.success(dto)));
-    }
-
-    private SlackMessageDTO buildSlackMessageDTO(BlockMessageDTO dto) {
-        SlackMessageDTO slackMessageDTO = sender.ofBlockMessageBased();
-        slackMessageDTO.addSectionBlock(dto.getTitle());
-        dto.getBlocks().forEach(slackMessageDTO::addSectionBlock);
-        slackMessageDTO.addBlock(SlackBaseMessageBlock.ofDivider());
-        return slackMessageDTO;
-    }
-}

sender/src/main/resources/application.yml

@@ -1,25 +0,0 @@
-spring:
-  application:
-    name: ppn-sender
-  main:
-    allow-bean-definition-overriding: true
-  profiles:
-    active: ${SPRING_ACTIVE_PROFILE:local}
-    group:
-      local: "local,slackapi-local,webclient-local"
-      development: "development,slackapi-development,webclient-development"
-      production: "production,slackapi-production,webclient-production"
-  freemarker:
-    enabled: false
-
-server:
-  port: 20080
-  error:
-    whitelabel:
-      enabled: false
-
-management:
-  endpoints:
-    web:
-      exposure:
-        include: refresh,health

shopping-crawler/build.gradle

@@ -1,7 +1,7 @@
 dependencies {
     developmentOnly 'org.springframework.boot:spring-boot-devtools'
     runtimeOnly 'com.h2database:h2'
-    runtimeOnly 'com.mysql:mysql-connector-j:8.4.0'
+    runtimeOnly 'com.mysql:mysql-connector-j'
     compileOnly 'org.projectlombok:lombok'
 
     implementation project(':support')
@@ -17,6 +17,7 @@ dependencies {
     implementation 'org.springframework.boot:spring-boot-configuration-processor'
     implementation 'org.springframework.cloud:spring-cloud-starter-config'
     implementation 'org.springframework.boot:spring-boot-starter-actuator'
+    implementation 'com.rometools:rome:2.1.0'
     implementation 'org.jsoup:jsoup:1.17.2'
     implementation 'com.h2database:h2:2.2.224'
     implementation "org.springframework.cloud:spring-cloud-starter-openfeign"

shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/datasource/ShoppingCrawlerDatasourceConfiguration.java

@@ -7,12 +7,13 @@ import com.myoa.engineering.crawl.shopping.domain.entity.BaseScanDomain;
 import com.myoa.engineering.crawl.shopping.infra.repository.BaseScanRepository;
 import com.zaxxer.hikari.HikariConfig;
 import com.zaxxer.hikari.HikariDataSource;
+import jakarta.persistence.EntityManagerFactory;
 import lombok.NonNull;
+import org.hibernate.boot.model.naming.CamelCaseToUnderscoresNamingStrategy;
+import org.hibernate.boot.model.naming.ImplicitNamingStrategyJpaCompliantImpl;
 import org.hibernate.cfg.AvailableSettings;
 import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.boot.orm.jpa.EntityManagerFactoryBuilder;
-import org.springframework.boot.orm.jpa.hibernate.SpringImplicitNamingStrategy;
-import org.springframework.boot.orm.jpa.hibernate.SpringPhysicalNamingStrategy;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 import org.springframework.data.jpa.repository.config.EnableJpaAuditing;
@@ -21,7 +22,6 @@ import org.springframework.orm.jpa.JpaTransactionManager;
 import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean;
 import org.springframework.transaction.PlatformTransactionManager;
 
-import javax.persistence.EntityManagerFactory;
 import javax.sql.DataSource;
 import java.util.Enumeration;
 import java.util.HashMap;
@@ -96,8 +96,8 @@ public class ShoppingCrawlerDatasourceConfiguration {
         properties.put(AvailableSettings.SHOW_SQL, hibernateProperties.getShowSql());
         properties.put(AvailableSettings.HBM2DDL_AUTO, hibernateProperties.getHbm2ddlAuto());
         properties.put(AvailableSettings.CONNECTION_PROVIDER_DISABLES_AUTOCOMMIT, hibernateProperties.getDisableAutoCommit());
-        properties.put(AvailableSettings.IMPLICIT_NAMING_STRATEGY, SpringImplicitNamingStrategy.class.getName());
+        properties.put(AvailableSettings.IMPLICIT_NAMING_STRATEGY, ImplicitNamingStrategyJpaCompliantImpl.class.getName());
-        properties.put(AvailableSettings.PHYSICAL_NAMING_STRATEGY, SpringPhysicalNamingStrategy.class.getName());
+        properties.put(AvailableSettings.PHYSICAL_NAMING_STRATEGY, CamelCaseToUnderscoresNamingStrategy.class.getName());
         properties.put(AvailableSettings.GENERATE_STATISTICS, "false");
 //        properties.put(AvailableSettings.GLOBALLY_QUOTED_IDENTIFIERS, "true");
 //        properties.put(AvailableSettings.GLOBALLY_QUOTED_IDENTIFIERS_SKIP_COLUMN_DEFINITIONS, "true");

shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/Auditable.java

@@ -3,11 +3,10 @@ package com.myoa.engineering.crawl.shopping.domain.entity;
 import lombok.Getter;
 import org.springframework.data.annotation.CreatedDate;
 import org.springframework.data.annotation.LastModifiedDate;
+
+import jakarta.persistence.*;
 import org.springframework.data.jpa.domain.support.AuditingEntityListener;
 
-import javax.persistence.Column;
-import javax.persistence.EntityListeners;
-import javax.persistence.MappedSuperclass;
 import java.io.Serializable;
 import java.time.Instant;
 

shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v1/PpomppuArticle.java

@@ -2,13 +2,15 @@ package com.myoa.engineering.crawl.shopping.domain.entity.v1;
 
 import com.myoa.engineering.crawl.shopping.domain.entity.Auditable;
 import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName;
+import jakarta.persistence.*;
 import lombok.Builder;
 import lombok.Getter;
 import lombok.NoArgsConstructor;
+import lombok.ToString;
 
-import javax.persistence.*;
+import java.time.ZonedDateTime;
-import java.time.Instant;
 
+@ToString
 @Getter
 @NoArgsConstructor
 @Entity
@@ -29,9 +31,6 @@ public class PpomppuArticle extends Auditable {
     @Column
     private String articleUrl;
 
-    @Column
-    private String thumbnailUrl;
-
     @Column
     private String title;
 
@@ -42,25 +41,19 @@ public class PpomppuArticle extends Auditable {
     private Integer recommended;
 
     @Column
-    private Instant registeredAt;
+    private ZonedDateTime registeredAt;
 
     @Builder
     public PpomppuArticle(Long id, Long articleId, PpomppuBoardName boardName, String articleUrl,
-                          String thumbnailUrl, String title, Integer recommended, Integer hit,
+                          String title, Integer recommended, Integer hit, ZonedDateTime registeredAt) {
-                          Instant registeredAt) {
         this.id = id;
         this.articleId = articleId;
         this.boardName = boardName;
         this.articleUrl = articleUrl;
-        this.thumbnailUrl = thumbnailUrl;
         this.title = title;
         this.recommended = recommended;
         this.hit = hit;
         this.registeredAt = registeredAt;
     }
 
-    public PpomppuArticle updateBoardName(PpomppuBoardName boardName) {
-        this.boardName = boardName;
-        return this;
-    }
 }

shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v1/PpomppuBoardFeedStatus.java

@@ -2,11 +2,11 @@ package com.myoa.engineering.crawl.shopping.domain.entity.v1;
 
 import com.myoa.engineering.crawl.shopping.domain.entity.Auditable;
 import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName;
+import jakarta.persistence.*;
 import lombok.Builder;
 import lombok.Getter;
 import lombok.NoArgsConstructor;
 
-import javax.persistence.*;
 import java.time.Instant;
 
 @Getter

shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v1/PublishedHistory.java

@@ -4,7 +4,7 @@ import com.myoa.engineering.crawl.shopping.domain.entity.Auditable;
 import lombok.Getter;
 import lombok.NoArgsConstructor;
 
-import javax.persistence.*;
+import jakarta.persistence.*;
 import java.time.Instant;
 
 @Getter

shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v1/SubscribedBoard.java

@@ -5,7 +5,7 @@ import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName
 import lombok.Getter;
 import lombok.NoArgsConstructor;
 
-import javax.persistence.*;
+import jakarta.persistence.*;
 
 @Getter
 @NoArgsConstructor

shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v1/SubscribedUser.java

@@ -4,7 +4,7 @@ import com.myoa.engineering.crawl.shopping.domain.entity.Auditable;
 import lombok.Getter;
 import lombok.NoArgsConstructor;
 
-import javax.persistence.*;
+import jakarta.persistence.*;
 import java.time.Instant;
 
 @Getter

shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v2/AppUser.java

@@ -1,7 +1,7 @@
 package com.myoa.engineering.crawl.shopping.domain.entity.v2;
 
 import com.myoa.engineering.crawl.shopping.domain.entity.Auditable;
-import javax.persistence.*;
+import jakarta.persistence.*;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Getter;

shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v2/Article.java

@@ -2,7 +2,7 @@ package com.myoa.engineering.crawl.shopping.domain.entity.v2;
 
 import com.myoa.engineering.crawl.shopping.domain.entity.Auditable;
 import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget;
-import javax.persistence.*;
+import jakarta.persistence.*;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Getter;

shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v2/SubscribedKeyword.java

@@ -2,7 +2,7 @@ package com.myoa.engineering.crawl.shopping.domain.entity.v2;
 
 import com.myoa.engineering.crawl.shopping.domain.entity.Auditable;
 import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget;
-import javax.persistence.*;
+import jakarta.persistence.*;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Getter;

shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/PpomppuArticleDTO.java

@@ -0,0 +1,10 @@
+package com.myoa.engineering.crawl.shopping.dto;
+
+/**
+ * PpomppuArticle
+ *
+ * @author Shin Woo-jin (woozu.shin@kakaoent.com)
+ * @since 2021-09-08
+ */
+public class PpomppuArticleDTO {
+}

shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/PpomppuArticleParseDTO.java

@@ -1,65 +0,0 @@
-package com.myoa.engineering.crawl.shopping.dto;
-
-import com.myoa.engineering.crawl.shopping.domain.entity.v1.PpomppuArticle;
-import com.myoa.engineering.crawl.shopping.support.util.DateUtil;
-import lombok.Builder;
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-
-import java.time.Instant;
-
-/**
- * PpomppuArticleParseDTO
- *
- * @author Shin Woo-jin (woozu.shin@kakaoent.com)
- * @since 2021-09-08
- */
-@Getter
-@NoArgsConstructor
-public class PpomppuArticleParseDTO {
-
-    private String id;
-    private String articleId;
-    private String boardName;
-    private String articleUrl;
-    private String thumbnailUrl;
-    private String title;
-    private String hit;
-    private Integer recommended;
-    private String registeredAt;
-
-    @Builder
-    public PpomppuArticleParseDTO(String id, String articleId, String boardName, String articleUrl,
-                                  String thumbnailUrl, String title, String hit, Integer recommended,
-                                  String registeredAt) {
-        this.id = id;
-        this.articleId = articleId;
-        this.boardName = boardName;
-        this.articleUrl = articleUrl;
-        this.thumbnailUrl = thumbnailUrl;
-        this.title = title;
-        this.hit = hit;
-        this.recommended = recommended;
-        this.registeredAt = registeredAt;
-    }
-
-    public boolean isInValidated() {
-        return articleId == null || articleId.isEmpty()
-                || hit == null || hit.isEmpty();
-    }
-
-    public PpomppuArticle convert() {
-        if (isInValidated()) {
-            throw new IllegalArgumentException("PpomppuArticleParseDTO was invalidated");
-        }
-        return PpomppuArticle.builder()
-                             .articleId(Long.parseLong(articleId))
-                             .title(title)
-                             .articleUrl(articleUrl)
-                             .thumbnailUrl(thumbnailUrl)
-                             .recommended(recommended)
-                             .hit(Integer.parseInt(hit))
-                             .registeredAt(DateUtil.DATE_TIME_FORMATTER.parse(registeredAt, Instant::from))
-                             .build();
-    }
-}

shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/PpomppuArticleTransformer.java

@@ -1,14 +1,12 @@
 package com.myoa.engineering.crawl.shopping.dto;
 
 import com.myoa.engineering.crawl.shopping.domain.entity.v1.PpomppuArticle;
-import com.myoa.engineering.crawl.shopping.support.dto.BlockMessageDTO;
 import com.myoa.engineering.crawl.shopping.support.dto.SimpleMessageDTO;
-import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName;
-import com.myoa.engineering.crawl.shopping.support.util.DateUtil;
 
 import java.time.Instant;
+import java.time.ZoneId;
+import java.time.format.DateTimeFormatter;
 import java.util.List;
-import java.util.function.Function;
 import java.util.stream.Collectors;
 
 /**
@@ -22,50 +20,34 @@ public final class PpomppuArticleTransformer {
     private PpomppuArticleTransformer() {
     }
 
-    private static final String MESSAGE_FORMAT_V1 = "%s)) <%s|LINK> `%s` ";
+    private static final String MESSAGE_FORMAT_V1 = "%s)) `%s` <%s:LINK>";
-    private static final String MESSAGE_FORMAT_V2 = "%s *<%s|LINK>*\n%s";
-    private static final String TITLE_FORMAT_V1 = "_*:hearts: %s | %s*_";
 
-    public static final Function<PpomppuArticle, SimpleMessageDTO> TRANSFORM_TO_MESSAGE_DTO = article ->
+    private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")
+                                                                                  .withZone(ZoneId.of("Asia/Seoul"));
+/*
+    public static final Function<PpomppuArticle, SimpleMessageDTO> TRANSFORM_TO_MESSAGE_DTO = entity ->
             SimpleMessageDTO.builder()
                             .requestedAt(Instant.now())
-                            .publishedAt(article.getRegisteredAt())
+                            .publishedAt(entity.getRegisteredAt())
-                            .title(String.format(MESSAGE_FORMAT_V1,
+                            .title(String.format(MESSAGE_FORMAT_V1, entity.getBoardName().getMenuName(), entity.getTitle()))
-                                                 article.getBoardName().getMenuName(), article.getArticleUrl(),
+                            .body(entity.getArticleUrl())
-                                                 article.getTitle()))
-                            .body(article.getArticleUrl())
                             .build();
-
+*/
     // https://stackoverflow.com/questions/24882927/using-streams-to-convert-a-list-of-objects-into-a-string-obtained-from-the-tostr
-    public static SimpleMessageDTO transformToSimpleMessage(List<PpomppuArticle> articles) {
+    public static SimpleMessageDTO transform(List<PpomppuArticle> articles) {
         Instant requestedAt = Instant.now();
         String body = articles.stream()
                               .map(PpomppuArticleTransformer::convertToInlineMessage)
                               .collect(Collectors.joining("\n\n"));
         return SimpleMessageDTO.builder()
                                .requestedAt(requestedAt)
-                               .title(DateUtil.DATE_TIME_FORMATTER.format(requestedAt))
+                               .title(DATE_TIME_FORMATTER.format(requestedAt))
                                .body(body)
                                .build();
     }
 
-    public static BlockMessageDTO transformToBlockMessage(PpomppuBoardName boardName, List<PpomppuArticle> articles) {
-        Instant requestedAt = Instant.now();
-        List<BlockMessageDTO.Block> body = articles.stream()
-                                                   .map(e -> BlockMessageDTO.createBlock(convertToInlineMessage(e),
-                                                                                         e.getThumbnailUrl()))
-                                                   .collect(Collectors.toList());
-        return BlockMessageDTO.builder()
-                              .requestedAt(requestedAt)
-                              .title(String.format(TITLE_FORMAT_V1,
-                                                   boardName.getMenuName(),
-                                                   DateUtil.DATE_TIME_FORMATTER.format(requestedAt)))
-                              .blocks(body)
-                              .build();
-    }
-
     public static String convertToInlineMessage(PpomppuArticle article) {
-        return String.format(MESSAGE_FORMAT_V2,
+        return String.format(MESSAGE_FORMAT_V1,
-                             article.getBoardName().getMenuName(), article.getArticleUrl(), article.getTitle());
+                             article.getBoardName().getMenuName(), article.getTitle(), article.getArticleUrl());
     }
 }

shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/slack/SlackBaseMessageBlock.java

@@ -1,47 +0,0 @@
-package com.myoa.engineering.crawl.shopping.dto.slack;
-
-import com.fasterxml.jackson.annotation.JsonInclude;
-
-import lombok.Builder;
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-
-/**
- * MessageBlock
- * @author Shin Woo-jin (woo-jin.shin@linecorp.com)
- * @since 2021-11-30
- *
- */
-@Getter
-@NoArgsConstructor
-@JsonInclude(JsonInclude.Include.NON_NULL)
-public class SlackBaseMessageBlock implements SlackMessageBlock {
-    private static final long serialVersionUID = 1597984001727808419L;
-
-    private SlackMessageBlockType type;
-    private String text;
-
-    @Builder
-    private SlackBaseMessageBlock(SlackMessageBlockType type, String text) {
-        this.type = type;
-        this.text = text;
-    }
-
-    public static SlackBaseMessageBlock ofMarkDown(String message) {
-        return SlackBaseMessageBlock.builder()
-                                    .type(SlackMessageBlockType.MARKDOWN)
-                                    .text(message)
-                                    .build();
-    }
-
-    public static SlackBaseMessageBlock ofDivider() {
-        return SlackBaseMessageBlock.builder()
-                                    .type(SlackMessageBlockType.DIVIDER)
-                                    .build();
-    }
-
-    @Override
-    public String getType() {
-        return type.getType();
-    }
-}

shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/slack/SlackImageMessageBlock.java

@@ -1,49 +0,0 @@
-package com.myoa.engineering.crawl.shopping.dto.slack;
-
-import com.fasterxml.jackson.annotation.JsonInclude;
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-import lombok.Builder;
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-
-/**
- * SlackImageMessageBlock
- * @author Shin Woo-jin (woo-jin.shin@linecorp.com)
- * @since 2021-11-30
- *
- */
-@Getter
-@NoArgsConstructor
-@JsonInclude(JsonInclude.Include.NON_NULL)
-public class SlackImageMessageBlock implements SlackMessageBlock {
-    private static final long serialVersionUID = 1597984001727808419L;
-
-    private SlackMessageBlockType type;
-
-    @JsonProperty(value = "image_url", required = true)
-    private String imageUrl;
-
-    @JsonProperty(value = "alt_text", required = true)
-    private String altText;
-
-    @Builder
-    private SlackImageMessageBlock(SlackMessageBlockType type, String imageUrl, String altText) {
-        this.type = type;
-        this.imageUrl = imageUrl;
-        this.altText = altText;
-    }
-
-    public static SlackImageMessageBlock of(String imageUrl, String altText) {
-        return SlackImageMessageBlock.builder()
-                                     .type(SlackMessageBlockType.IMAGE)
-                                     .imageUrl(imageUrl)
-                                     .altText(altText)
-                                     .build();
-    }
-
-    @Override
-    public String getType() {
-        return type.getType();
-    }
-}

shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/slack/SlackMessageBlock.java

@@ -1,15 +0,0 @@
-package com.myoa.engineering.crawl.shopping.dto.slack;
-
-import java.io.Serializable;
-
-/**
- * SlackMessageBlock
- * @author Shin Woo-jin (woo-jin.shin@linecorp.com)
- * @since 2021-12-01
- *
- */
-public interface SlackMessageBlock extends Serializable {
-
-    String getType();
-
-}

shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/slack/SlackMessageBlockType.java

@@ -1,22 +0,0 @@
-package com.myoa.engineering.crawl.shopping.dto.slack;
-
-import lombok.AllArgsConstructor;
-import lombok.Getter;
-
-/**
- * BlockType
- * @author Shin Woo-jin (woo-jin.shin@linecorp.com)
- * @since 2021-11-30
- *
- */
-@Getter
-@AllArgsConstructor
-public enum SlackMessageBlockType {
-    SECTION("section"),
-    MARKDOWN("mrkdwn"),
-    DIVIDER("divider"),
-    IMAGE("image"),
-    ;
-
-    private String type;
-}

shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/slack/SlackMessageDTO.java

@@ -1,23 +1,19 @@
 package com.myoa.engineering.crawl.shopping.dto.slack;
 
-import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonProperty;
-import com.myoa.engineering.crawl.shopping.support.dto.BlockMessageDTO;
+
 import lombok.Builder;
 import lombok.Getter;
 import lombok.NoArgsConstructor;
 
-import java.util.List;
-
 /**
  * SlackMessageDTO
- *
  * @author Shin Woo-jin (woo-jin.shin@linecorp.com)
  * @since 2021-11-14
+ *
  */
 @Getter
 @NoArgsConstructor
-@JsonInclude(JsonInclude.Include.NON_NULL)
 public class SlackMessageDTO implements MessageDTO {
 
     private final static long serialVersionUID = 4737608709660494713L;
@@ -25,37 +21,19 @@ public class SlackMessageDTO implements MessageDTO {
     private String text;
     private String channel;
     private String username;
-    private List<SlackMessageBlock> blocks;
 
     @JsonProperty("icon_emoji")
     private String iconEmoji;
 
     @Builder
-    public SlackMessageDTO(String text, String channel, String username,
+    public SlackMessageDTO(String text, String channel, String username, String iconEmoji) {
-                           List<SlackMessageBlock> blocks, String iconEmoji) {
         this.text = text;
         this.channel = channel;
         this.username = username;
-        this.blocks = blocks;
         this.iconEmoji = iconEmoji;
     }
 
     public void applyText(String text) {
         this.text = text;
     }
-
-    public void addSectionBlock(BlockMessageDTO.Block block) {
-        SlackSectionMessageBlock slackSectionMessageBlock = SlackSectionMessageBlock.ofMarkDown(block.getText());
-        slackSectionMessageBlock.applyImageaccessory(block.getImageUrl(), block.getAltText());
-        addBlock(slackSectionMessageBlock);
-    }
-
-    public void addSectionBlock(String rawBlockMessage) {
-        addBlock(SlackSectionMessageBlock.ofMarkDown(rawBlockMessage));
-    }
-
-    public void addBlock(SlackMessageBlock block) {
-        blocks.add(block);
-    }
-
 }

shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/slack/SlackSectionMessageBlock.java

@@ -1,48 +0,0 @@
-package com.myoa.engineering.crawl.shopping.dto.slack;
-
-import com.fasterxml.jackson.annotation.JsonInclude;
-import lombok.Builder;
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-
-/**
- * SectionBlock
- *
- * @author Shin Woo-jin (woo-jin.shin@linecorp.com)
- * @since 2021-11-30
- */
-@Getter
-@NoArgsConstructor
-@JsonInclude(JsonInclude.Include.NON_NULL)
-public class SlackSectionMessageBlock implements SlackMessageBlock {
-    private static final long serialVersionUID = -7600944576753160168L;
-
-    private SlackMessageBlockType type;
-    private SlackBaseMessageBlock text;
-    private SlackImageMessageBlock accessory;
-
-    @Builder
-    private SlackSectionMessageBlock(SlackMessageBlockType type, SlackBaseMessageBlock text,
-                                     SlackImageMessageBlock accessory) {
-        this.type = type;
-        this.text = text;
-        this.accessory = accessory;
-    }
-
-    public static SlackSectionMessageBlock ofMarkDown(String message) {
-        return SlackSectionMessageBlock.builder()
-                                       .type(SlackMessageBlockType.SECTION)
-                                       .text(SlackBaseMessageBlock.ofMarkDown(message))
-                                       .build();
-    }
-
-    public SlackSectionMessageBlock applyImageaccessory(String imageUrl, String altText) {
-        this.accessory = SlackImageMessageBlock.of(imageUrl, altText);
-        return this;
-    }
-
-    @Override
-    public String getType() {
-        return type.getType();
-    }
-}

shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/MessageSenderAPIClient.java

@@ -0,0 +1,43 @@
+package com.myoa.engineering.crawl.shopping.infra.client;
+
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Component;
+
+/**
+ * PpomppuNotifierSenderAPIClient
+ *
+ * @author Shin Woo-jin (woo-jin.shin@linecorp.com)
+ * @since 2021-11-17
+ */
+@Slf4j
+@Component
+public class MessageSenderAPIClient {
+/*
+    private final WebClient webClient;
+
+    public MessageSenderAPIClient(WebClientProperties webClientProperties) {
+        WebClientPropertiesUnit webClientPropertiesUnit =
+            webClientProperties.find(WebClientPropertiesUnitName.PPOMPPU_NOTIFIER_SENDER_API.getUnitName());
+        this.webClient = WebClient.builder()
+                                  .baseUrl(webClientPropertiesUnit.getBaseUrl())
+                                  .exchangeStrategies(WebFluxExchangeStragiesFactory.ofDefault())
+                                  .defaultHeader(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_VALUE)
+//                                  .filter(WebClientFilterFactory.logRequest())
+//                                  .filter(WebClientFilterFactory.logResponse())
+                                  .build();
+    }
+
+    public Mono<String> sendMessageToSlack(SimpleMessageDTO dto) {
+        return webClient.post()
+                        .uri("/api/v1/messages/sendMessage/messengers/slack")
+                        .bodyValue(dto)
+                        .exchangeToMono(e -> e.bodyToMono(new ParameterizedTypeReference<String>() {}))
+                        .publishOn(Schedulers.boundedElastic())
+                        .onErrorResume(WebClientRequestException.class, t -> {
+                            log.info("Exception occured, ignoring. : {}", t.getClass().getSimpleName());
+                            return Mono.empty();
+                        });
+    }
+
+ */
+}

shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/slack/MongeShoppingBotSlackMessageSender.java

@@ -0,0 +1,41 @@
+package com.myoa.engineering.crawl.shopping.infra.client.slack;
+
+import com.myoa.engineering.crawl.shopping.configuration.slack.properties.SlackSecretProperties;
+import com.myoa.engineering.crawl.shopping.dto.slack.SlackMessageDTO;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Component;
+
+@Slf4j
+@Component
+public class MongeShoppingBotSlackMessageSender {
+
+    private static final String SLACK_SECRET_UNIT_NAME = "shopping-crawler";
+
+    private final SlackSecretProperties.SlackSecretPropertiesUnit slackProperties;
+    private final SlackAPIClient slackAPIClient;
+    private final String token;
+
+    public MongeShoppingBotSlackMessageSender(SlackAPIClient slackAPIClient,
+                                              SlackSecretProperties slackSecretProperties) {
+        this.slackAPIClient = slackAPIClient;
+        this.slackProperties = slackSecretProperties.find(SLACK_SECRET_UNIT_NAME);
+        this.token = slackProperties.getToken();
+    }
+
+    public SlackMessageDTO ofMessageTemplate() {
+        return SlackMessageDTO.builder()
+                              .channel(slackProperties.getChannel())
+                              .iconEmoji(slackProperties.getIconEmoji())
+                              .username(slackProperties.getUsername())
+                              .build();
+    }
+
+    public SlackMessageDTO ofMessage(String text) {
+        return SlackMessageDTO.builder()
+                              .channel(slackProperties.getChannel())
+                              .iconEmoji(slackProperties.getIconEmoji())
+                              .username(slackProperties.getUsername())
+                              .text(text)
+                              .build();
+    }
+}

shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/slack/SlackMessageSender.java

@@ -0,0 +1,45 @@
+package com.myoa.engineering.crawl.shopping.infra.client.slack;
+
+import lombok.extern.slf4j.Slf4j;
+
+/**
+ * SlackMessageSender
+ *
+ * @author Shin Woo-jin (woozu.shin@kakaoent.com)
+ * @since 2021-09-08
+ */
+@Slf4j
+public class SlackMessageSender { /* implements MessageSender<SlackMessageDTO> {*/
+
+/*
+    private static final String SLACK_API_URL = "https://slack.com/api";
+
+    private final WebClient webClient;
+
+    public SlackMessageSender(String apiSecret) {
+        this.webClient = WebClient.builder()
+                                  .baseUrl(SLACK_API_URL)
+                                  .defaultHeader(HttpHeaders.AUTHORIZATION, "Bearer " + apiSecret)
+                                  .exchangeStrategies(WebFluxExchangeStragiesFactory.ofDefault())
+                                  .defaultHeader(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_UTF8_VALUE)
+                                  .defaultHeader(HttpHeaders.ACCEPT_CHARSET, "UTF-8")
+                                  .filter(WebClientFilterFactory.logResponse())
+                                  .build();
+    }
+
+    @Override
+    public Mono<String> sendMessage(SlackMessageDTO message) {
+        return webClient.post()
+                        .uri("/chat.postMessage")
+                        .bodyValue(message)
+                        .exchangeToMono(e -> e.bodyToMono(String.class))
+                        .publishOn(Schedulers.boundedElastic())
+                        .onErrorResume(WebClientRequestException.class, t -> {
+                            log.info("Exception occured, ignoring. : {}", t.getClass().getSimpleName());
+                            return Mono.empty();
+                        });
+    }
+
+ */
+
+}

shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/ArticleCommandService.java

@@ -4,7 +4,7 @@ import com.myoa.engineering.crawl.shopping.domain.entity.v2.Article;
 import com.myoa.engineering.crawl.shopping.domain.model.v2.ArticleModel;
 import com.myoa.engineering.crawl.shopping.event.ArticleUpsertEvent;
 import com.myoa.engineering.crawl.shopping.infra.repository.v2.ArticleRepository;
-import javax.transaction.Transactional;
+import jakarta.transaction.Transactional;
 import org.springframework.context.ApplicationEventPublisher;
 import org.springframework.stereotype.Service;
 

shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/MessageSenderService.java

@@ -0,0 +1,32 @@
+package com.myoa.engineering.crawl.shopping.service;
+
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Service;
+
+/**
+ * MessageSenderService
+ * @author Shin Woo-jin (woo-jin.shin@linecorp.com)
+ * @since 2021-11-21
+ *
+ */
+@Slf4j
+@Service
+public class MessageSenderService {
+
+    /*
+    private final MessageSenderAPIClient messageSenderAPIClient;
+
+    public MessageSenderService(MessageSenderAPIClient messageSenderAPIClient) {
+        this.messageSenderAPIClient = messageSenderAPIClient;
+    }
+
+    public String sendMessageToSlack(PpomppuArticle article) {
+        return messageSenderAPIClient.sendMessageToSlack(PpomppuArticleTransformer.TRANSFORM_TO_MESSAGE_DTO.apply(article));
+    }
+
+    public String sendMessageToSlack(List<PpomppuArticle> articles) {
+        return messageSenderAPIClient.sendMessageToSlack(PpomppuArticleTransformer.transform(articles));
+    }
+    */
+
+}

shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/PpomppuArticleService.java

@@ -48,10 +48,7 @@ public class PpomppuArticleService {
 
         // save PpomppuBoardFeedStatus
         Optional<PpomppuBoardFeedStatus> boardFeedStatus = ppomppuBoardFeedStatusRepository.findByBoardName(boardName);
-        log.info("[save] boardName: {}, isPresent?: {}, latestArticleId: {}",
+        log.info("boardName: {}, isPresent?: {}", boardName, boardFeedStatus.isPresent());
-                 boardName, boardFeedStatus.isPresent(), latestArticleId);
-        log.info("[save] articles count: {}, article ids: {}",
-                 articles.size(), articles.stream().map(PpomppuArticle::getArticleId).toArray());
         boardFeedStatus.ifPresentOrElse(e -> {
                                             if (latestArticleId.longValue() > 0L) {
                                                 e.updateArticleId(latestArticleId);

shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/PpomppuFeedService.java

@@ -0,0 +1,57 @@
+package com.myoa.engineering.crawl.shopping.service;
+
+import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName;
+import lombok.extern.slf4j.Slf4j;
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Element;
+import org.springframework.stereotype.Component;
+
+import java.util.List;
+
+/**
+ * PpomppuFeedService
+ *
+ * @author Shin Woo-jin (woozu.shin@kakaoent.com)
+ * @since 2021-09-08
+ */
+@Slf4j
+@Component
+public class PpomppuFeedService {
+/*
+    private final PpomppuBoardFeedRetriever ppomppuBoardFeedRetriever;
+
+    public PpomppuFeedService(PpomppuBoardFeedRetriever ppomppuBoardFeedRetriever) {
+        this.ppomppuBoardFeedRetriever = ppomppuBoardFeedRetriever;
+    }
+
+    public Mono<List<PpomppuArticle>> getArticles(PpomppuBoardName boardName) {
+        final Mono<String> html = ppomppuBoardFeedRetriever.getHtml(boardName.getResourcePath());
+        final Mono<Element> tbody = extractTbodyFromHtml(html);
+            // .doOnNext(e -> log.info("pre tbody - {}", e.html()));
+        return extractArticlesFromTbody(tbody).map(this::convertFromElement)
+                                              .map(e -> e.updateBoardName(boardName))
+                                              // .doOnNext(e -> log.info("parsed Result: {}", e))
+                                              .collectList();
+    }
+
+    private Mono<Element> extractTbodyFromHtml(Mono<String> html) {
+        return html.map(Jsoup::parse)
+                   .mapNotNull(e -> e.getElementById("revolution_main_table"))
+                   .map(e -> e.getElementsByTag("tbody"))
+                   // .doOnNext(e -> log.info("tbody - {}", e.html()))
+                   .map(e -> e.stream()
+                              .findFirst()
+                              .orElseThrow(() -> new IndexOutOfBoundsException("no tbody")));
+    }
+
+    private Flux<Element> extractArticlesFromTbody(Mono<Element> tbody) {
+        return Flux.concat(tbody.flatMapMany(e -> Flux.fromArray(e.select("tr.list0").toArray(Element[]::new))),
+                           tbody.flatMapMany(e -> Flux.fromArray(e.select("tr.list1").toArray(Element[]::new))));
+    }
+
+    private PpomppuArticle convertFromElement(Element element) {
+        return PpomppuArticleParser.toArticle(element.getElementsByTag("td"));
+    }
+
+ */
+}

shopping-crawler/src/main/resources/application-development.yml

@@ -2,11 +2,11 @@ spring:
   config:
     activate:
       on-profile: development
-    import:
+#    import:
-      - "configserver:http://192.168.0.100:11080"
+#      - "configserver:http://192.168.0.100:20085"
 
 
 server:
-  port: 20081
+  port: 20080
 
     # import: optional:configserver:http://localhost:11080 # can be start up even config server was not found.

shopping-crawler/src/main/resources/application.yml

@@ -8,7 +8,7 @@ spring:
     group:
       local: "local,datasource-local,webclient-local"
       development: "development,datasource-development,webclient-development"
-      production: "production,datasource-production,webclient-production"
+      production: "production, datasource-production,webclient-production"
   freemarker:
     enabled: false
   cloud:
@@ -25,4 +25,4 @@ management:
   endpoints:
     web:
       exposure:
-        include: refresh,health
+        include: refresh

shopping-crawler/src/test/java/com/myoa/engineering/crawl/shopping/crawlhandler/parser/PpomppuArticleParserV2Test.java

@@ -1,6 +1,118 @@
 package com.myoa.engineering.crawl.shopping.crawlhandler.parser;
 
+import com.myoa.engineering.crawl.shopping.domain.entity.v1.PpomppuArticle;
+import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName;
+import com.myoa.engineering.crawl.shopping.util.DateTimeUtils;
+import com.myoa.engineering.crawl.shopping.util.NumberUtils;
+import com.myoa.engineering.crawl.shopping.util.TestDataUtils;
+import io.micrometer.core.instrument.util.StringUtils;
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Document;
+import org.jsoup.nodes.Element;
+import org.jsoup.select.Elements;
+import org.junit.jupiter.api.Test;
+
+import java.time.ZonedDateTime;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
 class PpomppuArticleParserV2Test {
 
+    @Test
+    void test1() {
+        String data = TestDataUtils.fileToString("testdata/zboard/file1.html");
+        Document document = Jsoup.parse(data);
+        Elements trList = document.getElementById("revolution_main_table").getElementsByTag("tr");
 
+        List<PpomppuArticle> articles = trList.stream()
+                                              .filter(this::isRealArticle)
+                                              .map(this::parse)
+                                              .toList();
+        System.out.println(articles.size());
+        System.out.println(articles.get(0));
+        System.out.println(articles.get(1));
+    }
+
+    private boolean isRealArticle(Element tr) {
+        Elements tdList = tr.getElementsByTag("td");
+        if (tdList.size() != 6) {
+            return false;
+        }
+
+        if (!hasOnlyNumeric(tdList.get(0))) {
+            return false;
+        }
+
+        return true;
+    }
+
+
+    Pattern pattern_numeric = Pattern.compile("\\d+");
+
+    private boolean hasOnlyNumeric(Element td) {
+        return pattern_numeric.matcher(td.text()).matches();
+    }
+
+    public PpomppuArticle parse(Element tr) {
+        Elements tdList = tr.getElementsByTag("td");
+        Long articleId = Long.parseLong(tdList.get(0).text());
+
+        String title = tdList.get(2).text();
+        String articleUrl = parseArticleUrl(tdList.get(2).getElementsByTag("a").attr("href"));
+        PpomppuBoardName boardName = parseBoardName(title);
+        Integer recommended = parseRecommended(tdList.get(4));
+        Integer hit = NumberUtils.parseInt(tdList.get(5).text(), 0);
+
+
+        ZonedDateTime registeredAt = DateTimeUtils.parse(tdList.get(3).text());
+
+        return PpomppuArticle.builder()
+                             .articleId(articleId)
+                             .title(title)
+                             .boardName(boardName)
+                             .articleUrl(articleUrl)
+                             .recommended(recommended)
+                             .hit(hit)
+                             .registeredAt(registeredAt)
+                             .build();
+    }
+
+    public Long parseArticleId(Element td) {
+        return Long.parseLong(td.text().trim());
+    }
+
+    public static Integer parseRecommended(Element td) {
+        final String voteString = td.text();
+        if (StringUtils.isEmpty(voteString)) {
+            return null;
+        }
+
+        final int voteUp = Integer.parseInt(td.text().split(" - ")[0]);
+        final int voteDown = Integer.parseInt(td.text().split(" - ")[1]);
+        int recommended = voteUp - voteDown;
+        return recommended;
+    }
+
+    public static String parseArticleUrl(String data) {
+        return PpomppuBoardName.ofViewPageUrl(data);
+    }
+
+
+    Pattern patternBoardName = Pattern.compile("\\[(.+?)\\]");
+
+    public PpomppuBoardName parseBoardName(String fullTitle) {
+        Matcher matcher = patternBoardName.matcher(fullTitle);
+        String lastMatched = null;
+        while (matcher.find()) {
+            lastMatched = matcher.group(1);
+        }
+        return PpomppuBoardName.ofBoardName(lastMatched, true);
+    }
+
+    @Test
+    void test2() {
+        PpomppuBoardName boardName = parseBoardName("[자사몰]푸마 메쉬 폼스트라이프 [에디션] 5종 [세트] (18,220원/무료)6 [의류/잡화]");
+        System.out.println(boardName);
+    }
 }

support/src/main/java/com/myoa/engineering/crawl/shopping/support/dto/BlockMessageDTO.java

@@ -1,67 +0,0 @@
-package com.myoa.engineering.crawl.shopping.support.dto;
-
-import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName;
-import lombok.Builder;
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-
-import java.io.Serializable;
-import java.time.Instant;
-import java.util.List;
-
-/**
- * SimpleMessageDTO
- *
- * @author Shin Woo-jin (woo-jin.shin@linecorp.com)
- * @since 2021-11-21
- */
-@Getter
-@NoArgsConstructor
-public class BlockMessageDTO implements Serializable {
-
-    private static final long serialVersionUID = -6992039884035135523L;
-
-    private PpomppuBoardName boardName;
-    private String title;
-    private List<Block> blocks;
-    private String url;
-    private Instant publishedAt;
-    private Instant requestedAt;
-
-    @Builder
-    public BlockMessageDTO(PpomppuBoardName boardName, String title,
-                           List<Block> blocks, String url, Instant publishedAt,
-                           Instant requestedAt) {
-        this.boardName = boardName;
-        this.title = title;
-        this.blocks = blocks;
-        this.url = url;
-        this.publishedAt = publishedAt;
-        this.requestedAt = requestedAt;
-    }
-
-    @Getter
-    @NoArgsConstructor
-    public static class Block implements Serializable {
-        private static final long serialVersionUID = 3633781631892663709L;
-
-        private String text;
-        private String imageUrl;
-        private String altText;
-
-        public Block(String text, String imageUrl, String altText) {
-            this.text = text;
-            this.imageUrl = imageUrl;
-            this.altText = altText;
-        }
-    }
-
-    public static Block createBlock(String text, String imageUrl) {
-        return new Block(text, imageUrl, "");
-    }
-
-    public static Block createBlock(String text, String imageUrl, String altText) {
-        return new Block(text, imageUrl, altText);
-    }
-
-}

support/src/main/java/com/myoa/engineering/crawl/shopping/support/util/DateUtil.java

@@ -1,18 +0,0 @@
-package com.myoa.engineering.crawl.shopping.support.util;
-
-import java.time.ZoneId;
-import java.time.format.DateTimeFormatter;
-
-/**
- * DateUtil
- * @author Shin Woo-jin (woo-jin.shin@linecorp.com)
- * @since 2022-01-02
- *
- */
-public final class DateUtil {
-
-    private DateUtil() {    }
-
-    public static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yy.MM.dd HH:mm:ss")
-                                                                                  .withZone(ZoneId.of("Asia/Seoul"));
-}

support/src/main/java/com/myoa/engineering/crawl/shopping/support/util/ObjectMapperFactory.java

@@ -2,7 +2,6 @@ package com.myoa.engineering.crawl.shopping.support.util;
 
 import com.fasterxml.jackson.core.JsonGenerator;
 import com.fasterxml.jackson.core.JsonParser.Feature;
-import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.DeserializationFeature;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.SerializationFeature;
@@ -59,15 +58,6 @@ public final class ObjectMapperFactory {
         return objectMapper;
     }
 
-    public static String writeAsString(Object o) {
-        try {
-            return defaultMapper().writeValueAsString(o);
-        } catch (JsonProcessingException e) {
-            e.printStackTrace();
-            return null;
-        }
-    }
-
 
     /**
      * Copy from {@link GenericJackson2JsonRedisSerializer.NullValueSerializer}.