Merge pull request '[PPN-210906-2] Implement Ppomppu Board Feed retriever' (#4) from feature/PPN-210906-2 into develop

Reviewed-on: OutworldDestroyer/PpomppuNotifier#4

processor/build.gradle

@@ -7,7 +7,9 @@ dependencies {
     implementation project(':support')
     // https://projectreactor.io/docs/core/release/reference/#debug-activate
     implementation 'org.springframework.boot:spring-boot-starter-webflux'
-    implementation 'org.telegram:telegrambots:5.3.0'
+    implementation 'org.springframework.boot:spring-boot-starter-data-jpa'
+    implementation 'com.rometools:rome:1.16.0'
+    implementation 'org.jsoup:jsoup:1.14.2'
 
     annotationProcessor 'org.springframework.boot:spring-boot-configuration-processor'
     annotationProcessor 'org.projectlombok:lombok'

processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/factory/WebClientFilterFactory.java

@@ -0,0 +1,48 @@
+package com.myoa.engineering.crawl.ppomppu.processor.configuration.factory;
+
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.web.reactive.function.client.ClientRequest;
+import org.springframework.web.reactive.function.client.ClientResponse;
+import org.springframework.web.reactive.function.client.ExchangeFilterFunction;
+import reactor.core.publisher.Mono;
+
+/**
+ * WebClientFilterFactory
+ *
+ * @author Shin Woo-jin (woozu.shin@kakaoent.com)
+ * @since 2021-09-07
+ */
+@Slf4j
+public final class WebClientFilterFactory {
+
+    private WebClientFilterFactory() {}
+
+    public static ExchangeFilterFunction logRequest() {
+        return ExchangeFilterFunction.ofRequestProcessor(WebClientFilterFactory::writeRequest);
+    }
+
+    public static ExchangeFilterFunction logResponse() {
+        return ExchangeFilterFunction.ofResponseProcessor(WebClientFilterFactory::writeResponse);
+    }
+
+    private static Mono<ClientRequest> writeRequest(ClientRequest clientRequest) {
+        try {
+            log.info("[WEBCLIENT REQUEST] uri : {} method : {} headers : {}",
+                     clientRequest.url(), clientRequest.method(), clientRequest.headers());
+        } catch (Exception e) {
+            log.error("[WEBCLIENT REQUEST] write request failed", e);
+        }
+        return Mono.just(clientRequest);
+    }
+
+    private static Mono<ClientResponse> writeResponse(ClientResponse clientResponse) {
+        try {
+            log.info("[WEBCLIENT REQUEST] statusCode : {} headers : {}",
+                     clientResponse.rawStatusCode(), clientResponse.headers().asHttpHeaders());
+        } catch (Exception e) {
+            log.error("[WEBCLIENT RESPONSE] write response failed", e);
+        }
+        return Mono.just(clientResponse);
+    }
+
+}

processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/factory/WebFluxExchangeStragiesFactory.java

@@ -0,0 +1,49 @@
+package com.myoa.engineering.crawl.ppomppu.processor.configuration.factory;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.myoa.engineering.crawl.ppomppu.processor.util.ObjectMapperFactory;
+import org.springframework.http.codec.json.Jackson2JsonDecoder;
+import org.springframework.http.codec.json.Jackson2JsonEncoder;
+import org.springframework.util.MimeTypeUtils;
+import org.springframework.web.reactive.function.client.ExchangeStrategies;
+
+/**
+ * WebFluxExchangeStragiesFactory
+ *
+ * @author Shin Woo-jin (woozu.shin@kakaoent.com)
+ * @since 2021-09-08
+ */
+public final class WebFluxExchangeStragiesFactory {
+
+    private WebFluxExchangeStragiesFactory() {}
+
+    public static ExchangeStrategies ofDefault() {
+        final ObjectMapper mapper = ObjectMapperFactory.defaultMapper();
+        return ExchangeStrategies.builder()
+                                 .codecs(configurer -> {
+                                     configurer.defaultCodecs().maxInMemorySize(-1);
+                                     configurer.defaultCodecs()
+                                               .jackson2JsonEncoder(new Jackson2JsonEncoder(mapper,
+                                                                                            MimeTypeUtils.APPLICATION_JSON));
+                                     configurer.defaultCodecs()
+                                               .jackson2JsonDecoder(new Jackson2JsonDecoder(mapper,
+                                                                                            MimeTypeUtils.APPLICATION_JSON));
+                                 })
+                                 .build();
+    }
+    public static ExchangeStrategies ofTextHtml() {
+        final ObjectMapper mapper = ObjectMapperFactory.defaultMapper();
+        return ExchangeStrategies.builder()
+                                 .codecs(configurer -> {
+                                     configurer.defaultCodecs().maxInMemorySize(-1);
+                                     configurer.defaultCodecs()
+                                               .jackson2JsonEncoder(new Jackson2JsonEncoder(mapper,
+                                                                                            MimeTypeUtils.TEXT_HTML));
+                                     configurer.defaultCodecs()
+                                               .jackson2JsonDecoder(new Jackson2JsonDecoder(mapper,
+                                                                                            MimeTypeUtils.TEXT_HTML));
+                                 })
+                                 .build();
+    }
+
+}

processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/controller/CrawlAPIController.java

@@ -1,30 +1,42 @@
 package com.myoa.engineering.crawl.ppomppu.processor.controller;
 
+import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
+import com.myoa.engineering.crawl.ppomppu.support.dto.APIResponse;
+import com.myoa.engineering.crawl.ppomppu.processor.dto.FeedParsedResult;
+import com.myoa.engineering.crawl.ppomppu.processor.service.PpomppuFeedService;
+import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
+import lombok.extern.slf4j.Slf4j;
 import org.springframework.web.bind.annotation.PathVariable;
 import org.springframework.web.bind.annotation.PostMapping;
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RestController;
-
+import reactor.core.publisher.Flux;
-import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
-
-import lombok.extern.slf4j.Slf4j;
 import reactor.core.publisher.Mono;
 
 /**
  * CrawlAPIController
+ *
  * @author Shin Woo-jin (woo-jin.shin@linecorp.com)
  * @since 2021-09-05
- *
  */
 @Slf4j
 @RestController
 @RequestMapping("/api/v1/crawl")
 public class CrawlAPIController {
 
+    private final PpomppuFeedService ppomppuRSSFeedService;
+
+    public CrawlAPIController(PpomppuFeedService ppomppuRSSFeedService) {
+        this.ppomppuRSSFeedService = ppomppuRSSFeedService;
+    }
+
     @PostMapping("/boards/{boardName}")
-    public Mono<String> crawlBoard(@PathVariable("boardName") PpomppuBoardName boardName) {
+    public Mono<APIResponse<FeedParsedResult>> crawlBoard(@PathVariable("boardName") PpomppuBoardName boardName) {
         log.info("got request... {}", boardName);
-        return Mono.just(boardName.getBoardPath());
+        FeedParsedResult result = FeedParsedResult.of(boardName);
+        Flux<PpomppuArticle> articles = ppomppuRSSFeedService.getArticles(boardName);
+
+        return articles.then(Mono.just(APIResponse.success(result.done())));
     }
 
 }

processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/Auditable.java

@@ -0,0 +1,26 @@
+package com.myoa.engineering.crawl.ppomppu.processor.domain;
+
+import java.io.Serializable;
+import java.time.Instant;
+import javax.persistence.Column;
+import org.springframework.data.annotation.CreatedDate;
+import org.springframework.data.annotation.LastModifiedDate;
+
+/**
+ * Auditable
+ *
+ * @author Shin Woo-jin (woozu.shin@kakaoent.com)
+ * @since 2021-09-08
+ */
+public abstract class Auditable implements Serializable {
+    private static final long serialVersionUID = -7105030870015828551L;
+
+    @Column
+    @CreatedDate
+    private Instant createdAt;
+
+    @Column
+    @LastModifiedDate
+    private Instant modifiedAt;
+
+}

processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/BaseScanDomain.java

@@ -0,0 +1,10 @@
+package com.myoa.engineering.crawl.ppomppu.processor.domain;
+
+/**
+ * BaseScanDomain
+ *
+ * @author Shin Woo-jin (woozu.shin@kakaoent.com)
+ * @since 2021-09-08
+ */
+public interface BaseScanDomain {
+}

processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/PpomppuArticle.java

@@ -0,0 +1,68 @@
+package com.myoa.engineering.crawl.ppomppu.processor.domain;
+
+import java.time.Instant;
+import javax.persistence.Column;
+import javax.persistence.GeneratedValue;
+import javax.persistence.GenerationType;
+import javax.persistence.Id;
+import lombok.Builder;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+
+/**
+ * PpomppuArticle
+ *
+ * @author Shin Woo-jin (woozu.shin@kakaoent.com)
+ * @since 2021-09-08
+ */
+@Getter
+@NoArgsConstructor
+public class PpomppuArticle extends Auditable {
+
+    @Id
+    @GeneratedValue(strategy = GenerationType.IDENTITY)
+    private Long id;
+
+    @Column
+    private Long articleId;
+
+    @Column
+    private String articleUrl;
+
+    @Column
+    private String title;
+
+    @Column
+    private Integer hit;
+
+    @Column
+    private Integer recommended;
+
+    @Column
+    private Instant registeredAt;
+
+    @Builder
+    public PpomppuArticle(Long id, Long articleId, String articleUrl, String title,
+                          Integer recommended, Integer hit, Instant registeredAt) {
+        this.id = id;
+        this.articleId = articleId;
+        this.articleUrl = articleUrl;
+        this.title = title;
+        this.recommended = recommended;
+        this.hit = hit;
+        this.registeredAt = registeredAt;
+    }
+
+    @Override
+    public String toString() {
+        return "PpomppuArticle{" +
+            "id=" + id +
+            ", articleId=" + articleId +
+            ", articleUrl='" + articleUrl + '\'' +
+            ", title='" + title + '\'' +
+            ", hit=" + hit +
+            ", recommended=" + recommended +
+            ", registeredAt=" + registeredAt +
+            '}';
+    }
+}

processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/FeedParsedResult.java

@@ -0,0 +1,45 @@
+package com.myoa.engineering.crawl.ppomppu.processor.dto;
+
+import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
+import java.io.Serializable;
+import java.time.Instant;
+import lombok.Builder;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+
+/**
+ * FeedParsedResult
+ *
+ * @author Shin Woo-jin (woozu.shin@kakaoent.com)
+ * @since 2021-09-08
+ */
+@Getter
+@NoArgsConstructor
+public class FeedParsedResult implements Serializable {
+
+    private static final long serialVersionUID = -3771310078623481348L;
+
+    private PpomppuBoardName boardName;
+    private Instant requestedAt;
+    private Instant processedAt;
+
+    @Builder
+    public FeedParsedResult(PpomppuBoardName boardName, Instant requestedAt, Instant processedAt) {
+        this.boardName = boardName;
+        this.requestedAt = requestedAt;
+        this.processedAt = processedAt;
+    }
+
+    public static FeedParsedResult of(PpomppuBoardName boardName) {
+        return FeedParsedResult.builder()
+                               .boardName(boardName)
+                               .requestedAt(Instant.now())
+                               .build();
+    }
+
+    public FeedParsedResult done() {
+        this.processedAt = Instant.now();
+        return this;
+    }
+
+}

processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/PpomppuArticleDTO.java

@@ -0,0 +1,10 @@
+package com.myoa.engineering.crawl.ppomppu.processor.dto;
+
+/**
+ * PpomppuArticle
+ *
+ * @author Shin Woo-jin (woozu.shin@kakaoent.com)
+ * @since 2021-09-08
+ */
+public class PpomppuArticleDTO {
+}

processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/PpomppuArticleTransformer.java

@@ -0,0 +1,76 @@
+package com.myoa.engineering.crawl.ppomppu.processor.dto;
+
+import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
+import java.time.Instant;
+import java.time.ZoneId;
+import java.time.format.DateTimeFormatter;
+import org.jsoup.nodes.Element;
+import org.jsoup.select.Elements;
+
+/**
+ * PpomppuArticleTransformer
+ *
+ * @author Shin Woo-jin (woozu.shin@kakaoent.com)
+ * @since 2021-09-08
+ */
+public final class PpomppuArticleTransformer {
+
+    private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yy.MM.dd HH:mm:ss")
+                                                                                  .withZone(ZoneId.of("Asia/Seoul"));
+
+    private PpomppuArticleTransformer() {}
+
+    public static PpomppuArticle toArticle(Elements articleElement) {
+        final long articleId = PpomppuArticleTransformer.toArticleId(articleElement.get(0));
+        final String title = PpomppuArticleTransformer.toTitle(articleElement.get(2));
+        final String articleUrl = PpomppuArticleTransformer.toArticleUrl(articleElement.get(2));
+        final int recommended = PpomppuArticleTransformer.toRecommended(articleElement.get(6));
+        final int hit = PpomppuArticleTransformer.toHit(articleElement.get(7));
+        final Instant registeredAt = PpomppuArticleTransformer.toRegisteredAt(articleElement.get(5));
+
+        return PpomppuArticle.builder()
+                             .articleId(articleId)
+                             .title(title)
+                             .articleUrl(articleUrl)
+                             .recommended(recommended)
+                             .hit(hit)
+                             .registeredAt(registeredAt)
+                             .build();
+    }
+
+    public static Long toArticleId(Element td) {
+        return Long.parseLong(td.text().trim());
+    }
+
+    public static String toTitle(Element td) {
+        return td.text();
+    }
+
+    public static String toArticleUrl(Element td) {
+        return td.getElementsByTag("a").attr("href");
+    }
+
+    public static Integer toRecommended(Element td) {
+        final String voteString = td.text();
+        final int recommended;
+
+        if (voteString.isEmpty()) {
+            recommended = 0;
+        } else {
+            final int voteUp = Integer.parseInt(td.text().split(" - ")[0]);
+            final int voteDown = Integer.parseInt(td.text().split(" - ")[1]);
+            recommended = voteUp - voteDown;
+        }
+        return recommended;
+    }
+
+    public static Integer toHit(Element td) {
+        return Integer.parseInt(td.text());
+    }
+
+    public static Instant toRegisteredAt(Element td) {
+        final String registeredAtString = td.attr("title");
+        return DATE_TIME_FORMATTER.parse(registeredAtString, Instant::from);
+    }
+
+}

processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/client/PpomppuBoardFeedRetriever.java

@@ -0,0 +1,46 @@
+package com.myoa.engineering.crawl.ppomppu.processor.infrastructure.client;
+
+import com.myoa.engineering.crawl.ppomppu.processor.configuration.factory.WebClientFilterFactory;
+import com.myoa.engineering.crawl.ppomppu.processor.configuration.factory.WebFluxExchangeStragiesFactory;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Component;
+import org.springframework.web.reactive.function.client.WebClient;
+import org.springframework.web.reactive.function.client.WebClientRequestException;
+import reactor.core.publisher.Mono;
+import reactor.core.scheduler.Schedulers;
+
+/**
+ * PpomppuBoardFeedRetriever
+ *
+ * @author Shin Woo-jin (woozu.shin@kakaoent.com)
+ * @since 2021-09-08
+ */
+@Slf4j
+@Component
+public class PpomppuBoardFeedRetriever {
+
+    private static final String PPOMPPU_URL = "https://www.ppomppu.co.kr/";
+
+    private final WebClient webClient;
+
+    public PpomppuBoardFeedRetriever(WebClient.Builder webClientBuilder) {
+        this.webClient = webClientBuilder.baseUrl(PPOMPPU_URL)
+                                         .exchangeStrategies(WebFluxExchangeStragiesFactory.ofTextHtml())
+                                         .filter(WebClientFilterFactory.logRequest())
+                                         .filter(WebClientFilterFactory.logResponse())
+                                         .build();
+    }
+
+    public Mono<String> getHtml(String uri) {
+        return webClient.get()
+                        .uri("/zboard/zboard.php?id=ppomppu")
+                        .exchangeToMono(e -> e.bodyToMono(String.class))
+                        .publishOn(Schedulers.boundedElastic())
+                        .onErrorResume(WebClientRequestException.class, t -> {
+                            log.info("Exception occured, ignoring. : {}", t.getClass().getSimpleName());
+                            return Mono.empty();
+                        })
+                        .doOnNext(e -> log.info("[getHtml] {}", e));
+    }
+
+}

processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/PpomppuFeedService.java

@@ -0,0 +1,56 @@
+package com.myoa.engineering.crawl.ppomppu.processor.service;
+
+import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
+import com.myoa.engineering.crawl.ppomppu.processor.dto.PpomppuArticleTransformer;
+import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.client.PpomppuBoardFeedRetriever;
+import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
+import lombok.extern.slf4j.Slf4j;
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Element;
+import org.springframework.stereotype.Component;
+import reactor.core.publisher.Flux;
+import reactor.core.publisher.Mono;
+
+/**
+ * PpomppuFeedService
+ *
+ * @author Shin Woo-jin (woozu.shin@kakaoent.com)
+ * @since 2021-09-08
+ */
+@Slf4j
+@Component
+public class PpomppuFeedService {
+
+    private final PpomppuBoardFeedRetriever ppomppuBoardFeedRetriever;
+
+    public PpomppuFeedService(PpomppuBoardFeedRetriever ppomppuBoardFeedRetriever) {
+        this.ppomppuBoardFeedRetriever = ppomppuBoardFeedRetriever;
+    }
+
+    public Flux<PpomppuArticle> getArticles(PpomppuBoardName boardName) {
+        final Mono<String> html = ppomppuBoardFeedRetriever.getHtml(boardName.getResourcePath());
+        final Mono<Element> tbody = extractTbodyFromHtml(html)
+            .doOnNext(e -> log.info("pre tbody - {}", e.html()));
+        return extractArticlesFromTbody(tbody).map(this::convertFromElement)
+                                              .doOnNext(e -> log.info("parsed Result: {}", e));
+    }
+
+    private Mono<Element> extractTbodyFromHtml(Mono<String> html) {
+        return html.map(Jsoup::parse)
+                   .mapNotNull(e -> e.getElementById("revolution_main_table"))
+                   .map(e -> e.getElementsByTag("tbody"))
+                   .doOnNext(e -> log.info("tbody - {}", e.html()))
+                   .map(e -> e.stream()
+                              .findFirst()
+                              .orElseThrow(() -> new IndexOutOfBoundsException("no tbody")));
+    }
+
+    private Flux<Element> extractArticlesFromTbody(Mono<Element> tbody) {
+        return Flux.concat(tbody.flatMapMany(e -> Flux.fromArray(e.select("tr.list0").toArray(Element[]::new))),
+                           tbody.flatMapMany(e -> Flux.fromArray(e.select("tr.list1").toArray(Element[]::new))));
+    }
+
+    private PpomppuArticle convertFromElement(Element element) {
+        return PpomppuArticleTransformer.toArticle(element.getElementsByTag("td"));
+    }
+}

processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/util/ObjectMapperFactory.java

@@ -0,0 +1,92 @@
+package com.myoa.engineering.crawl.ppomppu.processor.util;
+
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.core.JsonParser.Feature;
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.SerializationFeature;
+import com.fasterxml.jackson.databind.SerializerProvider;
+import com.fasterxml.jackson.databind.ser.std.StdSerializer;
+import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
+import java.io.IOException;
+import org.springframework.cache.support.NullValue;
+import org.springframework.util.StringUtils;
+
+/**
+ * ObjectMapperFactory
+ *
+ * @author Shin Woo-jin (woozu.shin@kakaoent.com)
+ * @since 2021-08-31
+ */
+public final class ObjectMapperFactory {
+
+    private ObjectMapperFactory() {
+    }
+
+    private static final ObjectMapper defaultMapper;
+    private static final ObjectMapper httpMapper;
+
+    static {
+        defaultMapper = initDefaultMapper();
+        httpMapper = initHttpMapper();
+    }
+
+    public static ObjectMapper defaultMapper() {
+        return defaultMapper;
+    }
+
+    public static ObjectMapper httpMapper() {
+        return httpMapper;
+    }
+
+    private static ObjectMapper initDefaultMapper() {
+        final ObjectMapper objectMapper = new ObjectMapper();
+        objectMapper.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES);
+        objectMapper.disable(SerializationFeature.FAIL_ON_EMPTY_BEANS);
+        objectMapper.disable(SerializationFeature.FAIL_ON_UNWRAPPED_TYPE_IDENTIFIERS);
+        objectMapper.enable(DeserializationFeature.READ_UNKNOWN_ENUM_VALUES_AS_NULL);
+        objectMapper.configure(Feature.ALLOW_UNQUOTED_CONTROL_CHARS, true);
+        objectMapper.enable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS);
+
+        objectMapper.registerModule(new JavaTimeModule());
+        return objectMapper;
+    }
+
+    private static ObjectMapper initHttpMapper() {
+        final ObjectMapper objectMapper = initDefaultMapper();
+        objectMapper.configure(DeserializationFeature.READ_UNKNOWN_ENUM_VALUES_USING_DEFAULT_VALUE, true);
+        return objectMapper;
+    }
+
+
+    /**
+     * Copy from {@link GenericJackson2JsonRedisSerializer.NullValueSerializer}.
+     */
+    public static class NullValueSerializer extends StdSerializer<NullValue> {
+
+        private static final long serialVersionUID = 6776419544239897328L;
+        private final String classIdentifier;
+
+        /**
+         * @param classIdentifier can be {@literal null} and will be defaulted to {@code @class}.
+         */
+        NullValueSerializer(String classIdentifier) {
+            super(NullValue.class);
+            this.classIdentifier = StringUtils.hasText(classIdentifier) ? classIdentifier : "@class";
+        }
+
+        /*
+         * (non-Javadoc)
+         * @see com.fasterxml.jackson.databind.ser.std.StdSerializer#serialize(java.lang.Object, com
+         * .fasterxml.jackson.core.JsonGenerator, com.fasterxml.jackson.databind.SerializerProvider)
+         */
+        @Override
+        public void serialize(NullValue value, JsonGenerator jgen, SerializerProvider provider)
+            throws IOException {
+
+            jgen.writeStartObject();
+            jgen.writeStringField(classIdentifier, NullValue.class.getName());
+            jgen.writeEndObject();
+        }
+    }
+}

processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/util/ObjectUtil.java

@@ -0,0 +1,121 @@
+/*
+ * Copyright (c) 2019 LINE Corporation. All rights reserved.
+ * LINE Corporation PROPRIETARY/CONFIDENTIAL. Use is subject to license terms.
+ */
+
+package com.myoa.engineering.crawl.ppomppu.processor.util;
+
+import java.util.Arrays;
+import java.util.Collection;
+
+/**
+ * NumberUtils
+ *
+ * @author Shin Woo-jin (lp12254@linecorp.com)
+ * @since 2019-10-28
+ */
+public final class ObjectUtil {
+
+    private ObjectUtil() {
+    }
+
+    /**
+     * Check if given object is null.
+     * <code>
+     * e == object 	== &gt; false e == null 	== &gt; true
+     * </code>
+     *
+     * @param e   Target object
+     * @param <E> Unfixed specific type. If you want restrict specific interface, Copy and extend qualifier.
+     * @return Is null given object?
+     */
+    public static <E> boolean isNullObject(final E e) {
+        return e == null;
+    }
+
+    /**
+     * Check if given object is not null.
+     * <code>
+     * e == object 	== &gt; false e == null 	== &gt; true
+     * </code>
+     *
+     * @param e   Target object
+     * @param <E> Unfixed specific type. If you want restrict specific interface, Copy and extend qualifier.
+     * @return Is not null given object?
+     */
+    public static <E> boolean isNotEmpty(final E e) {
+        return !isNullObject(e);
+    }
+
+    /**
+     * Check if there are any null object in given objects.
+     * <code>
+     * args == object				= &gt; false args == object, object		= &gt; false args == null, null, object 	= &gt; true args
+     * == null					= &gt; true args == null, null			= &gt; true
+     * </code>
+     *
+     * @param args Want to check objects that have null.
+     * @return Is there objects array has null?
+     */
+    public static boolean hasNullObject(Object... args) {
+        return Arrays.stream(args).anyMatch(ObjectUtil::isNullObject);
+    }
+
+    /**
+     * Check given objects are not empty.
+     * <code>
+     * args == object				= &gt; true args == object, object		= &gt; true args == null, null, object 	= &gt; false args
+     * == null					= &gt; false args == null, null			= &gt; false
+     * </code>
+     *
+     * @param args Want to check objects that have null.
+     * @return Is there objects array has null?
+     */
+    public static boolean hasAllObject(Object... args) {
+        return Arrays.stream(args).noneMatch(ObjectUtil::isNullObject);
+    }
+
+    /**
+     * Check if there are all null object in given objects.
+     * <code>
+     * args == object				= &gt; false args == object, object		= &gt; false args == null, null, object 	= &gt; false args
+     * == null 				= &gt; true args == null, null 			= &gt; true
+     * </code>
+     *
+     * @param args Want to check objects that have null.
+     * @return Is there null all of given objects?
+     */
+    public static boolean hasAllNullObjects(final Object... args) {
+        return Arrays.stream(args).allMatch(ObjectUtil::isNullObject);
+    }
+
+    /**
+     * Check if given collection object is null or empty collecton.
+     * <code>
+     * e == null 				= &gt; true e == emptyCollection 	= &gt; true e == hasElement 			= &gt; false
+     * </code>
+     *
+     * @param e   e is must be Collection object
+     * @param <E> E is must be extended Collection Class
+     * @return boolean. given collection is null or empty?
+     */
+    public static <E extends Collection<?>> boolean isNullOrEmptyCollection(final E e) {
+        return e == null || e.isEmpty();
+    }
+
+    /**
+     * Get collection's size. Even it pointed null
+     *
+     * @param e   e is must be Collection object
+     * @param <E> E is must be extended Collection Class
+     * @return integer value. given collection's size.
+     */
+    public static <E extends Collection<?>> int getCollectionSize(final E e) {
+        if (isNullOrEmptyCollection(e)) {
+            return 0;
+        }
+
+        return e.size();
+    }
+
+}

processor/src/main/resources/application-development.yml

@@ -3,4 +3,5 @@ spring:
     activate:
       on-profile: development
     import:
-      - classpath:/development/webclient.yml
+      - classpath:/development/webclient.yml
+      - classpath:/development/database.yml

processor/src/main/resources/logback-spring.xml

@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<configuration>
+  <springProperty name="DEFAULT_LEVEL_CONFIG" source="log.defaultLevel" />
+  <springProfile name="development">
+    <include resource="logback/logback-development.xml" />
+    <logger name="org.apache.kafka" level="INFO" />
+  </springProfile>
+  <springProfile name="production">
+    <include resource="logback/logback-production.xml" />
+  </springProfile>
+</configuration>

processor/src/main/resources/logback/component/logback-file.xml

@@ -0,0 +1,23 @@
+<included>
+  <property name="FILE_LOG_PATTERN"
+            value="%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{45}:%L - %msg%n" />
+  <property name="LOG_FILE_BASE" value="lcp-benefit-benefit-api" />
+  <appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
+    <file>${DIRECTORY}/${LOG_FILE_BASE}_log</file>
+    <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+      <fileNamePattern>${DIRECTORY}/${LOG_FILE_BASE}_log.%d{yyyyMMdd}.%i</fileNamePattern>
+      <timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
+        <maxFileSize>1000MB</maxFileSize>
+      </timeBasedFileNamingAndTriggeringPolicy>
+      <maxHistory>60</maxHistory>
+    </rollingPolicy>
+    <encoder>
+      <pattern>${FILE_LOG_PATTERN}</pattern>
+      <immediateFlush>${IMMEDIATE_FLUSH}</immediateFlush>
+    </encoder>
+  </appender>
+  <appender name="ASYNC_FILE" class="ch.qos.logback.classic.AsyncAppender">
+    <queueSize>1024</queueSize>
+    <appender-ref ref="FILE" />
+  </appender>
+</included>

processor/src/main/resources/logback/logback-development.xml

@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<included>
+  <!-- =========== property BETA ========= -->
+  <property name="DEFAULT_LEVEL" value="${DEFAULT_LEVEL_CONFIG:-INFO}"/>
+  <!--file-->
+  <property name="DIRECTORY" value="/home1/www/logs/supervisor"/>
+  <property name="IMMEDIATE_FLUSH" value="true"/>
+  <!--nelo2-->
+  <property name="NELO2_LEVEL" value="WARN"/>
+  <!-- =========== include appender =========== -->
+  <include resource="org/springframework/boot/logging/logback/defaults.xml"/>
+  <include resource="org/springframework/boot/logging/logback/console-appender.xml"/>
+  <include resource="logback/component/logback-nelo2.xml"/>
+  <include resource="logback/component/logback-datachain.xml"/>
+  <!-- =========== root logger ============== -->
+  <root level="${DEFAULT_LEVEL}">
+    <appender-ref ref="CONSOLE"/>
+  </root>
+</included>

processor/src/main/resources/logback/logback-production.xml

@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<included>
+  <!-- =========== property RELEASE ========= -->
+  <property name="DEFAULT_LEVEL" value="${DEFAULT_LEVEL_CONFIG:-INFO}"/>
+  <!--file-->
+  <property name="DIRECTORY" value="/home1/www/logs/supervisor"/>
+  <property name="IMMEDIATE_FLUSH" value="true"/>
+  <!--nelo2-->
+  <property name="NELO2_LEVEL" value="WARN"/>
+  <!-- =========== include appender =========== -->
+  <include resource="org/springframework/boot/logging/logback/defaults.xml"/>
+  <include resource="org/springframework/boot/logging/logback/console-appender.xml"/>
+  <include resource="logback/component/logback-nelo2.xml"/>
+  <include resource="logback/component/logback-datachain.xml"/>
+  <!-- =========== root logger ============== -->
+  <root level="${DEFAULT_LEVEL}">
+    <appender-ref ref="CONSOLE"/>
+  </root>
+</included>

receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/dto/FeedParsedResult.java

@@ -0,0 +1,25 @@
+package com.myoa.engineering.crawl.ppomppu.receiver.dto;
+
+import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
+import java.io.Serializable;
+import java.time.Instant;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+
+/**
+ * FeedParsedResult
+ *
+ * @author Shin Woo-jin (woozu.shin@kakaoent.com)
+ * @since 2021-09-08
+ */
+@Getter
+@NoArgsConstructor
+public class FeedParsedResult implements Serializable {
+
+    private static final long serialVersionUID = -3771310078623481348L;
+
+    private PpomppuBoardName boardName;
+    private Instant requestedAt;
+    private Instant processedAt;
+
+}

receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/shceduler/ParseEventEmitter.java

@@ -29,12 +29,12 @@ public class ParseEventEmitter {
     @Scheduled(fixedRate = 60 * 1000L)
     public void emitDomesticBoard() {
         log.info("[emitDomesticBoard] trigger fired!");
-        processorAPIService.emitParseEvent(PpomppuBoardName.PPOMPPU_DOMESTIC).block();
+        processorAPIService.emitParseEvent(PpomppuBoardName.PPOMPPU_DOMESTIC_ETC).block();
     }
 
     @Scheduled(fixedRate = 300 * 1000L)
     public void emitOverseaBoard() {
         log.info("[emitOverseaBoard] trigger fired!");
-        processorAPIService.emitParseEvent(PpomppuBoardName.PPOMPPU_OVERSEA).block();
+        processorAPIService.emitParseEvent(PpomppuBoardName.PPOMPPU_OVERSEA_ETC).block();
     }
 }

support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/dto/APIResponse.java

@@ -0,0 +1,63 @@
+package com.myoa.engineering.crawl.ppomppu.support.dto;
+
+import java.io.Serializable;
+import java.util.Map;
+import lombok.Getter;
+
+/**
+ * APIResponse
+ *
+ * @author Shin Woo-jin (woozu.shin@kakaoent.com)
+ * @since 2021-09-07
+ */
+@Getter
+public class APIResponse<T> implements Serializable {
+    private static final long serialVersionUID = 1523350013713908487L;
+
+    private boolean success;
+    private T result;
+    private APIResponseError error;
+
+    public APIResponse(boolean success, T result, APIResponseError error) {
+        this.success = success;
+        this.error = error;
+        this.result = result;
+    }
+
+    public APIResponse(boolean success, T result) {
+        this.success = success;
+        this.result = result;
+    }
+
+    public APIResponse(boolean success, APIResponseError error) {
+        this.success = success;
+        this.error = error;
+    }
+
+    public APIResponse() {}
+
+    public static <T> APIResponse<T> success(T result) {
+        return new APIResponse<>(true, result);
+    }
+
+    public static APIResponse<Void> success() {
+        return new APIResponse<>(true, null);
+    }
+
+    public static <T> APIResponse<T> fail(T result, String code) {
+        return new APIResponse<T>(false, result, APIResponseError.of(code));
+    }
+
+    public static APIResponse<Void> fail(String code) {
+        return new APIResponse<Void>(false, APIResponseError.of(code));
+    }
+
+    public static APIResponse<Void> fail(String code, String message) {
+        return new APIResponse<Void>(false, APIResponseError.of(code, message));
+    }
+
+    public static <K, V> APIResponse<Void> fail(String code, String message, Map<K, V> reasons) {
+        return new APIResponse<Void>(false, APIResponseError.of(code, message, reasons));
+    }
+
+}

support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/dto/APIResponseError.java

@@ -0,0 +1,49 @@
+package com.myoa.engineering.crawl.ppomppu.support.dto;
+
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.Map;
+import lombok.Getter;
+
+/**
+ * ResponseError
+ *
+ * @author Shin Woo-jin (woozu.shin@kakaoent.com)
+ * @since 2021-09-07
+ */
+@Getter
+public class APIResponseError<K, V> implements Serializable {
+
+    private static final long serialVersionUID = 931593091836887301L;
+
+    private String code;
+    private String message;
+    private Map<K, V> reasons;
+
+    public APIResponseError() {}
+
+    public APIResponseError(String code, String message, Map<K, V> reasons) {
+        this.code = code;
+        this.message = message;
+        this.reasons = reasons;
+    }
+
+    public static <K, V> APIResponseError<K, V> of(String code, String message, Map<K, V> reasons) {
+        return new APIResponseError<>(code, message, reasons);
+    }
+
+    public static APIResponseError<String, String> of(
+        String code, String message, String reasonKey, String reasonValue) {
+        final Map<String, String> reasons = new HashMap<>();
+        reasons.put(reasonKey, reasonValue);
+        return new APIResponseError<String, String>(code, message, reasons);
+    }
+
+    public static APIResponseError<Void, Void> of(String code, String message) {
+        return new APIResponseError<>(code, message, null);
+    }
+
+    public static APIResponseError<Void, Void> of(String code) {
+        return new APIResponseError<>(code, null, null);
+    }
+}

support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/dto/code/PpomppuBoardName.java

@@ -5,20 +5,41 @@ import lombok.NoArgsConstructor;
 
 /**
  * PpomppuBoardName
+ *
  * @author Shin Woo-jin (woo-jin.shin@linecorp.com)
  * @since 2021-09-05
- *
  */
 @Getter
 @NoArgsConstructor
 public enum PpomppuBoardName {
-    PPOMPPU_DOMESTIC("ppomppu"),
+    PPOMPPU_DOMESTIC_ETC("zboard/zboard.php?id=ppomppu&category=1"),
-    PPOMPPU_OVERSEA("ppomppu4"),
+    PPOMPPU_DOMESTIC_COMPUTER("zboard/zboard.php?id=ppomppu&category=4"),
+    PPOMPPU_DOMESTIC_DIGITAL("zboard/zboard.php?id=ppomppu&category=5"),
+    PPOMPPU_DOMESTIC_FOOD("zboard/zboard.php?id=ppomppu&category=6"),
+    PPOMPPU_DOMESTIC_BOOK("zboard/zboard.php?id=ppomppu&category=8"),
+    PPOMPPU_DOMESTIC_APPLIANCES("zboard/zboard.php?id=ppomppu&category=9"),
+    PPOMPPU_DOMESTIC_PARENTING("zboard/zboard.php?id=ppomppu&category=10"),
+    PPOMPPU_DOMESTIC_GIFTCARD("zboard/zboard.php?id=ppomppu&category=11"),
+    PPOMPPU_DOMESTIC_CLOTHES("zboard/zboard.php?id=ppomppu&category=12"),
+    PPOMPPU_DOMESTIC_COSMETIC("zboard/zboard.php?id=ppomppu&category=13"),
+    PPOMPPU_DOMESTIC_OUTDOOR("zboard/zboard.php?id=ppomppu&category=15"),
+    PPOMPPU_OVERSEA_ETC("zboard/zboard.php?id=ppomppu4&category=1"),
+    PPOMPPU_OVERSEA_APPLIANCES("zboard/zboard.php?id=ppomppu4&category=7"),
+    PPOMPPU_OVERSEA_TVAV("zboard/zboard.php?id=ppomppu4&category=8"),
+    PPOMPPU_OVERSEA_COMPUTER("zboard/zboard.php?id=ppomppu4&category=3"),
+    PPOMPPU_OVERSEA_DIGITAL("zboard/zboard.php?id=ppomppu4&category=4"),
+    PPOMPPU_OVERSEA_MOBILEACCESSORY("zboard/zboard.php?id=ppomppu4&category=9"),
+    PPOMPPU_OVERSEA_CLOTHES("zboard/zboard.php?id=ppomppu4&category=5"),
+    PPOMPPU_OVERSEA_WATCH("zboard/zboard.php?id=ppomppu4&category=2"),
+    PPOMPPU_OVERSEA_SHOES("zboard/zboard.php?id=ppomppu4&category=11"),
+    PPOMPPU_OVERSEA_FOOD("zboard/zboard.php?id=ppomppu4&category=10"),
+    PPOMPPU_OVERSEA_PARENTING("zboard/zboard.php?id=ppomppu4&category=6"),
     ;
 
-    private String boardPath;
+    private String resourcePath;
 
     PpomppuBoardName(String boardPath) {
-        this.boardPath = boardPath;
+        this.resourcePath = boardPath;
     }
+
 }

support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/util/WebRequestUtil.java

@@ -0,0 +1,15 @@
+package com.myoa.engineering.crawl.ppomppu.support.util;
+
+/**
+ * WebUtil
+ *
+ * @author Shin Woo-jin (woozu.shin@kakaoent.com)
+ * @since 2021-09-08
+ */
+public final class WebRequestUtil {
+
+    private WebRequestUtil() {}
+
+    public static final String HEADER_USER_AGENT_KEY = "User-Agent";
+    public static final String HEADER_USER_AGENT_VALUE = "Mozilla/5.0 (Linux; Android 8.0.0; SM-G960F Build/R16NW) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.84 Mobile Safari/537.36";
+}

support/src/main/resources/logback-spring.xml

@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<configuration>
+  <springProperty name="DEFAULT_LEVEL_CONFIG" source="log.defaultLevel" />
+  <springProfile name="development">
+    <include resource="logback/logback-development.xml" />
+    <logger name="org.apache.kafka" level="INFO" />
+  </springProfile>
+  <springProfile name="production">
+    <include resource="logback/logback-production.xml" />
+  </springProfile>
+</configuration>

support/src/main/resources/logback/component/logback-file.xml

@@ -0,0 +1,23 @@
+<included>
+  <property name="FILE_LOG_PATTERN"
+            value="%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{45}:%L - %msg%n" />
+  <property name="LOG_FILE_BASE" value="lcp-benefit-benefit-api" />
+  <appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
+    <file>${DIRECTORY}/${LOG_FILE_BASE}_log</file>
+    <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+      <fileNamePattern>${DIRECTORY}/${LOG_FILE_BASE}_log.%d{yyyyMMdd}.%i</fileNamePattern>
+      <timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
+        <maxFileSize>1000MB</maxFileSize>
+      </timeBasedFileNamingAndTriggeringPolicy>
+      <maxHistory>60</maxHistory>
+    </rollingPolicy>
+    <encoder>
+      <pattern>${FILE_LOG_PATTERN}</pattern>
+      <immediateFlush>${IMMEDIATE_FLUSH}</immediateFlush>
+    </encoder>
+  </appender>
+  <appender name="ASYNC_FILE" class="ch.qos.logback.classic.AsyncAppender">
+    <queueSize>1024</queueSize>
+    <appender-ref ref="FILE" />
+  </appender>
+</included>

support/src/main/resources/logback/logback-development.xml

@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<included>
+  <!-- =========== property BETA ========= -->
+  <property name="DEFAULT_LEVEL" value="${DEFAULT_LEVEL_CONFIG:-INFO}"/>
+  <!--file-->
+  <property name="DIRECTORY" value="/home1/www/logs/supervisor"/>
+  <property name="IMMEDIATE_FLUSH" value="true"/>
+  <!--nelo2-->
+  <property name="NELO2_LEVEL" value="WARN"/>
+  <!-- =========== include appender =========== -->
+  <include resource="org/springframework/boot/logging/logback/defaults.xml"/>
+  <include resource="org/springframework/boot/logging/logback/console-appender.xml"/>
+  <include resource="logback/component/logback-nelo2.xml"/>
+  <include resource="logback/component/logback-datachain.xml"/>
+  <!-- =========== root logger ============== -->
+  <root level="${DEFAULT_LEVEL}">
+    <appender-ref ref="CONSOLE"/>
+  </root>
+</included>

support/src/main/resources/logback/logback-production.xml

@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<included>
+  <!-- =========== property RELEASE ========= -->
+  <property name="DEFAULT_LEVEL" value="${DEFAULT_LEVEL_CONFIG:-INFO}"/>
+  <!--file-->
+  <property name="DIRECTORY" value="/home1/www/logs/supervisor"/>
+  <property name="IMMEDIATE_FLUSH" value="true"/>
+  <!--nelo2-->
+  <property name="NELO2_LEVEL" value="WARN"/>
+  <!-- =========== include appender =========== -->
+  <include resource="org/springframework/boot/logging/logback/defaults.xml"/>
+  <include resource="org/springframework/boot/logging/logback/console-appender.xml"/>
+  <include resource="logback/component/logback-nelo2.xml"/>
+  <include resource="logback/component/logback-datachain.xml"/>
+  <!-- =========== root logger ============== -->
+  <root level="${DEFAULT_LEVEL}">
+    <appender-ref ref="CONSOLE"/>
+  </root>
+</included>