MyoaEngineering/ShoppingCrawler
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge branch 'feature/NO-ISSUE-v2' into develop

Browse Source
This commit is contained in:
woozu-shin
2024-05-09 08:50:05 +09:00
parent 4738b930df 7b015e8093
commit 08737664f4
150 changed files with 2775 additions and 1230 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
support/build.gradle
View File

@@ -3,7 +3,7 @@ dependencies {
// https://projectreactor.io/docs/core/release/reference/#debug-activate
annotationProcessor 'org.projectlombok:lombok'
implementation 'org.springframework.boot:spring-boot-starter-webflux'
implementation 'org.springframework.boot:spring-boot-starter-web'
}

19
support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/webclient/PpomppuNotifierWebClientConfiguration.java
View File

@@ -1,19 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.support.webclient;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.Configuration;
/**
* PpomppuNotifierWebClientConfiguration
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-11-18
*
*/
@Configuration
@ConditionalOnProperty(value = "webclient.init", havingValue = "true")
@ComponentScan(basePackageClasses = WebClientBaseScan.class)
public class PpomppuNotifierWebClientConfiguration {
}

4
support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/webclient/WebClientBaseScan.java
View File

@@ -1,4 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.support.webclient;
public interface WebClientBaseScan {
}

2
support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/dto/APIResponse.java → support/src/main/java/com/myoa/engineering/crawl/shopping/support/dto/APIResponse.java
View File

@@ -1,4 +1,4 @@
package com.myoa.engineering.crawl.ppomppu.support.dto;
package com.myoa.engineering.crawl.shopping.support.dto;
import java.io.Serializable;
import java.util.Map;

2
support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/dto/APIResponseError.java → support/src/main/java/com/myoa/engineering/crawl/shopping/support/dto/APIResponseError.java
View File

@@ -1,4 +1,4 @@
package com.myoa.engineering.crawl.ppomppu.support.dto;
package com.myoa.engineering.crawl.shopping.support.dto;
import java.io.Serializable;
import java.util.HashMap;

0
support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/dto/BlockMessageDTO.java → support/src/main/java/com/myoa/engineering/crawl/shopping/support/dto/BlockMessageDTO.java
View File

2
support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/dto/SimpleMessageDTO.java → support/src/main/java/com/myoa/engineering/crawl/shopping/support/dto/SimpleMessageDTO.java
View File

@@ -1,4 +1,4 @@
package com.myoa.engineering.crawl.ppomppu.support.dto;
package com.myoa.engineering.crawl.shopping.support.dto;
import java.io.Serializable;
import java.time.Instant;

13
support/src/main/java/com/myoa/engineering/crawl/shopping/support/dto/constant/CrawlTarget.java Normal file
View File

@@ -0,0 +1,13 @@
package com.myoa.engineering.crawl.shopping.support.dto.constant;
import lombok.AllArgsConstructor;
import lombok.Getter;
@Getter
@AllArgsConstructor
public enum CrawlTarget {
PPOMPPU_DOMESTIC,
PPOMPPU_OVERSEA,
FMKOREA,
;
}

62
support/src/main/java/com/myoa/engineering/crawl/shopping/support/dto/constant/PpomppuBoardName.java Normal file
View File

@@ -0,0 +1,62 @@
package com.myoa.engineering.crawl.shopping.support.dto.constant;
import lombok.AllArgsConstructor;
import lombok.Getter;
import java.util.Arrays;
/**
* PpomppuBoardName
*
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-09-05
*/
@Getter
@AllArgsConstructor
public enum PpomppuBoardName {
UNKNOWN("", "UNKNOWN", false, false),
PPOMPPU_DOMESTIC_ALL("id=ppomppu", "전체", false, true),
PPOMPPU_DOMESTIC_ETC("id=ppomppu&category=1", "기타", true, true),
PPOMPPU_DOMESTIC_COMPUTER("id=ppomppu&category=4", "컴퓨터", true, true),
PPOMPPU_DOMESTIC_DIGITAL("id=ppomppu&category=5", "디지털", true, true),
PPOMPPU_DOMESTIC_FOOD("id=ppomppu&category=6", "식품/건강", true, true),
PPOMPPU_DOMESTIC_BOOK("id=ppomppu&category=8", "서적", true, true),
PPOMPPU_DOMESTIC_APPLIANCES("id=ppomppu&category=9", "가전/가구", true, true),
PPOMPPU_DOMESTIC_PARENTING("id=ppomppu&category=10", "육아", true, true),
PPOMPPU_DOMESTIC_GIFTCARD("id=ppomppu&category=11", "상품권", true, true),
PPOMPPU_DOMESTIC_CLOTHES("id=ppomppu&category=12", "의류/잡화", true, true),
PPOMPPU_DOMESTIC_COSMETIC("id=ppomppu&category=13", "화장품", true, true),
PPOMPPU_DOMESTIC_OUTDOOR("id=ppomppu&category=15", "등산/캠핑", true, true),
PPOMPPU_OVERSEA_ALL("id=ppomppu4", "전체", false, false),
PPOMPPU_OVERSEA_ETC("id=ppomppu4&category=1", "기타", true, false),
PPOMPPU_OVERSEA_APPLIANCES("id=ppomppu4&category=7", "가전", true, false),
PPOMPPU_OVERSEA_TVAV("id=ppomppu4&category=8", "TV/영상", true, false),
PPOMPPU_OVERSEA_COMPUTER("id=ppomppu4&category=3", "컴퓨터", true, false),
PPOMPPU_OVERSEA_DIGITAL("id=ppomppu4&category=4", "디지털", true, false),
PPOMPPU_OVERSEA_MOBILEACCESSORY("id=ppomppu4&category=9", "액세서리", false, true),
PPOMPPU_OVERSEA_CLOTHES("id=ppomppu4&category=5", "의류/잡화", true, false),
PPOMPPU_OVERSEA_WATCH("id=ppomppu4&category=2", "시계", true, false),
PPOMPPU_OVERSEA_SHOES("id=ppomppu4&category=11", "신발", true, false),
PPOMPPU_OVERSEA_FOOD("id=ppomppu4&category=10", "식품/건강", true, false),
PPOMPPU_OVERSEA_PARENTING("id=ppomppu4&category=6", "육아", true, false),
;
private String resourcePath;
private String menuName;
private boolean crawlWithDefaultTimer;
private boolean isDomestic;
public static final String PPOMPPU_URL = "https://www.ppomppu.co.kr";
public static String ofViewPageUrl(String articleUrl) {
return PPOMPPU_URL + "/zboard/" + articleUrl;
}
public static PpomppuBoardName ofBoardName(String boardName, boolean isDomestic) {
return Arrays.stream(values())
.filter(e -> e.getMenuName().equals(boardName) && e.isDomestic() == isDomestic)
.findFirst()
.orElse(UNKNOWN);
}
}

0
support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/util/DateUtil.java → support/src/main/java/com/myoa/engineering/crawl/shopping/support/util/DateUtil.java
View File

2
support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/util/NumberUtil.java → support/src/main/java/com/myoa/engineering/crawl/shopping/support/util/NumberUtil.java
View File

@@ -1,4 +1,4 @@
package com.myoa.engineering.crawl.ppomppu.support.util;
package com.myoa.engineering.crawl.shopping.support.util;
public final class NumberUtil {

2
support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/util/ObjectMapperFactory.java → support/src/main/java/com/myoa/engineering/crawl/shopping/support/util/ObjectMapperFactory.java
View File

@@ -1,4 +1,4 @@
package com.myoa.engineering.crawl.ppomppu.support.util;
package com.myoa.engineering.crawl.shopping.support.util;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.core.JsonParser.Feature;

2
support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/util/ObjectUtil.java → support/src/main/java/com/myoa/engineering/crawl/shopping/support/util/ObjectUtil.java
View File

@@ -1,4 +1,4 @@
package com.myoa.engineering.crawl.ppomppu.support.util;
package com.myoa.engineering.crawl.shopping.support.util;
public final class ObjectUtil {

2
support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/util/WebRequestUtil.java → support/src/main/java/com/myoa/engineering/crawl/shopping/support/util/WebRequestUtil.java
View File

@@ -1,4 +1,4 @@
package com.myoa.engineering.crawl.ppomppu.support.util;
package com.myoa.engineering.crawl.shopping.support.util;
/**
* WebUtil

4
support/src/main/java/com/myoa/engineering/crawl/shopping/support/webclient/WebClientBaseScan.java Normal file
View File

@@ -0,0 +1,4 @@
package com.myoa.engineering.crawl.shopping.support.webclient;
public interface WebClientBaseScan {
}

8
support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/webclient/factory/WebClientFilterFactory.java → support/src/main/java/com/myoa/engineering/crawl/shopping/support/webclient/factory/WebClientFilterFactory.java
View File

@@ -1,10 +1,6 @@
package com.myoa.engineering.crawl.ppomppu.support.webclient.factory;
package com.myoa.engineering.crawl.shopping.support.webclient.factory;
import lombok.extern.slf4j.Slf4j;
import org.springframework.web.reactive.function.client.ClientRequest;
import org.springframework.web.reactive.function.client.ClientResponse;
import org.springframework.web.reactive.function.client.ExchangeFilterFunction;
import reactor.core.publisher.Mono;
/**
* WebClientFilterFactory
@@ -15,6 +11,7 @@ import reactor.core.publisher.Mono;
@Slf4j
public final class WebClientFilterFactory {
/*
private WebClientFilterFactory() {}
public static ExchangeFilterFunction logRequest() {
@@ -44,5 +41,6 @@ public final class WebClientFilterFactory {
}
return Mono.just(clientResponse);
}
*/
}

14
support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/webclient/factory/WebFluxExchangeStragiesFactory.java → support/src/main/java/com/myoa/engineering/crawl/shopping/support/webclient/factory/WebFluxExchangeStragiesFactory.java
View File

@@ -1,12 +1,4 @@
package com.myoa.engineering.crawl.ppomppu.support.webclient.factory;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.myoa.engineering.crawl.ppomppu.support.util.ObjectMapperFactory;
import org.springframework.http.codec.json.Jackson2JsonDecoder;
import org.springframework.http.codec.json.Jackson2JsonEncoder;
import org.springframework.util.MimeTypeUtils;
import org.springframework.web.reactive.function.client.ExchangeStrategies;
package com.myoa.engineering.crawl.shopping.support.webclient.factory;
/**
* WebFluxExchangeStragiesFactory
@@ -15,7 +7,7 @@ import org.springframework.web.reactive.function.client.ExchangeStrategies;
* @since 2021-09-08
*/
public final class WebFluxExchangeStragiesFactory {
/*
private WebFluxExchangeStragiesFactory() {}
public static ExchangeStrategies ofDefault() {
@@ -46,5 +38,5 @@ public final class WebFluxExchangeStragiesFactory {
})
.build();
}
*/
}

2
support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/webclient/properties/WebClientProperties.java → support/src/main/java/com/myoa/engineering/crawl/shopping/support/webclient/properties/WebClientProperties.java
View File

@@ -1,4 +1,4 @@
package com.myoa.engineering.crawl.ppomppu.support.webclient.properties;
package com.myoa.engineering.crawl.shopping.support.webclient.properties;
import java.util.ArrayList;
import java.util.List;

41
support/src/main/resources/development/database.yml
View File

@@ -1,41 +0,0 @@
spring:
jpa:
open-in-view: false
hibernate:
ddl-auto: create
datasource:
# driver-class-name: com.mysql.cj.jdbc.Driver
driver-class-name: org.h2.Driver
url: jdbc:h2:mem:ppomppu-local;DB_CLOSE_DELAY=-1
hikari:
minimum:idle: 5
maximum-pool-size: 10
idle-timeout: 600000
validation-timeout: 5000
connection-timeout: 5000
max-lifetime: 1800000
auto-commit: false
h2:
console:
enabled: true
path: /h2
port: 20082
datasource:
init: true
units:
- unit-name: ppomppu
schema-name: ppomppu-local
db-connection-url: jdbc:h2:mem:ppomppu-local
simple-connection-url: true
hibernate:
units:
- unit-name: ppomppu
dialect: org.hibernate.dialect.H2Dialect
format-sql: true
show-sql: true
hbm2ddl-auto: create
disable-auto-commit: true

2
support/src/main/resources/logback/logback-development.xml
View File

@@ -10,8 +10,6 @@
<!-- =========== include appender =========== -->
<include resource="org/springframework/boot/logging/logback/defaults.xml"/>
<include resource="org/springframework/boot/logging/logback/console-appender.xml"/>
<include resource="logback/component/logback-nelo2.xml"/>
<include resource="logback/component/logback-datachain.xml"/>
<!-- =========== root logger ============== -->
<root level="${DEFAULT_LEVEL}">
<appender-ref ref="CONSOLE"/>

2
support/src/main/resources/logback/logback-production.xml
View File

@@ -10,8 +10,6 @@
<!-- =========== include appender =========== -->
<include resource="org/springframework/boot/logging/logback/defaults.xml"/>
<include resource="org/springframework/boot/logging/logback/console-appender.xml"/>
<include resource="logback/component/logback-nelo2.xml"/>
<include resource="logback/component/logback-datachain.xml"/>
<!-- =========== root logger ============== -->
<root level="${DEFAULT_LEVEL}">
<appender-ref ref="CONSOLE"/>