MyoaEngineering/ShoppingCrawler
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

[NO-ISSUE] Implement v2

Browse Source
This commit is contained in:
woozu-shin
2024-04-30 22:34:49 +09:00
parent 0524a18ee5
commit 0c4be3cc05
147 changed files with 3873 additions and 546 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
support/build.gradle
View File

@@ -3,7 +3,7 @@ dependencies {
// https://projectreactor.io/docs/core/release/reference/#debug-activate
annotationProcessor 'org.projectlombok:lombok'
implementation 'org.springframework.boot:spring-boot-starter-webflux'
implementation 'org.springframework.boot:spring-boot-starter-web'
}

51
support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/dto/code/PpomppuBoardName.java
View File

@@ -1,51 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.support.dto.code;
import lombok.AllArgsConstructor;
import lombok.Getter;
/**
* PpomppuBoardName
*
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-09-05
*/
@Getter
@AllArgsConstructor
public enum PpomppuBoardName {
PPOMPPU_DOMESTIC_ALL("/zboard/zboard.php?id=ppomppu", "전체", false),
PPOMPPU_DOMESTIC_ETC("/zboard/zboard.php?id=ppomppu&category=1", "기타", true),
PPOMPPU_DOMESTIC_COMPUTER("/zboard/zboard.php?id=ppomppu&category=4", "컴퓨터", true),
PPOMPPU_DOMESTIC_DIGITAL("/zboard/zboard.php?id=ppomppu&category=5", "디지털", true),
PPOMPPU_DOMESTIC_FOOD("/zboard/zboard.php?id=ppomppu&category=6", "식품/건강", true),
PPOMPPU_DOMESTIC_BOOK("/zboard/zboard.php?id=ppomppu&category=8", "서적", true),
PPOMPPU_DOMESTIC_APPLIANCES("/zboard/zboard.php?id=ppomppu&category=9", "가전/가구", true),
PPOMPPU_DOMESTIC_PARENTING("/zboard/zboard.php?id=ppomppu&category=10", "육아", true),
PPOMPPU_DOMESTIC_GIFTCARD("/zboard/zboard.php?id=ppomppu&category=11", "상품권", true),
PPOMPPU_DOMESTIC_CLOTHES("/zboard/zboard.php?id=ppomppu&category=12", "의류/잡화", true),
PPOMPPU_DOMESTIC_COSMETIC("/zboard/zboard.php?id=ppomppu&category=13", "화장품", true),
PPOMPPU_DOMESTIC_OUTDOOR("/zboard/zboard.php?id=ppomppu&category=15", "등산/캠핑", true),
PPOMPPU_OVERSEA_ALL("/zboard/zboard.php?id=ppomppu4", "전체", false),
PPOMPPU_OVERSEA_ETC("/zboard/zboard.php?id=ppomppu4&category=1", "기타", true),
PPOMPPU_OVERSEA_APPLIANCES("/zboard/zboard.php?id=ppomppu4&category=7", "가전", true),
PPOMPPU_OVERSEA_TVAV("/zboard/zboard.php?id=ppomppu4&category=8", "TV/영상", true),
PPOMPPU_OVERSEA_COMPUTER("/zboard/zboard.php?id=ppomppu4&category=3", "컴퓨터", true),
PPOMPPU_OVERSEA_DIGITAL("/zboard/zboard.php?id=ppomppu4&category=4", "디지털", true),
PPOMPPU_OVERSEA_MOBILEACCESSORY("/zboard/zboard.php?id=ppomppu4&category=9", "액세서리", true),
PPOMPPU_OVERSEA_CLOTHES("/zboard/zboard.php?id=ppomppu4&category=5", "의류/잡화", true),
PPOMPPU_OVERSEA_WATCH("/zboard/zboard.php?id=ppomppu4&category=2", "시계", true),
PPOMPPU_OVERSEA_SHOES("/zboard/zboard.php?id=ppomppu4&category=11", "신발", true),
PPOMPPU_OVERSEA_FOOD("/zboard/zboard.php?id=ppomppu4&category=10", "식품/건강", true),
PPOMPPU_OVERSEA_PARENTING("/zboard/zboard.php?id=ppomppu4&category=6", "육아", true),
;
private String resourcePath;
private String menuName;
private boolean crawlWithDefaultTimer;
public static final String PPOMPPU_URL = "https://www.ppomppu.co.kr";
public static String ofViewPageUrl(String articleUrl) {
return PPOMPPU_URL + "/zboard/" + articleUrl;
}
}

19
support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/webclient/PpomppuNotifierWebClientConfiguration.java
View File

@@ -1,19 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.support.webclient;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.Configuration;
/**
* PpomppuNotifierWebClientConfiguration
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-11-18
*
*/
@Configuration
@ConditionalOnProperty(value = "webclient.init", havingValue = "true")
@ComponentScan(basePackageClasses = WebClientBaseScan.class)
public class PpomppuNotifierWebClientConfiguration {
}

4
support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/webclient/WebClientBaseScan.java
View File

@@ -1,4 +0,0 @@
package com.myoa.engineering.crawl.ppomppu.support.webclient;
public interface WebClientBaseScan {
}

2
support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/dto/APIResponse.java → support/src/main/java/com/myoa/engineering/crawl/shopping/support/dto/APIResponse.java
View File

@@ -1,4 +1,4 @@
package com.myoa.engineering.crawl.ppomppu.support.dto;
package com.myoa.engineering.crawl.shopping.support.dto;
import java.io.Serializable;
import java.util.Map;

2
support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/dto/APIResponseError.java → support/src/main/java/com/myoa/engineering/crawl/shopping/support/dto/APIResponseError.java
View File

@@ -1,4 +1,4 @@
package com.myoa.engineering.crawl.ppomppu.support.dto;
package com.myoa.engineering.crawl.shopping.support.dto;
import java.io.Serializable;
import java.util.HashMap;

2
support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/dto/SimpleMessageDTO.java → support/src/main/java/com/myoa/engineering/crawl/shopping/support/dto/SimpleMessageDTO.java
View File

@@ -1,4 +1,4 @@
package com.myoa.engineering.crawl.ppomppu.support.dto;
package com.myoa.engineering.crawl.shopping.support.dto;
import java.io.Serializable;
import java.time.Instant;

13
support/src/main/java/com/myoa/engineering/crawl/shopping/support/dto/constant/CrawlTarget.java Normal file
View File

@@ -0,0 +1,13 @@
package com.myoa.engineering.crawl.shopping.support.dto.constant;
import lombok.AllArgsConstructor;
import lombok.Getter;
@Getter
@AllArgsConstructor
public enum CrawlTarget {
PPOMPPU_DOMESTIC,
PPOMPPU_OVERSEA,
FMKOREA,
;
}

62
support/src/main/java/com/myoa/engineering/crawl/shopping/support/dto/constant/PpomppuBoardName.java Normal file
View File

@@ -0,0 +1,62 @@
package com.myoa.engineering.crawl.shopping.support.dto.constant;
import lombok.AllArgsConstructor;
import lombok.Getter;
import java.util.Arrays;
/**
* PpomppuBoardName
*
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-09-05
*/
@Getter
@AllArgsConstructor
public enum PpomppuBoardName {
UNKNOWN("", "UNKNOWN", false, false),
PPOMPPU_DOMESTIC_ALL("id=ppomppu", "전체", false, true),
PPOMPPU_DOMESTIC_ETC("id=ppomppu&category=1", "기타", true, true),
PPOMPPU_DOMESTIC_COMPUTER("id=ppomppu&category=4", "컴퓨터", true, true),
PPOMPPU_DOMESTIC_DIGITAL("id=ppomppu&category=5", "디지털", true, true),
PPOMPPU_DOMESTIC_FOOD("id=ppomppu&category=6", "식품/건강", true, true),
PPOMPPU_DOMESTIC_BOOK("id=ppomppu&category=8", "서적", true, true),
PPOMPPU_DOMESTIC_APPLIANCES("id=ppomppu&category=9", "가전/가구", true, true),
PPOMPPU_DOMESTIC_PARENTING("id=ppomppu&category=10", "육아", true, true),
PPOMPPU_DOMESTIC_GIFTCARD("id=ppomppu&category=11", "상품권", true, true),
PPOMPPU_DOMESTIC_CLOTHES("id=ppomppu&category=12", "의류/잡화", true, true),
PPOMPPU_DOMESTIC_COSMETIC("id=ppomppu&category=13", "화장품", true, true),
PPOMPPU_DOMESTIC_OUTDOOR("id=ppomppu&category=15", "등산/캠핑", true, true),
PPOMPPU_OVERSEA_ALL("id=ppomppu4", "전체", false, false),
PPOMPPU_OVERSEA_ETC("id=ppomppu4&category=1", "기타", true, false),
PPOMPPU_OVERSEA_APPLIANCES("id=ppomppu4&category=7", "가전", true, false),
PPOMPPU_OVERSEA_TVAV("id=ppomppu4&category=8", "TV/영상", true, false),
PPOMPPU_OVERSEA_COMPUTER("id=ppomppu4&category=3", "컴퓨터", true, false),
PPOMPPU_OVERSEA_DIGITAL("id=ppomppu4&category=4", "디지털", true, false),
PPOMPPU_OVERSEA_MOBILEACCESSORY("id=ppomppu4&category=9", "액세서리", false, true),
PPOMPPU_OVERSEA_CLOTHES("id=ppomppu4&category=5", "의류/잡화", true, false),
PPOMPPU_OVERSEA_WATCH("id=ppomppu4&category=2", "시계", true, false),
PPOMPPU_OVERSEA_SHOES("id=ppomppu4&category=11", "신발", true, false),
PPOMPPU_OVERSEA_FOOD("id=ppomppu4&category=10", "식품/건강", true, false),
PPOMPPU_OVERSEA_PARENTING("id=ppomppu4&category=6", "육아", true, false),
;
private String resourcePath;
private String menuName;
private boolean crawlWithDefaultTimer;
private boolean isDomestic;
public static final String PPOMPPU_URL = "https://www.ppomppu.co.kr";
public static String ofViewPageUrl(String articleUrl) {
return PPOMPPU_URL + "/zboard/" + articleUrl;
}
public static PpomppuBoardName ofBoardName(String boardName, boolean isDomestic) {
return Arrays.stream(values())
.filter(e -> e.getMenuName().equals(boardName) && e.isDomestic() == isDomestic)
.findFirst()
.orElse(UNKNOWN);
}
}

2
support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/util/NumberUtil.java → support/src/main/java/com/myoa/engineering/crawl/shopping/support/util/NumberUtil.java
View File

@@ -1,4 +1,4 @@
package com.myoa.engineering.crawl.ppomppu.support.util;
package com.myoa.engineering.crawl.shopping.support.util;
public final class NumberUtil {

2
support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/util/ObjectMapperFactory.java → support/src/main/java/com/myoa/engineering/crawl/shopping/support/util/ObjectMapperFactory.java
View File

@@ -1,4 +1,4 @@
package com.myoa.engineering.crawl.ppomppu.support.util;
package com.myoa.engineering.crawl.shopping.support.util;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.core.JsonParser.Feature;

2
support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/util/ObjectUtil.java → support/src/main/java/com/myoa/engineering/crawl/shopping/support/util/ObjectUtil.java
View File

@@ -1,4 +1,4 @@
package com.myoa.engineering.crawl.ppomppu.support.util;
package com.myoa.engineering.crawl.shopping.support.util;
public final class ObjectUtil {

2
support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/util/WebRequestUtil.java → support/src/main/java/com/myoa/engineering/crawl/shopping/support/util/WebRequestUtil.java
View File

@@ -1,4 +1,4 @@
package com.myoa.engineering.crawl.ppomppu.support.util;
package com.myoa.engineering.crawl.shopping.support.util;
/**
* WebUtil

4
support/src/main/java/com/myoa/engineering/crawl/shopping/support/webclient/WebClientBaseScan.java Normal file
View File

@@ -0,0 +1,4 @@
package com.myoa.engineering.crawl.shopping.support.webclient;
public interface WebClientBaseScan {
}

8
support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/webclient/factory/WebClientFilterFactory.java → support/src/main/java/com/myoa/engineering/crawl/shopping/support/webclient/factory/WebClientFilterFactory.java
View File

@@ -1,10 +1,6 @@
package com.myoa.engineering.crawl.ppomppu.support.webclient.factory;
package com.myoa.engineering.crawl.shopping.support.webclient.factory;
import lombok.extern.slf4j.Slf4j;
import org.springframework.web.reactive.function.client.ClientRequest;
import org.springframework.web.reactive.function.client.ClientResponse;
import org.springframework.web.reactive.function.client.ExchangeFilterFunction;
import reactor.core.publisher.Mono;
/**
* WebClientFilterFactory
@@ -15,6 +11,7 @@ import reactor.core.publisher.Mono;
@Slf4j
public final class WebClientFilterFactory {
/*
private WebClientFilterFactory() {}
public static ExchangeFilterFunction logRequest() {
@@ -44,5 +41,6 @@ public final class WebClientFilterFactory {
}
return Mono.just(clientResponse);
}
*/
}

14
support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/webclient/factory/WebFluxExchangeStragiesFactory.java → support/src/main/java/com/myoa/engineering/crawl/shopping/support/webclient/factory/WebFluxExchangeStragiesFactory.java
View File

@@ -1,12 +1,4 @@
package com.myoa.engineering.crawl.ppomppu.support.webclient.factory;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.myoa.engineering.crawl.ppomppu.support.util.ObjectMapperFactory;
import org.springframework.http.codec.json.Jackson2JsonDecoder;
import org.springframework.http.codec.json.Jackson2JsonEncoder;
import org.springframework.util.MimeTypeUtils;
import org.springframework.web.reactive.function.client.ExchangeStrategies;
package com.myoa.engineering.crawl.shopping.support.webclient.factory;
/**
* WebFluxExchangeStragiesFactory
@@ -15,7 +7,7 @@ import org.springframework.web.reactive.function.client.ExchangeStrategies;
* @since 2021-09-08
*/
public final class WebFluxExchangeStragiesFactory {
/*
private WebFluxExchangeStragiesFactory() {}
public static ExchangeStrategies ofDefault() {
@@ -46,5 +38,5 @@ public final class WebFluxExchangeStragiesFactory {
})
.build();
}
*/
}

2
support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/webclient/properties/WebClientProperties.java → support/src/main/java/com/myoa/engineering/crawl/shopping/support/webclient/properties/WebClientProperties.java
View File

@@ -1,4 +1,4 @@
package com.myoa.engineering.crawl.ppomppu.support.webclient.properties;
package com.myoa.engineering.crawl.shopping.support.webclient.properties;
import java.util.ArrayList;
import java.util.List;

42
support/src/main/resources/datasource/local.yml Normal file
View File

@@ -0,0 +1,42 @@
spring:
jpa:
open-in-view: false
hibernate:
ddl-auto: create
datasource:
# driver-class-name: com.mysql.cj.jdbc.Driver
driver-class-name: org.h2.Driver
url: jdbc:h2:mem:crawler-shopping;DB_CLOSE_DELAY=-1
hikari:
minimum:idle: 5
maximum-pool-size: 10
idle-timeout: 600000
validation-timeout: 5000
connection-timeout: 5000
max-lifetime: 1800000
auto-commit: false
h2:
console:
enabled: true
path: /h2
port: 20082
datasource:
init: true
units:
- unit-name: crawler-shopping
schema-name: crawler-shopping
db-connection-url: jdbc:h2:mem:crawler-shopping
is-simple-connection-url: true
driver-class-name: org.h2.Driver
username: sa
password: sa
hibernate:
units:
- unit-name: crawler-shopping
dialect: org.hibernate.dialect.H2Dialect
format-sql: true
show-sql: true
hbm2ddl-auto: create
disable-auto-commit: true

11
support/src/main/resources/development/database.yml
View File

@@ -6,7 +6,7 @@ spring:
datasource:
# driver-class-name: com.mysql.cj.jdbc.Driver
driver-class-name: org.h2.Driver
url: jdbc:h2:mem:ppomppu-local;DB_CLOSE_DELAY=-1
url: jdbc:h2:mem:crawler-shopping;DB_CLOSE_DELAY=-1
hikari:
minimum:idle: 5
maximum-pool-size: 10
@@ -24,18 +24,17 @@ spring:
datasource:
init: true
units:
- unit-name: ppomppu
schema-name: ppomppu-local
db-connection-url: jdbc:h2:mem:ppomppu-local
- unit-name: crawler-shopping
schema-name: crawler-shopping
db-connection-url: jdbc:h2:mem:crawler-shopping
simple-connection-url: true
hibernate:
units:
- unit-name: ppomppu
- unit-name: crawler-shopping
dialect: org.hibernate.dialect.H2Dialect
format-sql: true
show-sql: true
hbm2ddl-auto: create
disable-auto-commit: true

8
support/src/main/resources/slack/local.yml Normal file
View File

@@ -0,0 +1,8 @@
slack:
bot:
units:
- bot-unit-name: shopping-crawler
username: "몽이 탈호구봇"
icon-emoji: ":monge_big:"
channel: "notify_shopping"
token: "xoxb-2688454277126-2695026012277-YKlaeoSBY42NtF6Teh4z7dLK"