Remove directory

This commit is contained in:
woozu-shin
2024-05-09 09:34:39 +09:00
parent 99bdae2ed9
commit 4d7bf4fb3c
113 changed files with 5006 additions and 0 deletions

23
support/build.gradle Normal file
View File

@@ -0,0 +1,23 @@
dependencies {
compileOnly 'org.projectlombok:lombok'
// https://projectreactor.io/docs/core/release/reference/#debug-activate
annotationProcessor 'org.projectlombok:lombok'
implementation 'org.springframework.boot:spring-boot-starter-web'
}
test {
useJUnitPlatform()
testLogging {
events "passed", "skipped", "failed"
}
}
jar {
enabled = true
}
bootJar {
enabled = false
}

View File

@@ -0,0 +1,31 @@
{
"blocks": [
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": "기타)) `[쿠팡] 타푸코 이중 진공 스텐 세라믹 코팅 텀블러 (화이트만 11,130원/로켓와우무료) 4 [기타]`<https://www.ppomppu.co.kr/zboardview.php?id=ppomppu&page=1&divpage=69&category=1&&no=404126|LINK>"
},
"accessory": {
"type": "image",
"image_url": "cdn.ppomppu.co.kr/zboard/data3/2021/1121/m_20211121184835_zfyahnow.png",
"alt_text": "alt text for image"
}
},
{
"type": "divider"
},
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": "기타)) `[롯데온] 1매 149원 올국산 KF94새부리마스크 대형 200매 (29,920원/무료배송) 58 [기타]`<https://www.ppomppu.co.kr/zboardview.php?id=ppomppu&page=1&divpage=69&category=1&&no=404113|LINK>"
},
"accessory": {
"type": "image",
"image_url": "https://s3-media3.fl.yelpcdn.com/bphoto/c7ed05m9lC2EmA3Aruue7A/o.jpg",
"alt_text": "alt text for image"
}
}
]
}

View File

@@ -0,0 +1,63 @@
package com.myoa.engineering.crawl.shopping.support.dto;
import java.io.Serializable;
import java.util.Map;
import lombok.Getter;
/**
* APIResponse
*
* @author Shin Woo-jin (woozu.shin@kakaoent.com)
* @since 2021-09-07
*/
@Getter
public class APIResponse<T> implements Serializable {
private static final long serialVersionUID = 1523350013713908487L;
private boolean success;
private T result;
private APIResponseError error;
public APIResponse(boolean success, T result, APIResponseError error) {
this.success = success;
this.error = error;
this.result = result;
}
public APIResponse(boolean success, T result) {
this.success = success;
this.result = result;
}
public APIResponse(boolean success, APIResponseError error) {
this.success = success;
this.error = error;
}
public APIResponse() {}
public static <T> APIResponse<T> success(T result) {
return new APIResponse<>(true, result);
}
public static APIResponse<Void> success() {
return new APIResponse<>(true, null);
}
public static <T> APIResponse<T> fail(T result, String code) {
return new APIResponse<T>(false, result, APIResponseError.of(code));
}
public static APIResponse<Void> fail(String code) {
return new APIResponse<Void>(false, APIResponseError.of(code));
}
public static APIResponse<Void> fail(String code, String message) {
return new APIResponse<Void>(false, APIResponseError.of(code, message));
}
public static <K, V> APIResponse<Void> fail(String code, String message, Map<K, V> reasons) {
return new APIResponse<Void>(false, APIResponseError.of(code, message, reasons));
}
}

View File

@@ -0,0 +1,49 @@
package com.myoa.engineering.crawl.shopping.support.dto;
import java.io.Serializable;
import java.util.HashMap;
import java.util.Map;
import lombok.Getter;
/**
* ResponseError
*
* @author Shin Woo-jin (woozu.shin@kakaoent.com)
* @since 2021-09-07
*/
@Getter
public class APIResponseError<K, V> implements Serializable {
private static final long serialVersionUID = 931593091836887301L;
private String code;
private String message;
private Map<K, V> reasons;
public APIResponseError() {}
public APIResponseError(String code, String message, Map<K, V> reasons) {
this.code = code;
this.message = message;
this.reasons = reasons;
}
public static <K, V> APIResponseError<K, V> of(String code, String message, Map<K, V> reasons) {
return new APIResponseError<>(code, message, reasons);
}
public static APIResponseError<String, String> of(
String code, String message, String reasonKey, String reasonValue) {
final Map<String, String> reasons = new HashMap<>();
reasons.put(reasonKey, reasonValue);
return new APIResponseError<String, String>(code, message, reasons);
}
public static APIResponseError<Void, Void> of(String code, String message) {
return new APIResponseError<>(code, message, null);
}
public static APIResponseError<Void, Void> of(String code) {
return new APIResponseError<>(code, null, null);
}
}

View File

@@ -0,0 +1,67 @@
package com.myoa.engineering.crawl.shopping.support.dto;
import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName;
import lombok.Builder;
import lombok.Getter;
import lombok.NoArgsConstructor;
import java.io.Serializable;
import java.time.Instant;
import java.util.List;
/**
* SimpleMessageDTO
*
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-11-21
*/
@Getter
@NoArgsConstructor
public class BlockMessageDTO implements Serializable {
private static final long serialVersionUID = -6992039884035135523L;
private PpomppuBoardName boardName;
private String title;
private List<Block> blocks;
private String url;
private Instant publishedAt;
private Instant requestedAt;
@Builder
public BlockMessageDTO(PpomppuBoardName boardName, String title,
List<Block> blocks, String url, Instant publishedAt,
Instant requestedAt) {
this.boardName = boardName;
this.title = title;
this.blocks = blocks;
this.url = url;
this.publishedAt = publishedAt;
this.requestedAt = requestedAt;
}
@Getter
@NoArgsConstructor
public static class Block implements Serializable {
private static final long serialVersionUID = 3633781631892663709L;
private String text;
private String imageUrl;
private String altText;
public Block(String text, String imageUrl, String altText) {
this.text = text;
this.imageUrl = imageUrl;
this.altText = altText;
}
}
public static Block createBlock(String text, String imageUrl) {
return new Block(text, imageUrl, "");
}
public static Block createBlock(String text, String imageUrl, String altText) {
return new Block(text, imageUrl, altText);
}
}

View File

@@ -0,0 +1,37 @@
package com.myoa.engineering.crawl.shopping.support.dto;
import java.io.Serializable;
import java.time.Instant;
import lombok.Builder;
import lombok.Getter;
import lombok.NoArgsConstructor;
/**
* SimpleMessageDTO
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-11-21
*
*/
@Getter
@NoArgsConstructor
public class SimpleMessageDTO implements Serializable {
private static final long serialVersionUID = 2203955567672404428L;
private String title;
private String body;
private String url;
private Instant publishedAt;
private Instant requestedAt;
@Builder
public SimpleMessageDTO(String title, String body, String url, Instant publishedAt, Instant requestedAt) {
this.title = title;
this.body = body;
this.url = url;
this.publishedAt = publishedAt;
this.requestedAt = requestedAt;
}
}

View File

@@ -0,0 +1,13 @@
package com.myoa.engineering.crawl.shopping.support.dto.constant;
import lombok.AllArgsConstructor;
import lombok.Getter;
@Getter
@AllArgsConstructor
public enum CrawlTarget {
PPOMPPU_DOMESTIC,
PPOMPPU_OVERSEA,
FMKOREA,
;
}

View File

@@ -0,0 +1,62 @@
package com.myoa.engineering.crawl.shopping.support.dto.constant;
import lombok.AllArgsConstructor;
import lombok.Getter;
import java.util.Arrays;
/**
* PpomppuBoardName
*
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-09-05
*/
@Getter
@AllArgsConstructor
public enum PpomppuBoardName {
UNKNOWN("", "UNKNOWN", false, false),
PPOMPPU_DOMESTIC_ALL("id=ppomppu", "전체", false, true),
PPOMPPU_DOMESTIC_ETC("id=ppomppu&category=1", "기타", true, true),
PPOMPPU_DOMESTIC_COMPUTER("id=ppomppu&category=4", "컴퓨터", true, true),
PPOMPPU_DOMESTIC_DIGITAL("id=ppomppu&category=5", "디지털", true, true),
PPOMPPU_DOMESTIC_FOOD("id=ppomppu&category=6", "식품/건강", true, true),
PPOMPPU_DOMESTIC_BOOK("id=ppomppu&category=8", "서적", true, true),
PPOMPPU_DOMESTIC_APPLIANCES("id=ppomppu&category=9", "가전/가구", true, true),
PPOMPPU_DOMESTIC_PARENTING("id=ppomppu&category=10", "육아", true, true),
PPOMPPU_DOMESTIC_GIFTCARD("id=ppomppu&category=11", "상품권", true, true),
PPOMPPU_DOMESTIC_CLOTHES("id=ppomppu&category=12", "의류/잡화", true, true),
PPOMPPU_DOMESTIC_COSMETIC("id=ppomppu&category=13", "화장품", true, true),
PPOMPPU_DOMESTIC_OUTDOOR("id=ppomppu&category=15", "등산/캠핑", true, true),
PPOMPPU_OVERSEA_ALL("id=ppomppu4", "전체", false, false),
PPOMPPU_OVERSEA_ETC("id=ppomppu4&category=1", "기타", true, false),
PPOMPPU_OVERSEA_APPLIANCES("id=ppomppu4&category=7", "가전", true, false),
PPOMPPU_OVERSEA_TVAV("id=ppomppu4&category=8", "TV/영상", true, false),
PPOMPPU_OVERSEA_COMPUTER("id=ppomppu4&category=3", "컴퓨터", true, false),
PPOMPPU_OVERSEA_DIGITAL("id=ppomppu4&category=4", "디지털", true, false),
PPOMPPU_OVERSEA_MOBILEACCESSORY("id=ppomppu4&category=9", "액세서리", false, true),
PPOMPPU_OVERSEA_CLOTHES("id=ppomppu4&category=5", "의류/잡화", true, false),
PPOMPPU_OVERSEA_WATCH("id=ppomppu4&category=2", "시계", true, false),
PPOMPPU_OVERSEA_SHOES("id=ppomppu4&category=11", "신발", true, false),
PPOMPPU_OVERSEA_FOOD("id=ppomppu4&category=10", "식품/건강", true, false),
PPOMPPU_OVERSEA_PARENTING("id=ppomppu4&category=6", "육아", true, false),
;
private String resourcePath;
private String menuName;
private boolean crawlWithDefaultTimer;
private boolean isDomestic;
public static final String PPOMPPU_URL = "https://www.ppomppu.co.kr";
public static String ofViewPageUrl(String articleUrl) {
return PPOMPPU_URL + "/zboard/" + articleUrl;
}
public static PpomppuBoardName ofBoardName(String boardName, boolean isDomestic) {
return Arrays.stream(values())
.filter(e -> e.getMenuName().equals(boardName) && e.isDomestic() == isDomestic)
.findFirst()
.orElse(UNKNOWN);
}
}

View File

@@ -0,0 +1,18 @@
package com.myoa.engineering.crawl.shopping.support.util;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
/**
* DateUtil
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2022-01-02
*
*/
public final class DateUtil {
private DateUtil() { }
public static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yy.MM.dd HH:mm:ss")
.withZone(ZoneId.of("Asia/Seoul"));
}

View File

@@ -0,0 +1,7 @@
package com.myoa.engineering.crawl.shopping.support.util;
public final class NumberUtil {
private NumberUtil() {}
}

View File

@@ -0,0 +1,102 @@
package com.myoa.engineering.crawl.shopping.support.util;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.core.JsonParser.Feature;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
import com.fasterxml.jackson.databind.SerializerProvider;
import com.fasterxml.jackson.databind.ser.std.StdSerializer;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import java.io.IOException;
import org.springframework.cache.support.NullValue;
import org.springframework.util.StringUtils;
/**
* ObjectMapperFactory
*
* @author Shin Woo-jin (woozu.shin@kakaoent.com)
* @since 2021-08-31
*/
public final class ObjectMapperFactory {
private ObjectMapperFactory() {
}
private static final ObjectMapper defaultMapper;
private static final ObjectMapper httpMapper;
static {
defaultMapper = initDefaultMapper();
httpMapper = initHttpMapper();
}
public static ObjectMapper defaultMapper() {
return defaultMapper;
}
public static ObjectMapper httpMapper() {
return httpMapper;
}
private static ObjectMapper initDefaultMapper() {
final ObjectMapper objectMapper = new ObjectMapper();
objectMapper.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES);
objectMapper.disable(SerializationFeature.FAIL_ON_EMPTY_BEANS);
objectMapper.disable(SerializationFeature.FAIL_ON_UNWRAPPED_TYPE_IDENTIFIERS);
objectMapper.enable(DeserializationFeature.READ_UNKNOWN_ENUM_VALUES_AS_NULL);
objectMapper.configure(Feature.ALLOW_UNQUOTED_CONTROL_CHARS, true);
objectMapper.enable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS);
objectMapper.registerModule(new JavaTimeModule());
return objectMapper;
}
private static ObjectMapper initHttpMapper() {
final ObjectMapper objectMapper = initDefaultMapper();
objectMapper.configure(DeserializationFeature.READ_UNKNOWN_ENUM_VALUES_USING_DEFAULT_VALUE, true);
return objectMapper;
}
public static String writeAsString(Object o) {
try {
return defaultMapper().writeValueAsString(o);
} catch (JsonProcessingException e) {
e.printStackTrace();
return null;
}
}
/**
* Copy from {@link GenericJackson2JsonRedisSerializer.NullValueSerializer}.
*/
public static class NullValueSerializer extends StdSerializer<NullValue> {
private static final long serialVersionUID = 6776419544239897328L;
private final String classIdentifier;
/**
* @param classIdentifier can be {@literal null} and will be defaulted to {@code @class}.
*/
NullValueSerializer(String classIdentifier) {
super(NullValue.class);
this.classIdentifier = StringUtils.hasText(classIdentifier) ? classIdentifier : "@class";
}
/*
* (non-Javadoc)
* @see com.fasterxml.jackson.databind.ser.std.StdSerializer#serialize(java.lang.Object, com
* .fasterxml.jackson.core.JsonGenerator, com.fasterxml.jackson.databind.SerializerProvider)
*/
@Override
public void serialize(NullValue value, JsonGenerator jgen, SerializerProvider provider)
throws IOException {
jgen.writeStartObject();
jgen.writeStringField(classIdentifier, NullValue.class.getName());
jgen.writeEndObject();
}
}
}

View File

@@ -0,0 +1,15 @@
package com.myoa.engineering.crawl.shopping.support.util;
public final class ObjectUtil {
private ObjectUtil() {}
public static boolean isEmpty(Object o) {
return o == null;
}
public static boolean isNotEmpty(Object o) {
return !isEmpty(o);
}
}

View File

@@ -0,0 +1,15 @@
package com.myoa.engineering.crawl.shopping.support.util;
/**
* WebUtil
*
* @author Shin Woo-jin (woozu.shin@kakaoent.com)
* @since 2021-09-08
*/
public final class WebRequestUtil {
private WebRequestUtil() {}
public static final String HEADER_USER_AGENT_KEY = "User-Agent";
public static final String HEADER_USER_AGENT_VALUE = "Mozilla/5.0 (Linux; Android 8.0.0; SM-G960F Build/R16NW) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.84 Mobile Safari/537.36";
}

View File

@@ -0,0 +1,4 @@
package com.myoa.engineering.crawl.shopping.support.webclient;
public interface WebClientBaseScan {
}

View File

@@ -0,0 +1,46 @@
package com.myoa.engineering.crawl.shopping.support.webclient.factory;
import lombok.extern.slf4j.Slf4j;
/**
* WebClientFilterFactory
*
* @author Shin Woo-jin (woozu.shin@kakaoent.com)
* @since 2021-09-07
*/
@Slf4j
public final class WebClientFilterFactory {
/*
private WebClientFilterFactory() {}
public static ExchangeFilterFunction logRequest() {
return ExchangeFilterFunction.ofRequestProcessor(WebClientFilterFactory::writeRequest);
}
public static ExchangeFilterFunction logResponse() {
return ExchangeFilterFunction.ofResponseProcessor(WebClientFilterFactory::writeResponse);
}
private static Mono<ClientRequest> writeRequest(ClientRequest clientRequest) {
try {
log.info("[WEBCLIENT REQUEST] uri : {} method : {} headers : {}, body: {}",
clientRequest.url(), clientRequest.method(), clientRequest.headers(), clientRequest.body());
} catch (Exception e) {
log.error("[WEBCLIENT REQUEST] write request failed", e);
}
return Mono.just(clientRequest);
}
private static Mono<ClientResponse> writeResponse(ClientResponse clientResponse) {
try {
log.info("[WEBCLIENT RESPONSE] statusCode : {} headers : {}",
clientResponse.rawStatusCode(), clientResponse.headers().asHttpHeaders());
} catch (Exception e) {
log.error("[WEBCLIENT RESPONSE] write response failed", e);
}
return Mono.just(clientResponse);
}
*/
}

View File

@@ -0,0 +1,42 @@
package com.myoa.engineering.crawl.shopping.support.webclient.factory;
/**
* WebFluxExchangeStragiesFactory
*
* @author Shin Woo-jin (woozu.shin@kakaoent.com)
* @since 2021-09-08
*/
public final class WebFluxExchangeStragiesFactory {
/*
private WebFluxExchangeStragiesFactory() {}
public static ExchangeStrategies ofDefault() {
final ObjectMapper mapper = ObjectMapperFactory.defaultMapper();
return ExchangeStrategies.builder()
.codecs(configurer -> {
configurer.defaultCodecs().maxInMemorySize(-1);
configurer.defaultCodecs()
.jackson2JsonEncoder(new Jackson2JsonEncoder(mapper,
MimeTypeUtils.APPLICATION_JSON));
configurer.defaultCodecs()
.jackson2JsonDecoder(new Jackson2JsonDecoder(mapper,
MimeTypeUtils.APPLICATION_JSON));
})
.build();
}
public static ExchangeStrategies ofTextHtml() {
final ObjectMapper mapper = ObjectMapperFactory.defaultMapper();
return ExchangeStrategies.builder()
.codecs(configurer -> {
configurer.defaultCodecs().maxInMemorySize(-1);
configurer.defaultCodecs()
.jackson2JsonEncoder(new Jackson2JsonEncoder(mapper,
MimeTypeUtils.TEXT_HTML));
configurer.defaultCodecs()
.jackson2JsonDecoder(new Jackson2JsonDecoder(mapper,
MimeTypeUtils.TEXT_HTML));
})
.build();
}
*/
}

View File

@@ -0,0 +1,44 @@
package com.myoa.engineering.crawl.shopping.support.webclient.properties;
import java.util.ArrayList;
import java.util.List;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.lang.NonNull;
import org.springframework.stereotype.Component;
import lombok.Data;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
/**
* WebClientPropertiesUnit
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-11-18
*
*/
@NoArgsConstructor
@Setter
@Getter
@Component
@ConfigurationProperties(prefix = "webclient")
public class WebClientProperties {
private List<WebClientPropertiesUnit> units = new ArrayList<>();
@Data
public static class WebClientPropertiesUnit {
private String unitName;
private String baseUrl;
// TODO headers
}
public WebClientPropertiesUnit find(@NonNull String unitName) {
return units.stream()
.filter(x -> x.getUnitName().equals(unitName))
.findFirst()
.orElseThrow(
() -> new IllegalArgumentException("Not found properties unit. unitName : " + unitName));
}
}

View File

@@ -0,0 +1,11 @@
<?xml version="1.0" encoding="UTF-8"?>
<configuration>
<springProperty name="DEFAULT_LEVEL_CONFIG" source="log.defaultLevel" />
<springProfile name="development">
<include resource="logback/logback-development.xml" />
<logger name="org.apache.kafka" level="INFO" />
</springProfile>
<springProfile name="production">
<include resource="logback/logback-production.xml" />
</springProfile>
</configuration>

View File

@@ -0,0 +1,23 @@
<included>
<property name="FILE_LOG_PATTERN"
value="%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{45}:%L - %msg%n" />
<property name="LOG_FILE_BASE" value="lcp-benefit-benefit-api" />
<appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${DIRECTORY}/${LOG_FILE_BASE}_log</file>
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<fileNamePattern>${DIRECTORY}/${LOG_FILE_BASE}_log.%d{yyyyMMdd}.%i</fileNamePattern>
<timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
<maxFileSize>1000MB</maxFileSize>
</timeBasedFileNamingAndTriggeringPolicy>
<maxHistory>60</maxHistory>
</rollingPolicy>
<encoder>
<pattern>${FILE_LOG_PATTERN}</pattern>
<immediateFlush>${IMMEDIATE_FLUSH}</immediateFlush>
</encoder>
</appender>
<appender name="ASYNC_FILE" class="ch.qos.logback.classic.AsyncAppender">
<queueSize>1024</queueSize>
<appender-ref ref="FILE" />
</appender>
</included>

View File

@@ -0,0 +1,17 @@
<?xml version="1.0" encoding="UTF-8"?>
<included>
<!-- =========== property BETA ========= -->
<property name="DEFAULT_LEVEL" value="${DEFAULT_LEVEL_CONFIG:-INFO}"/>
<!--file-->
<property name="DIRECTORY" value="/home1/www/logs/supervisor"/>
<property name="IMMEDIATE_FLUSH" value="true"/>
<!--nelo2-->
<property name="NELO2_LEVEL" value="WARN"/>
<!-- =========== include appender =========== -->
<include resource="org/springframework/boot/logging/logback/defaults.xml"/>
<include resource="org/springframework/boot/logging/logback/console-appender.xml"/>
<!-- =========== root logger ============== -->
<root level="${DEFAULT_LEVEL}">
<appender-ref ref="CONSOLE"/>
</root>
</included>

View File

@@ -0,0 +1,17 @@
<?xml version="1.0" encoding="UTF-8"?>
<included>
<!-- =========== property RELEASE ========= -->
<property name="DEFAULT_LEVEL" value="${DEFAULT_LEVEL_CONFIG:-INFO}"/>
<!--file-->
<property name="DIRECTORY" value="/home1/www/logs/supervisor"/>
<property name="IMMEDIATE_FLUSH" value="true"/>
<!--nelo2-->
<property name="NELO2_LEVEL" value="WARN"/>
<!-- =========== include appender =========== -->
<include resource="org/springframework/boot/logging/logback/defaults.xml"/>
<include resource="org/springframework/boot/logging/logback/console-appender.xml"/>
<!-- =========== root logger ============== -->
<root level="${DEFAULT_LEVEL}">
<appender-ref ref="CONSOLE"/>
</root>
</included>