MyoaEngineering/ShoppingCrawler
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

[PPN-211113] Implement spring config client

Browse Source
This commit is contained in:
woo-jin.shin
2021-11-17 09:39:49 +09:00
parent 0afa52aa53
commit 5b4b44f093
22 changed files with 332 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

52
support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/dto/code/PpomppuBoardName.java
View File

@@ -12,40 +12,40 @@ import lombok.Getter;
@Getter
@AllArgsConstructor
public enum PpomppuBoardName {
PPOMPPU_DOMESTIC_ALL("zboard/zboard.php?id=ppomppu", "전체", true),
PPOMPPU_DOMESTIC_ETC("zboard/zboard.php?id=ppomppu&category=1", "기타", true),
PPOMPPU_DOMESTIC_COMPUTER("zboard/zboard.php?id=ppomppu&category=4", "컴퓨터", true),
PPOMPPU_DOMESTIC_DIGITAL("zboard/zboard.php?id=ppomppu&category=5", "디지털", true),
PPOMPPU_DOMESTIC_FOOD("zboard/zboard.php?id=ppomppu&category=6", "식품/건강", true),
PPOMPPU_DOMESTIC_BOOK("zboard/zboard.php?id=ppomppu&category=8", "서적", true),
PPOMPPU_DOMESTIC_APPLIANCES("zboard/zboard.php?id=ppomppu&category=9", "가전/가구", true),
PPOMPPU_DOMESTIC_PARENTING("zboard/zboard.php?id=ppomppu&category=10", "육아", true),
PPOMPPU_DOMESTIC_GIFTCARD("zboard/zboard.php?id=ppomppu&category=11", "상품권", true),
PPOMPPU_DOMESTIC_CLOTHES("zboard/zboard.php?id=ppomppu&category=12", "의류/잡화", true),
PPOMPPU_DOMESTIC_COSMETIC("zboard/zboard.php?id=ppomppu&category=13", "화장품", true),
PPOMPPU_DOMESTIC_OUTDOOR("zboard/zboard.php?id=ppomppu&category=15", "등산/캠핑", true),
PPOMPPU_OVERSEA_ALL("zboard/zboard.php?id=ppomppu4", "전체", true),
PPOMPPU_OVERSEA_ETC("zboard/zboard.php?id=ppomppu4&category=1", "기타", true),
PPOMPPU_OVERSEA_APPLIANCES("zboard/zboard.php?id=ppomppu4&category=7", "가전", true),
PPOMPPU_OVERSEA_TVAV("zboard/zboard.php?id=ppomppu4&category=8", "TV/영상", true),
PPOMPPU_OVERSEA_COMPUTER("zboard/zboard.php?id=ppomppu4&category=3", "컴퓨터", true),
PPOMPPU_OVERSEA_DIGITAL("zboard/zboard.php?id=ppomppu4&category=4", "디지털", true),
PPOMPPU_OVERSEA_MOBILEACCESSORY("zboard/zboard.php?id=ppomppu4&category=9", "액세서리", true),
PPOMPPU_OVERSEA_CLOTHES("zboard/zboard.php?id=ppomppu4&category=5", "의류/잡화", true),
PPOMPPU_OVERSEA_WATCH("zboard/zboard.php?id=ppomppu4&category=2", "시계", true),
PPOMPPU_OVERSEA_SHOES("zboard/zboard.php?id=ppomppu4&category=11", "신발", true),
PPOMPPU_OVERSEA_FOOD("zboard/zboard.php?id=ppomppu4&category=10", "식품/건강", true),
PPOMPPU_OVERSEA_PARENTING("zboard/zboard.php?id=ppomppu4&category=6", "육아", true),
PPOMPPU_DOMESTIC_ALL("/zboard/zboard.php?id=ppomppu", "전체", true),
PPOMPPU_DOMESTIC_ETC("/zboard/zboard.php?id=ppomppu&category=1", "기타", true),
PPOMPPU_DOMESTIC_COMPUTER("/zboard/zboard.php?id=ppomppu&category=4", "컴퓨터", true),
PPOMPPU_DOMESTIC_DIGITAL("/zboard/zboard.php?id=ppomppu&category=5", "디지털", true),
PPOMPPU_DOMESTIC_FOOD("/zboard/zboard.php?id=ppomppu&category=6", "식품/건강", true),
PPOMPPU_DOMESTIC_BOOK("/zboard/zboard.php?id=ppomppu&category=8", "서적", true),
PPOMPPU_DOMESTIC_APPLIANCES("/zboard/zboard.php?id=ppomppu&category=9", "가전/가구", true),
PPOMPPU_DOMESTIC_PARENTING("/zboard/zboard.php?id=ppomppu&category=10", "육아", true),
PPOMPPU_DOMESTIC_GIFTCARD("/zboard/zboard.php?id=ppomppu&category=11", "상품권", true),
PPOMPPU_DOMESTIC_CLOTHES("/zboard/zboard.php?id=ppomppu&category=12", "의류/잡화", true),
PPOMPPU_DOMESTIC_COSMETIC("/zboard/zboard.php?id=ppomppu&category=13", "화장품", true),
PPOMPPU_DOMESTIC_OUTDOOR("/zboard/zboard.php?id=ppomppu&category=15", "등산/캠핑", true),
PPOMPPU_OVERSEA_ALL("/zboard/zboard.php?id=ppomppu4", "전체", true),
PPOMPPU_OVERSEA_ETC("/zboard/zboard.php?id=ppomppu4&category=1", "기타", true),
PPOMPPU_OVERSEA_APPLIANCES("/zboard/zboard.php?id=ppomppu4&category=7", "가전", true),
PPOMPPU_OVERSEA_TVAV("/zboard/zboard.php?id=ppomppu4&category=8", "TV/영상", true),
PPOMPPU_OVERSEA_COMPUTER("/zboard/zboard.php?id=ppomppu4&category=3", "컴퓨터", true),
PPOMPPU_OVERSEA_DIGITAL("/zboard/zboard.php?id=ppomppu4&category=4", "디지털", true),
PPOMPPU_OVERSEA_MOBILEACCESSORY("/zboard/zboard.php?id=ppomppu4&category=9", "액세서리", true),
PPOMPPU_OVERSEA_CLOTHES("/zboard/zboard.php?id=ppomppu4&category=5", "의류/잡화", true),
PPOMPPU_OVERSEA_WATCH("/zboard/zboard.php?id=ppomppu4&category=2", "시계", true),
PPOMPPU_OVERSEA_SHOES("/zboard/zboard.php?id=ppomppu4&category=11", "신발", true),
PPOMPPU_OVERSEA_FOOD("/zboard/zboard.php?id=ppomppu4&category=10", "식품/건강", true),
PPOMPPU_OVERSEA_PARENTING("/zboard/zboard.php?id=ppomppu4&category=6", "육아", true),
;
private String resourcePath;
private String menuName;
private boolean crawlWithDefaultTimer;
public static final String PPOMPPU_URL = "https://www.ppomppu.co.kr/";
public static final String PPOMPPU_URL = "https://www.ppomppu.co.kr";
public static String ofViewPageUrl(String articleUrl) {
return PPOMPPU_URL + "zboard/" + articleUrl;
return PPOMPPU_URL + "/zboard" + articleUrl;
}
}

92
support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/util/ObjectMapperFactory.java Normal file
View File

@@ -0,0 +1,92 @@
package com.myoa.engineering.crawl.ppomppu.support.util;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.core.JsonParser.Feature;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
import com.fasterxml.jackson.databind.SerializerProvider;
import com.fasterxml.jackson.databind.ser.std.StdSerializer;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import java.io.IOException;
import org.springframework.cache.support.NullValue;
import org.springframework.util.StringUtils;
/**
* ObjectMapperFactory
*
* @author Shin Woo-jin (woozu.shin@kakaoent.com)
* @since 2021-08-31
*/
public final class ObjectMapperFactory {
private ObjectMapperFactory() {
}
private static final ObjectMapper defaultMapper;
private static final ObjectMapper httpMapper;
static {
defaultMapper = initDefaultMapper();
httpMapper = initHttpMapper();
}
public static ObjectMapper defaultMapper() {
return defaultMapper;
}
public static ObjectMapper httpMapper() {
return httpMapper;
}
private static ObjectMapper initDefaultMapper() {
final ObjectMapper objectMapper = new ObjectMapper();
objectMapper.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES);
objectMapper.disable(SerializationFeature.FAIL_ON_EMPTY_BEANS);
objectMapper.disable(SerializationFeature.FAIL_ON_UNWRAPPED_TYPE_IDENTIFIERS);
objectMapper.enable(DeserializationFeature.READ_UNKNOWN_ENUM_VALUES_AS_NULL);
objectMapper.configure(Feature.ALLOW_UNQUOTED_CONTROL_CHARS, true);
objectMapper.enable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS);
objectMapper.registerModule(new JavaTimeModule());
return objectMapper;
}
private static ObjectMapper initHttpMapper() {
final ObjectMapper objectMapper = initDefaultMapper();
objectMapper.configure(DeserializationFeature.READ_UNKNOWN_ENUM_VALUES_USING_DEFAULT_VALUE, true);
return objectMapper;
}
/**
* Copy from {@link GenericJackson2JsonRedisSerializer.NullValueSerializer}.
*/
public static class NullValueSerializer extends StdSerializer<NullValue> {
private static final long serialVersionUID = 6776419544239897328L;
private final String classIdentifier;
/**
* @param classIdentifier can be {@literal null} and will be defaulted to {@code @class}.
*/
NullValueSerializer(String classIdentifier) {
super(NullValue.class);
this.classIdentifier = StringUtils.hasText(classIdentifier) ? classIdentifier : "@class";
}
/*
* (non-Javadoc)
* @see com.fasterxml.jackson.databind.ser.std.StdSerializer#serialize(java.lang.Object, com
* .fasterxml.jackson.core.JsonGenerator, com.fasterxml.jackson.databind.SerializerProvider)
*/
@Override
public void serialize(NullValue value, JsonGenerator jgen, SerializerProvider provider)
throws IOException {
jgen.writeStartObject();
jgen.writeStringField(classIdentifier, NullValue.class.getName());
jgen.writeEndObject();
}
}
}

48
support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/webclient/factory/WebClientFilterFactory.java Normal file
View File

@@ -0,0 +1,48 @@
package com.myoa.engineering.crawl.ppomppu.support.webclient.factory;
import lombok.extern.slf4j.Slf4j;
import org.springframework.web.reactive.function.client.ClientRequest;
import org.springframework.web.reactive.function.client.ClientResponse;
import org.springframework.web.reactive.function.client.ExchangeFilterFunction;
import reactor.core.publisher.Mono;
/**
* WebClientFilterFactory
*
* @author Shin Woo-jin (woozu.shin@kakaoent.com)
* @since 2021-09-07
*/
@Slf4j
public final class WebClientFilterFactory {
private WebClientFilterFactory() {}
public static ExchangeFilterFunction logRequest() {
return ExchangeFilterFunction.ofRequestProcessor(WebClientFilterFactory::writeRequest);
}
public static ExchangeFilterFunction logResponse() {
return ExchangeFilterFunction.ofResponseProcessor(WebClientFilterFactory::writeResponse);
}
private static Mono<ClientRequest> writeRequest(ClientRequest clientRequest) {
try {
log.info("[WEBCLIENT REQUEST] uri : {} method : {} headers : {}, body: {}",
clientRequest.url(), clientRequest.method(), clientRequest.headers(), clientRequest.body());
} catch (Exception e) {
log.error("[WEBCLIENT REQUEST] write request failed", e);
}
return Mono.just(clientRequest);
}
private static Mono<ClientResponse> writeResponse(ClientResponse clientResponse) {
try {
log.info("[WEBCLIENT RESPONSE] statusCode : {} headers : {}",
clientResponse.rawStatusCode(), clientResponse.headers().asHttpHeaders());
} catch (Exception e) {
log.error("[WEBCLIENT RESPONSE] write response failed", e);
}
return Mono.just(clientResponse);
}
}

50
support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/webclient/factory/WebFluxExchangeStragiesFactory.java Normal file
View File

@@ -0,0 +1,50 @@
package com.myoa.engineering.crawl.ppomppu.support.webclient.factory;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.myoa.engineering.crawl.ppomppu.support.util.ObjectMapperFactory;
import org.springframework.http.codec.json.Jackson2JsonDecoder;
import org.springframework.http.codec.json.Jackson2JsonEncoder;
import org.springframework.util.MimeTypeUtils;
import org.springframework.web.reactive.function.client.ExchangeStrategies;
/**
* WebFluxExchangeStragiesFactory
*
* @author Shin Woo-jin (woozu.shin@kakaoent.com)
* @since 2021-09-08
*/
public final class WebFluxExchangeStragiesFactory {
private WebFluxExchangeStragiesFactory() {}
public static ExchangeStrategies ofDefault() {
final ObjectMapper mapper = ObjectMapperFactory.defaultMapper();
return ExchangeStrategies.builder()
.codecs(configurer -> {
configurer.defaultCodecs().maxInMemorySize(-1);
configurer.defaultCodecs()
.jackson2JsonEncoder(new Jackson2JsonEncoder(mapper,
MimeTypeUtils.APPLICATION_JSON));
configurer.defaultCodecs()
.jackson2JsonDecoder(new Jackson2JsonDecoder(mapper,
MimeTypeUtils.APPLICATION_JSON));
})
.build();
}
public static ExchangeStrategies ofTextHtml() {
final ObjectMapper mapper = ObjectMapperFactory.defaultMapper();
return ExchangeStrategies.builder()
.codecs(configurer -> {
configurer.defaultCodecs().maxInMemorySize(-1);
configurer.defaultCodecs()
.jackson2JsonEncoder(new Jackson2JsonEncoder(mapper,
MimeTypeUtils.TEXT_HTML));
configurer.defaultCodecs()
.jackson2JsonDecoder(new Jackson2JsonDecoder(mapper,
MimeTypeUtils.TEXT_HTML));
})
.build();
}
}