[PPN-211113] Implement spring config client

This commit is contained in:
woo-jin.shin 2021-11-17 09:39:49 +09:00
parent 0afa52aa53
commit 5b4b44f093
22 changed files with 332 additions and 36 deletions

View File

@ -32,6 +32,17 @@ allprojects {
}
}
ext {
set('springCloudVersion', "2020.0.4")
}
dependencyManagement {
imports {
mavenBom "org.springframework.cloud:spring-cloud-dependencies:${springCloudVersion}"
}
}
/* task initSourceFolders {
sourceSets*.java.srcDirs*.each {
if( !it.exists() ) {

View File

@ -1,3 +1,4 @@
dependencies {
developmentOnly 'org.springframework.boot:spring-boot-devtools'
runtimeOnly 'com.h2database:h2'
@ -9,6 +10,8 @@ dependencies {
implementation 'org.springframework.boot:spring-boot-starter-webflux'
implementation 'org.springframework.boot:spring-boot-starter-data-jpa'
implementation 'org.springframework.boot:spring-boot-starter-jdbc'
implementation 'org.springframework.cloud:spring-cloud-starter-config'
implementation 'org.springframework.boot:spring-boot-starter-actuator'
implementation 'com.rometools:rome:1.16.0'
implementation 'org.jsoup:jsoup:1.14.2'
implementation 'com.h2database:h2:1.4.200'

View File

@ -1,9 +1,11 @@
package com.myoa.engineering.crawl.ppomppu.processor.infrastructure.client;
import com.myoa.engineering.crawl.ppomppu.processor.configuration.factory.WebClientFilterFactory;
import com.myoa.engineering.crawl.ppomppu.processor.configuration.factory.WebFluxExchangeStragiesFactory;
import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebClientFilterFactory;
import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebFluxExchangeStragiesFactory;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import lombok.extern.slf4j.Slf4j;
import org.springframework.core.ParameterizedTypeReference;
import org.springframework.stereotype.Component;
import org.springframework.web.reactive.function.client.WebClient;
import org.springframework.web.reactive.function.client.WebClientRequestException;

View File

@ -3,6 +3,12 @@ spring:
activate:
on-profile: development
import:
- "configserver:http://192.168.0.100:11080"
- classpath:/development/webclient.yml
- classpath:/development/temppassword.yml
- classpath:/development/database.yml
profiles:
group:
development: development, datasource-development
# import: optional:configserver:http://localhost:11080 # can be start up even config server was not found.

View File

@ -1,4 +1,6 @@
spring:
application:
name: ppn-processor
main:
allow-bean-definition-overriding: true
profiles:
@ -12,3 +14,9 @@ server:
error:
whitelabel:
enabled: false
management:
endpoints:
web:
exposure:
include: refresh

View File

@ -1,4 +1,6 @@
spring:
application:
name: ppn-receiver
main:
allow-bean-definition-overriding: true
profiles:

23
sender/build.gradle Normal file
View File

@ -0,0 +1,23 @@
dependencies {
developmentOnly 'org.springframework.boot:spring-boot-devtools'
runtimeOnly 'mysql:mysql-connector-java'
compileOnly 'org.projectlombok:lombok'
implementation project(':support')
// https://projectreactor.io/docs/core/release/reference/#debug-activate
implementation 'org.springframework.boot:spring-boot-starter-webflux'
implementation 'org.springframework.boot:spring-boot-starter-actuator'
implementation 'org.springframework.cloud:spring-cloud-starter-config'
annotationProcessor 'org.springframework.boot:spring-boot-configuration-processor'
annotationProcessor 'org.projectlombok:lombok'
testImplementation 'org.springframework.boot:spring-boot-starter-test'
}
test {
useJUnitPlatform()
testLogging {
events "passed", "skipped", "failed"
}
}

View File

@ -1,10 +1,18 @@
package com.myoa.engineering.crawl.ppomppu.sender;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
/**
* SenderApplication
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-08-20
*
*/
@SpringBootApplication
public class SenderApplication {
public static void main(String[] args) {
SpringApplication.run(SenderApplication.class, args);
}
}

View File

@ -0,0 +1,37 @@
package com.myoa.engineering.crawl.ppomppu.sender.controller;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import com.myoa.engineering.crawl.ppomppu.sender.dto.SlackMessageDTO;
import com.myoa.engineering.crawl.ppomppu.sender.infrastructure.client.SlackMessageSender;
import reactor.core.publisher.Mono;
/**
* TestAPIController
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-11-15
*
*/
@RestController
@RequestMapping("/api/v1")
public class TestAPIController {
private final SlackMessageSender sender;
public TestAPIController() {
this.sender = new SlackMessageSender("xoxb-2688454277126-2695026012277-K2Ib13lKokmTiBSnSMrc0Bp2");
}
@GetMapping("/test")
public Mono<String> test() {
return sender.sendMessage(SlackMessageDTO.builder()
.text("test!")
.iconEmoji("monge")
.channel("notify_shopping")
.username("shopping notifier")
.build());
}
}

View File

@ -0,0 +1,12 @@
package com.myoa.engineering.crawl.ppomppu.sender.dto;
import java.io.Serializable;
/**
* MessageDTO
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-11-14
*
*/
public interface MessageDTO extends Serializable {
}

View File

@ -0,0 +1,36 @@
package com.myoa.engineering.crawl.ppomppu.sender.dto;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.Builder;
import lombok.Getter;
import lombok.NoArgsConstructor;
/**
* SlackMessageDTO
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-11-14
*
*/
@Getter
@NoArgsConstructor
public class SlackMessageDTO implements MessageDTO {
private final static long serialVersionUID = 4737608709660494713L;
private String text;
private String channel;
private String username;
@JsonProperty("icon_emoji")
private String iconEmoji;
@Builder
public SlackMessageDTO(String text, String channel, String username, String iconEmoji) {
this.text = text;
this.channel = channel;
this.username = username;
this.iconEmoji = iconEmoji;
}
}

View File

@ -0,0 +1,17 @@
package com.myoa.engineering.crawl.ppomppu.sender.infrastructure.client;
import com.myoa.engineering.crawl.ppomppu.sender.dto.MessageDTO;
import reactor.core.publisher.Mono;
/**
* MessageSender
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-11-14
*
*/
public interface MessageSender<T extends MessageDTO> {
Mono<String> sendMessage(T message);
}

View File

@ -0,0 +1,88 @@
package com.myoa.engineering.crawl.ppomppu.sender.infrastructure.client;
import org.springframework.http.HttpHeaders;
import org.springframework.http.MediaType;
import org.springframework.web.reactive.function.client.WebClient;
import org.springframework.web.reactive.function.client.WebClientRequestException;
import com.myoa.engineering.crawl.ppomppu.sender.dto.SlackMessageDTO;
import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebClientFilterFactory;
import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebFluxExchangeStragiesFactory;
import lombok.extern.slf4j.Slf4j;
import reactor.core.publisher.Mono;
import reactor.core.scheduler.Schedulers;
/**
* SlackMessageSender
*
* @author Shin Woo-jin (woozu.shin@kakaoent.com)
* @since 2021-09-08
*/
@Slf4j
public class SlackMessageSender implements MessageSender<SlackMessageDTO> {
private static final String SLACK_API_URL = "https://slack.com/api";
private final WebClient webClient;
private final String apiSecret;
public SlackMessageSender(String apiSecret) {
this.webClient = WebClient.builder()
.baseUrl(SLACK_API_URL)
.defaultHeader(HttpHeaders.AUTHORIZATION, "Bearer " + apiSecret)
.exchangeStrategies(WebFluxExchangeStragiesFactory.ofDefault())
.defaultHeader(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_UTF8_VALUE)
.defaultHeader(HttpHeaders.ACCEPT_CHARSET, "UTF-8")
.filter(WebClientFilterFactory.logRequest())
.filter(WebClientFilterFactory.logResponse())
.build();
this.apiSecret = apiSecret;
}
@Override
public Mono<String> sendMessage(SlackMessageDTO message) {
return webClient.post()
.uri("/chat.postMessage")
.bodyValue(message)
.exchangeToMono(e -> e.bodyToMono(String.class))
.publishOn(Schedulers.boundedElastic())
.onErrorResume(WebClientRequestException.class, t -> {
log.info("Exception occured, ignoring. : {}", t.getClass().getSimpleName());
return Mono.empty();
})
.doOnNext(e -> log.info("response: {}", e));
}
}
/*
import requests
import json
# woozuyeni_watchtower
SLACK_TOKEN="xoxb-2688454277126-2695026012277-K2Ib13lKokmTiBSnSMrc0Bp2"
CHANNEL = "notify_watchtower"
CHANNEL = "common"
POST_API = "https://slack.com/api/chat.postMessage"
HEADER = "Authorization: Bearer"
headers = {
"Content-Type" : "application/json",
"Authorization" : f"Bearer {SLACK_TOKEN}"
}
data = {}
data["text"] = "몽몽! :monge_big:"
data["channel"] = CHANNEL
data["username"] = "몽이봇"
data["icon_emoji"] = ":monge_big:"
# requests.post(POST_API, data=json.dumps(data), headers=headers)
attachments = [{"title": "Cat", "image_url": image_url}]
image_url = "https://myoa-universe.com/assets/love1.png"
*/

View File

@ -0,0 +1,11 @@
spring:
config:
activate:
on-profile: development
import:
- classpath:/development/webclient.yml
- "configserver:http://192.168.0.100:11080"
# import: optional:configserver:http://localhost:11080 # can be start up even config server was not found.
profiles:
group:
development: development, slackapi-development

View File

@ -0,0 +1,6 @@
spring:
config:
activate:
on-profile: production
# import:
# - classpath:/production/webclient.yml

View File

@ -0,0 +1,19 @@
spring:
application:
# name: ppn-sender
name: app1
main:
allow-bean-definition-overriding: true
profiles:
active: development
freemarker:
enabled: false
server:
port: 20080
management:
endpoints:
web:
exposure:
include: refresh

View File

@ -0,0 +1,5 @@
webclient:
base-url: http://localhost:20081
units:
- unit-name: processor-api
base-url: http://localhost:20081

View File

@ -3,6 +3,7 @@ dependencies {
compileOnly 'org.projectlombok:lombok'
// https://projectreactor.io/docs/core/release/reference/#debug-activate
implementation 'org.springframework.boot:spring-boot-starter-webflux'
annotationProcessor 'org.springframework.boot:spring-boot-configuration-processor'
annotationProcessor 'org.projectlombok:lombok'

View File

@ -12,40 +12,40 @@ import lombok.Getter;
@Getter
@AllArgsConstructor
public enum PpomppuBoardName {
PPOMPPU_DOMESTIC_ALL("zboard/zboard.php?id=ppomppu", "전체", true),
PPOMPPU_DOMESTIC_ETC("zboard/zboard.php?id=ppomppu&category=1", "기타", true),
PPOMPPU_DOMESTIC_COMPUTER("zboard/zboard.php?id=ppomppu&category=4", "컴퓨터", true),
PPOMPPU_DOMESTIC_DIGITAL("zboard/zboard.php?id=ppomppu&category=5", "디지털", true),
PPOMPPU_DOMESTIC_FOOD("zboard/zboard.php?id=ppomppu&category=6", "식품/건강", true),
PPOMPPU_DOMESTIC_BOOK("zboard/zboard.php?id=ppomppu&category=8", "서적", true),
PPOMPPU_DOMESTIC_APPLIANCES("zboard/zboard.php?id=ppomppu&category=9", "가전/가구", true),
PPOMPPU_DOMESTIC_PARENTING("zboard/zboard.php?id=ppomppu&category=10", "육아", true),
PPOMPPU_DOMESTIC_GIFTCARD("zboard/zboard.php?id=ppomppu&category=11", "상품권", true),
PPOMPPU_DOMESTIC_CLOTHES("zboard/zboard.php?id=ppomppu&category=12", "의류/잡화", true),
PPOMPPU_DOMESTIC_COSMETIC("zboard/zboard.php?id=ppomppu&category=13", "화장품", true),
PPOMPPU_DOMESTIC_OUTDOOR("zboard/zboard.php?id=ppomppu&category=15", "등산/캠핑", true),
PPOMPPU_OVERSEA_ALL("zboard/zboard.php?id=ppomppu4", "전체", true),
PPOMPPU_OVERSEA_ETC("zboard/zboard.php?id=ppomppu4&category=1", "기타", true),
PPOMPPU_OVERSEA_APPLIANCES("zboard/zboard.php?id=ppomppu4&category=7", "가전", true),
PPOMPPU_OVERSEA_TVAV("zboard/zboard.php?id=ppomppu4&category=8", "TV/영상", true),
PPOMPPU_OVERSEA_COMPUTER("zboard/zboard.php?id=ppomppu4&category=3", "컴퓨터", true),
PPOMPPU_OVERSEA_DIGITAL("zboard/zboard.php?id=ppomppu4&category=4", "디지털", true),
PPOMPPU_OVERSEA_MOBILEACCESSORY("zboard/zboard.php?id=ppomppu4&category=9", "액세서리", true),
PPOMPPU_OVERSEA_CLOTHES("zboard/zboard.php?id=ppomppu4&category=5", "의류/잡화", true),
PPOMPPU_OVERSEA_WATCH("zboard/zboard.php?id=ppomppu4&category=2", "시계", true),
PPOMPPU_OVERSEA_SHOES("zboard/zboard.php?id=ppomppu4&category=11", "신발", true),
PPOMPPU_OVERSEA_FOOD("zboard/zboard.php?id=ppomppu4&category=10", "식품/건강", true),
PPOMPPU_OVERSEA_PARENTING("zboard/zboard.php?id=ppomppu4&category=6", "육아", true),
PPOMPPU_DOMESTIC_ALL("/zboard/zboard.php?id=ppomppu", "전체", true),
PPOMPPU_DOMESTIC_ETC("/zboard/zboard.php?id=ppomppu&category=1", "기타", true),
PPOMPPU_DOMESTIC_COMPUTER("/zboard/zboard.php?id=ppomppu&category=4", "컴퓨터", true),
PPOMPPU_DOMESTIC_DIGITAL("/zboard/zboard.php?id=ppomppu&category=5", "디지털", true),
PPOMPPU_DOMESTIC_FOOD("/zboard/zboard.php?id=ppomppu&category=6", "식품/건강", true),
PPOMPPU_DOMESTIC_BOOK("/zboard/zboard.php?id=ppomppu&category=8", "서적", true),
PPOMPPU_DOMESTIC_APPLIANCES("/zboard/zboard.php?id=ppomppu&category=9", "가전/가구", true),
PPOMPPU_DOMESTIC_PARENTING("/zboard/zboard.php?id=ppomppu&category=10", "육아", true),
PPOMPPU_DOMESTIC_GIFTCARD("/zboard/zboard.php?id=ppomppu&category=11", "상품권", true),
PPOMPPU_DOMESTIC_CLOTHES("/zboard/zboard.php?id=ppomppu&category=12", "의류/잡화", true),
PPOMPPU_DOMESTIC_COSMETIC("/zboard/zboard.php?id=ppomppu&category=13", "화장품", true),
PPOMPPU_DOMESTIC_OUTDOOR("/zboard/zboard.php?id=ppomppu&category=15", "등산/캠핑", true),
PPOMPPU_OVERSEA_ALL("/zboard/zboard.php?id=ppomppu4", "전체", true),
PPOMPPU_OVERSEA_ETC("/zboard/zboard.php?id=ppomppu4&category=1", "기타", true),
PPOMPPU_OVERSEA_APPLIANCES("/zboard/zboard.php?id=ppomppu4&category=7", "가전", true),
PPOMPPU_OVERSEA_TVAV("/zboard/zboard.php?id=ppomppu4&category=8", "TV/영상", true),
PPOMPPU_OVERSEA_COMPUTER("/zboard/zboard.php?id=ppomppu4&category=3", "컴퓨터", true),
PPOMPPU_OVERSEA_DIGITAL("/zboard/zboard.php?id=ppomppu4&category=4", "디지털", true),
PPOMPPU_OVERSEA_MOBILEACCESSORY("/zboard/zboard.php?id=ppomppu4&category=9", "액세서리", true),
PPOMPPU_OVERSEA_CLOTHES("/zboard/zboard.php?id=ppomppu4&category=5", "의류/잡화", true),
PPOMPPU_OVERSEA_WATCH("/zboard/zboard.php?id=ppomppu4&category=2", "시계", true),
PPOMPPU_OVERSEA_SHOES("/zboard/zboard.php?id=ppomppu4&category=11", "신발", true),
PPOMPPU_OVERSEA_FOOD("/zboard/zboard.php?id=ppomppu4&category=10", "식품/건강", true),
PPOMPPU_OVERSEA_PARENTING("/zboard/zboard.php?id=ppomppu4&category=6", "육아", true),
;
private String resourcePath;
private String menuName;
private boolean crawlWithDefaultTimer;
public static final String PPOMPPU_URL = "https://www.ppomppu.co.kr/";
public static final String PPOMPPU_URL = "https://www.ppomppu.co.kr";
public static String ofViewPageUrl(String articleUrl) {
return PPOMPPU_URL + "zboard/" + articleUrl;
return PPOMPPU_URL + "/zboard" + articleUrl;
}
}

View File

@ -1,4 +1,4 @@
package com.myoa.engineering.crawl.ppomppu.processor.util;
package com.myoa.engineering.crawl.ppomppu.support.util;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.core.JsonParser.Feature;

View File

@ -1,4 +1,4 @@
package com.myoa.engineering.crawl.ppomppu.processor.configuration.factory;
package com.myoa.engineering.crawl.ppomppu.support.webclient.factory;
import lombok.extern.slf4j.Slf4j;
import org.springframework.web.reactive.function.client.ClientRequest;
@ -27,8 +27,8 @@ public final class WebClientFilterFactory {
private static Mono<ClientRequest> writeRequest(ClientRequest clientRequest) {
try {
log.info("[WEBCLIENT REQUEST] uri : {} method : {} headers : {}",
clientRequest.url(), clientRequest.method(), clientRequest.headers());
log.info("[WEBCLIENT REQUEST] uri : {} method : {} headers : {}, body: {}",
clientRequest.url(), clientRequest.method(), clientRequest.headers(), clientRequest.body());
} catch (Exception e) {
log.error("[WEBCLIENT REQUEST] write request failed", e);
}
@ -37,7 +37,7 @@ public final class WebClientFilterFactory {
private static Mono<ClientResponse> writeResponse(ClientResponse clientResponse) {
try {
log.info("[WEBCLIENT REQUEST] statusCode : {} headers : {}",
log.info("[WEBCLIENT RESPONSE] statusCode : {} headers : {}",
clientResponse.rawStatusCode(), clientResponse.headers().asHttpHeaders());
} catch (Exception e) {
log.error("[WEBCLIENT RESPONSE] write response failed", e);

View File

@ -1,7 +1,8 @@
package com.myoa.engineering.crawl.ppomppu.processor.configuration.factory;
package com.myoa.engineering.crawl.ppomppu.support.webclient.factory;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.myoa.engineering.crawl.ppomppu.processor.util.ObjectMapperFactory;
import com.myoa.engineering.crawl.ppomppu.support.util.ObjectMapperFactory;
import org.springframework.http.codec.json.Jackson2JsonDecoder;
import org.springframework.http.codec.json.Jackson2JsonEncoder;
import org.springframework.util.MimeTypeUtils;