Compare commits
No commits in common. "develop" and "master" have entirely different histories.
|
@ -3,6 +3,7 @@ plugins {
|
|||
id 'idea'
|
||||
id 'org.springframework.boot' version '2.5.4'
|
||||
id 'io.spring.dependency-management' version '1.0.11.RELEASE'
|
||||
id 'com.google.cloud.tools.jib' version '3.4.0'
|
||||
}
|
||||
|
||||
group = 'com.myoa.engineering.crawl.shopping'
|
||||
|
@ -19,14 +20,16 @@ repositories {
|
|||
mavenCentral()
|
||||
}
|
||||
|
||||
allprojects {
|
||||
subprojects {
|
||||
group = 'com.myoa.engineering.crawl.shopping'
|
||||
version = '1.1.1'
|
||||
sourceCompatibility = JavaVersion.VERSION_17
|
||||
|
||||
apply plugin: 'java'
|
||||
apply plugin: 'idea'
|
||||
apply plugin: 'org.springframework.boot'
|
||||
apply plugin: 'io.spring.dependency-management'
|
||||
apply plugin: "com.google.cloud.tools.jib"
|
||||
|
||||
repositories {
|
||||
mavenCentral()
|
||||
|
@ -37,8 +40,11 @@ allprojects {
|
|||
|
||||
ext {
|
||||
set('springCloudVersion', "2020.0.4")
|
||||
set("BASE_IMAGE_REGISTRY_URL", "registry.myoa-universe.com")
|
||||
}
|
||||
|
||||
apply from: "${project.rootDir}/jib.gradle"
|
||||
|
||||
dependencyManagement {
|
||||
imports {
|
||||
mavenBom "org.springframework.cloud:spring-cloud-dependencies:${springCloudVersion}"
|
||||
|
|
|
@ -4,3 +4,7 @@ distributionPath=wrapper/dists
|
|||
distributionUrl=https\://services.gradle.org/distributions/gradle-8.5-bin.zip
|
||||
zipStoreBase=GRADLE_USER_HOME
|
||||
zipStorePath=wrapper/dists
|
||||
org.gradle.caching=true
|
||||
org.gradle.parallel=true
|
||||
org.gradle.parallel.threads=4
|
||||
org.gradle.daemon=true
|
|
@ -0,0 +1,28 @@
|
|||
jib.setAllowInsecureRegistries(true)
|
||||
|
||||
jib {
|
||||
from {
|
||||
image = "eclipse-temurin:17-jdk"
|
||||
platforms {
|
||||
platform {
|
||||
architecture = "amd64"
|
||||
os = "linux"
|
||||
}
|
||||
}
|
||||
}
|
||||
to {
|
||||
image = "${BASE_IMAGE_REGISTRY_URL}/${project.name}"
|
||||
tags = ["latest"]
|
||||
}
|
||||
container {
|
||||
// entrypoint = ["INHERIT"]
|
||||
creationTime = "USE_CURRENT_TIMESTAMP"
|
||||
environment = [
|
||||
"TZ": "Asia/Seoul",
|
||||
]
|
||||
jvmFlags = [
|
||||
"--add-opens=java.base/java.time=ALL-UNNAMED"
|
||||
]
|
||||
}
|
||||
|
||||
}
|
|
@ -1,70 +0,0 @@
|
|||
package com.myoa.engineering.crawl.shopping.controller;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import com.myoa.engineering.crawl.shopping.processor.domain.PpomppuArticle;
|
||||
import com.myoa.engineering.crawl.shopping.processor.dto.FeedParsedResult;
|
||||
import com.myoa.engineering.crawl.shopping.processor.service.MessageSenderService;
|
||||
import com.myoa.engineering.crawl.shopping.processor.service.PpomppuArticleService;
|
||||
import com.myoa.engineering.crawl.shopping.processor.service.PpomppuFeedService;
|
||||
import com.myoa.engineering.crawl.shopping.support.dto.APIResponse;
|
||||
import com.myoa.engineering.crawl.shopping.support.dto.code.PpomppuBoardName;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import reactor.core.publisher.Mono;
|
||||
|
||||
/**
|
||||
* CrawlAPIController
|
||||
*
|
||||
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
|
||||
* @since 2021-09-05
|
||||
*/
|
||||
@Slf4j
|
||||
@RestController
|
||||
@RequestMapping("/api/v1/crawl")
|
||||
public class CrawlAPIController {
|
||||
|
||||
private final PpomppuFeedService ppomppuRSSFeedService;
|
||||
private final PpomppuArticleService ppomppuArticleService;
|
||||
private final MessageSenderService messageSenderService;
|
||||
|
||||
public CrawlAPIController(PpomppuFeedService ppomppuRSSFeedService,
|
||||
PpomppuArticleService ppomppuArticleService,
|
||||
MessageSenderService messageSenderService) {
|
||||
this.ppomppuRSSFeedService = ppomppuRSSFeedService;
|
||||
this.ppomppuArticleService = ppomppuArticleService;
|
||||
this.messageSenderService = messageSenderService;
|
||||
}
|
||||
|
||||
@PostMapping("/boards/{boardName}")
|
||||
public Mono<APIResponse<FeedParsedResult>> crawlBoard(@PathVariable("boardName") PpomppuBoardName boardName) {
|
||||
log.info("got request... {}", boardName);
|
||||
FeedParsedResult result = FeedParsedResult.of(boardName);
|
||||
|
||||
Mono<String> publishedMessages =
|
||||
ppomppuRSSFeedService.getArticles(boardName)
|
||||
.map(e -> ppomppuArticleService.filterOnlyNewArticles(boardName, e))
|
||||
.map(e -> ppomppuArticleService.save(boardName, e))
|
||||
.filter(e -> !e.isEmpty())
|
||||
.flatMap(e -> messageSenderService.sendBlockMessageToSlack(boardName, e));
|
||||
|
||||
return publishedMessages.then(Mono.just(APIResponse.success(result.done())));
|
||||
}
|
||||
|
||||
@PostMapping("/exploit/boards/{boardName}")
|
||||
public Mono<APIResponse<String>> crawlBoardDryRun(
|
||||
@PathVariable("boardName") PpomppuBoardName boardName) {
|
||||
log.info("got request... {}", boardName);
|
||||
Mono<String> publishedMessages =
|
||||
ppomppuRSSFeedService.getArticles(boardName)
|
||||
.flatMap(e -> messageSenderService.sendBlockMessageToSlack(boardName, e));
|
||||
|
||||
return publishedMessages.map(APIResponse::success);
|
||||
}
|
||||
|
||||
}
|
|
@ -1,78 +0,0 @@
|
|||
package com.myoa.engineering.crawl.shopping.dto;
|
||||
|
||||
import org.jsoup.nodes.Element;
|
||||
import org.jsoup.select.Elements;
|
||||
|
||||
import com.myoa.engineering.crawl.shopping.processor.domain.PpomppuArticle;
|
||||
import com.myoa.engineering.crawl.shopping.support.dto.code.PpomppuBoardName;
|
||||
|
||||
/**
|
||||
* PpomppuArticleTransformer
|
||||
*
|
||||
* @author Shin Woo-jin (woozu.shin@kakaoent.com)
|
||||
* @since 2021-09-08
|
||||
*/
|
||||
public final class PpomppuArticleParser {
|
||||
|
||||
private PpomppuArticleParser() {}
|
||||
|
||||
public static PpomppuArticle toArticle(Elements articleElement) {
|
||||
final String articleIdString = PpomppuArticleParser.parseArticleId(articleElement.get(0));
|
||||
final String title = PpomppuArticleParser.parseTitle(articleElement.get(2));
|
||||
final String articleUrl = PpomppuArticleParser.parseArticleUrl(articleElement.get(2));
|
||||
final String thumbnailUrl = PpomppuArticleParser.parseThumbnailUrl(articleElement.get(3));
|
||||
final Integer recommended = PpomppuArticleParser.parseRecommended(articleElement.get(6));
|
||||
final String hitString = PpomppuArticleParser.parseHit(articleElement.get(7));
|
||||
final String registeredAtString = PpomppuArticleParser.parseRegisteredAt(articleElement.get(5));
|
||||
|
||||
return PpomppuArticleParseDTO.builder()
|
||||
.articleId(articleIdString)
|
||||
.title(title)
|
||||
.articleUrl(articleUrl)
|
||||
.thumbnailUrl(thumbnailUrl)
|
||||
.recommended(recommended)
|
||||
.hit(hitString)
|
||||
.registeredAt(registeredAtString)
|
||||
.build()
|
||||
.convert();
|
||||
}
|
||||
|
||||
public static String parseArticleId(Element td) {
|
||||
return td.text().trim();
|
||||
}
|
||||
|
||||
public static String parseTitle(Element td) {
|
||||
return td.getElementsByTag("a").text();
|
||||
}
|
||||
|
||||
public static String parseArticleUrl(Element td) {
|
||||
return PpomppuBoardName.ofViewPageUrl(td.getElementsByTag("a").attr("href"));
|
||||
}
|
||||
|
||||
public static String parseThumbnailUrl(Element td) {
|
||||
return "https:" + td.getElementsByTag("img").get(0).attr("src");
|
||||
}
|
||||
|
||||
public static Integer parseRecommended(Element td) {
|
||||
final String voteString = td.text();
|
||||
final int recommended;
|
||||
|
||||
if (voteString.isEmpty()) {
|
||||
recommended = 0;
|
||||
} else {
|
||||
final int voteUp = Integer.parseInt(td.text().split(" - ")[0]);
|
||||
final int voteDown = Integer.parseInt(td.text().split(" - ")[1]);
|
||||
recommended = voteUp - voteDown;
|
||||
}
|
||||
return recommended;
|
||||
}
|
||||
|
||||
public static String parseHit(Element td) {
|
||||
return td.text();
|
||||
}
|
||||
|
||||
public static String parseRegisteredAt(Element td) {
|
||||
return td.attr("title");
|
||||
}
|
||||
|
||||
}
|
|
@ -1,38 +0,0 @@
|
|||
package com.myoa.engineering.crawl.shopping.receiver.scheduler;
|
||||
|
||||
import java.util.Arrays;
|
||||
|
||||
import org.springframework.scheduling.annotation.EnableScheduling;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import com.myoa.engineering.crawl.shopping.receiver.service.ProcessorAPIService;
|
||||
import com.myoa.engineering.crawl.shopping.support.dto.code.PpomppuBoardName;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
/**
|
||||
* ParseEventEmitter
|
||||
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
|
||||
* @since 2021-09-05
|
||||
*
|
||||
*/
|
||||
@Slf4j
|
||||
@Component
|
||||
@EnableScheduling
|
||||
public class ParseEventEmitter {
|
||||
|
||||
private final ProcessorAPIService processorAPIService;
|
||||
|
||||
public ParseEventEmitter(ProcessorAPIService processorAPIService) {
|
||||
this.processorAPIService = processorAPIService;
|
||||
}
|
||||
|
||||
@Scheduled(fixedRate = 600 * 1000L)
|
||||
public void emitBoards() {
|
||||
log.info("[emitDomesticBoard] trigger fired!");
|
||||
Arrays.stream(PpomppuBoardName.values())
|
||||
.filter(PpomppuBoardName::isCrawlWithDefaultTimer)
|
||||
.forEach(boardName -> processorAPIService.emitParseEvent(boardName).block());
|
||||
}
|
||||
}
|
|
@ -1,6 +0,0 @@
|
|||
spring:
|
||||
config:
|
||||
activate:
|
||||
on-profile: production
|
||||
import:
|
||||
- "configserver:http://ppn-config-server:20080"
|
|
@ -1,25 +0,0 @@
|
|||
spring:
|
||||
application:
|
||||
name: ppn-receiver
|
||||
main:
|
||||
allow-bean-definition-overriding: true
|
||||
profiles:
|
||||
active: ${SPRING_ACTIVE_PROFILE:local}
|
||||
group:
|
||||
local: "local,webclient-local"
|
||||
development: "development,webclient-development"
|
||||
production: "production,webclient-production"
|
||||
freemarker:
|
||||
enabled: false
|
||||
|
||||
server:
|
||||
port: 20080
|
||||
error:
|
||||
whitelabel:
|
||||
enabled: false
|
||||
|
||||
management:
|
||||
endpoints:
|
||||
web:
|
||||
exposure:
|
||||
include: refresh,health
|
|
@ -1,62 +0,0 @@
|
|||
package com.myoa.engineering.crawl.shopping.controller;
|
||||
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import com.myoa.engineering.crawl.shopping.sender.dto.SlackBaseMessageBlock;
|
||||
import com.myoa.engineering.crawl.shopping.sender.dto.SlackMessageDTO;
|
||||
import com.myoa.engineering.crawl.shopping.sender.infrastructure.client.MongeShoppingBotSlackMessageSender;
|
||||
import com.myoa.engineering.crawl.shopping.support.dto.APIResponse;
|
||||
import com.myoa.engineering.crawl.shopping.support.dto.BlockMessageDTO;
|
||||
import com.myoa.engineering.crawl.shopping.support.dto.SimpleMessageDTO;
|
||||
import com.myoa.engineering.crawl.shopping.support.util.ObjectMapperFactory;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import reactor.core.publisher.Mono;
|
||||
|
||||
/**
|
||||
* MessageSenderAPIController
|
||||
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
|
||||
* @since 2021-11-21
|
||||
*
|
||||
*/
|
||||
@Slf4j
|
||||
@RestController
|
||||
@RequestMapping("/api/v1")
|
||||
public class MessageSenderAPIController {
|
||||
|
||||
private final MongeShoppingBotSlackMessageSender sender;
|
||||
|
||||
public MessageSenderAPIController(MongeShoppingBotSlackMessageSender sender) {
|
||||
this.sender = sender;
|
||||
}
|
||||
|
||||
@PostMapping("/messages/sendSimpleMessage/messengers/slack")
|
||||
public Mono<APIResponse<SimpleMessageDTO>> sendSimpleMessageToSlack(@RequestBody SimpleMessageDTO dto) {
|
||||
return sender.sendMessage(sender.ofMessage(dto.getBody()))
|
||||
.then(Mono.just(APIResponse.success(dto)));
|
||||
}
|
||||
|
||||
@PostMapping("/messages/sendBlockMessage/messengers/slack")
|
||||
public Mono<APIResponse<BlockMessageDTO>> sendBlockMessageToSlack(@RequestBody BlockMessageDTO dto) {
|
||||
if (dto.getBlocks().isEmpty()) {
|
||||
return Mono.just(APIResponse.fail(dto, "empty blocks"));
|
||||
}
|
||||
|
||||
|
||||
return sender.sendMessage(buildSlackMessageDTO(dto))
|
||||
// .doOnNext(e -> log.info("[sendBlockMessageToSlack] slackMessageDTO: {}",
|
||||
// ObjectMapperFactory.writeAsString(buildSlackMessageDTO(dto))))
|
||||
.then(Mono.just(APIResponse.success(dto)));
|
||||
}
|
||||
|
||||
private SlackMessageDTO buildSlackMessageDTO(BlockMessageDTO dto) {
|
||||
SlackMessageDTO slackMessageDTO = sender.ofBlockMessageBased();
|
||||
slackMessageDTO.addSectionBlock(dto.getTitle());
|
||||
dto.getBlocks().forEach(slackMessageDTO::addSectionBlock);
|
||||
slackMessageDTO.addBlock(SlackBaseMessageBlock.ofDivider());
|
||||
return slackMessageDTO;
|
||||
}
|
||||
}
|
|
@ -1,25 +0,0 @@
|
|||
spring:
|
||||
application:
|
||||
name: ppn-sender
|
||||
main:
|
||||
allow-bean-definition-overriding: true
|
||||
profiles:
|
||||
active: ${SPRING_ACTIVE_PROFILE:local}
|
||||
group:
|
||||
local: "local,slackapi-local,webclient-local"
|
||||
development: "development,slackapi-development,webclient-development"
|
||||
production: "production,slackapi-production,webclient-production"
|
||||
freemarker:
|
||||
enabled: false
|
||||
|
||||
server:
|
||||
port: 20080
|
||||
error:
|
||||
whitelabel:
|
||||
enabled: false
|
||||
|
||||
management:
|
||||
endpoints:
|
||||
web:
|
||||
exposure:
|
||||
include: refresh,health
|
|
@ -9,6 +9,8 @@ dependencies {
|
|||
implementation("org.springframework.boot:spring-boot-starter-web") {
|
||||
exclude group: "org.springframework.boot", module: "spring-boot-starter-tomcat"
|
||||
}
|
||||
implementation("org.springframework.boot:spring-boot-starter-webflux")
|
||||
|
||||
|
||||
implementation("org.springframework.boot:spring-boot-starter-undertow") {
|
||||
exclude group: "io.undertow", module: "undertow-websockets-jsr"
|
||||
|
@ -22,6 +24,15 @@ dependencies {
|
|||
implementation "org.springframework.cloud:spring-cloud-starter-openfeign"
|
||||
implementation "io.github.openfeign:feign-hc5"
|
||||
implementation 'org.ahocorasick:ahocorasick:0.6.3'
|
||||
implementation "com.slack.api:slack-api-client:1.39.1"
|
||||
|
||||
|
||||
// implementation "io.github.resilience4j:resilience4j-spring-boot3:2.2.0"
|
||||
implementation 'io.github.resilience4j:resilience4j-all:2.2.0'
|
||||
implementation "io.github.resilience4j:resilience4j-feign:2.2.0"
|
||||
implementation "org.springframework.cloud:spring-cloud-starter-circuitbreaker-resilience4j"
|
||||
// implementation 'io.github.openfeign:feign-okhttp:13.1'
|
||||
implementation 'io.github.openfeign:feign-jackson:13.2'
|
||||
|
||||
annotationProcessor 'org.springframework.boot:spring-boot-configuration-processor'
|
||||
annotationProcessor 'org.projectlombok:lombok'
|
||||
|
|
|
@ -12,7 +12,7 @@ import org.springframework.context.event.EventListener;
|
|||
import java.sql.SQLException;
|
||||
|
||||
@Slf4j
|
||||
@Profile({"datasource-local", "datasource-development"})
|
||||
@Profile("!prod")
|
||||
@Configuration
|
||||
public class H2ConsoleConfiguration {
|
||||
|
||||
|
@ -24,7 +24,7 @@ public class H2ConsoleConfiguration {
|
|||
@EventListener(ContextRefreshedEvent.class)
|
||||
public void start() throws SQLException {
|
||||
log.info("starting h2 console");
|
||||
this.webServer = Server.createWebServer("-webPort", port, "-tcpAllowOthers").start();
|
||||
this.webServer = Server.createWebServer("-webPort", port, "-tcpAllowOthers", "-webAllowOthers").start();
|
||||
}
|
||||
|
||||
@EventListener(ContextClosedEvent.class)
|
||||
|
|
|
@ -0,0 +1,19 @@
|
|||
package com.myoa.engineering.crawl.shopping.configuration.feign;
|
||||
|
||||
import feign.RequestInterceptor;
|
||||
import feign.RequestTemplate;
|
||||
import org.apache.http.HttpHeaders;
|
||||
|
||||
public class FakeUserAgentInterceptor implements RequestInterceptor {
|
||||
|
||||
private static final String USER_AGENT_VALUE = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36";
|
||||
|
||||
@Override
|
||||
public void apply(RequestTemplate template) {
|
||||
template.header(HttpHeaders.USER_AGENT, USER_AGENT_VALUE);
|
||||
template.header(HttpHeaders.ACCEPT, "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8");
|
||||
template.header(HttpHeaders.ACCEPT_LANGUAGE, "ko-KR,ko;q=0.9,en-US;q=0.8,en;q=0.7");
|
||||
template.header(HttpHeaders.USER_AGENT, USER_AGENT_VALUE);
|
||||
template.header(HttpHeaders.USER_AGENT, USER_AGENT_VALUE);
|
||||
}
|
||||
}
|
|
@ -1,8 +1,7 @@
|
|||
package com.myoa.engineering.crawl.shopping.configuration;
|
||||
package com.myoa.engineering.crawl.shopping.configuration.feign;
|
||||
|
||||
import feign.Logger;
|
||||
import feign.RequestInterceptor;
|
||||
import feign.codec.ErrorDecoder;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
|
@ -0,0 +1,40 @@
|
|||
package com.myoa.engineering.crawl.shopping.configuration.feign;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.myoa.engineering.crawl.shopping.dto.ExceptionMessage;
|
||||
import feign.Response;
|
||||
import feign.codec.ErrorDecoder;
|
||||
import io.undertow.util.BadRequestException;
|
||||
import javassist.NotFoundException;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
|
||||
@Slf4j
|
||||
public class FmkoreaClientErrorDecoder implements ErrorDecoder {
|
||||
|
||||
private final ErrorDecoder errorDecoder = new Default();
|
||||
|
||||
@Override
|
||||
public Exception decode(String methodsKey, Response response) {
|
||||
ExceptionMessage message = null;
|
||||
try (InputStream bodyIs = response.body()
|
||||
.asInputStream()) {
|
||||
ObjectMapper mapper = new ObjectMapper();
|
||||
message = mapper.readValue(bodyIs, ExceptionMessage.class);
|
||||
} catch (IOException e) {
|
||||
return new Exception(e.getMessage());
|
||||
}
|
||||
switch (response.status()) {
|
||||
case 400:
|
||||
return new BadRequestException(message.getMessage() != null ? message.getMessage() : "Bad Request");
|
||||
case 404:
|
||||
return new NotFoundException(message.getMessage() != null ? message.getMessage() : "Not found");
|
||||
default:
|
||||
return errorDecoder.decode(methodsKey, response);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,52 @@
|
|||
package com.myoa.engineering.crawl.shopping.configuration.resilience;
|
||||
|
||||
import io.github.resilience4j.circuitbreaker.CircuitBreaker;
|
||||
import io.github.resilience4j.circuitbreaker.CircuitBreakerConfig;
|
||||
import io.github.resilience4j.circuitbreaker.CircuitBreakerRegistry;
|
||||
import io.github.resilience4j.core.RegistryStore;
|
||||
import io.github.resilience4j.core.registry.InMemoryRegistryStore;
|
||||
import io.github.resilience4j.ratelimiter.RateLimiter;
|
||||
import io.github.resilience4j.ratelimiter.RateLimiterConfig;
|
||||
import io.github.resilience4j.ratelimiter.RateLimiterRegistry;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
|
||||
import java.time.Duration;
|
||||
|
||||
@Configuration
|
||||
public class RateLimitConfiguration {
|
||||
|
||||
@Bean
|
||||
public RateLimiterRegistry rateLimiterRegistry() {
|
||||
RegistryStore<RateLimiter> stores = new InMemoryRegistryStore<>();
|
||||
|
||||
// TODO 개별 config 에서 등록하도록 변경
|
||||
RateLimiterConfig rateLimiterConfig = RateLimiterConfig.custom()
|
||||
.limitRefreshPeriod(Duration.ofMillis(500)) // 0.5 seconds
|
||||
.limitForPeriod(1) // number of permits in a refresh period
|
||||
.build();
|
||||
stores.putIfAbsent("fmkoreaAvoid429", RateLimiter.of("fmkoreaAvoid429", rateLimiterConfig));
|
||||
|
||||
return RateLimiterRegistry.custom()
|
||||
.withRateLimiterConfig(RateLimiterConfig.ofDefaults())
|
||||
.withRegistryStore(stores)
|
||||
.build();
|
||||
}
|
||||
|
||||
@Bean
|
||||
public CircuitBreakerRegistry circuitBreakerRegistry() {
|
||||
RegistryStore<CircuitBreaker> stores = new InMemoryRegistryStore<>();
|
||||
|
||||
CircuitBreakerConfig circuitBreakerConfig = CircuitBreakerConfig.custom()
|
||||
.slidingWindowSize(1)
|
||||
.build();
|
||||
stores.putIfAbsent("fmkoreaAvoid429", CircuitBreaker.of("fmkoreaAvoid429", circuitBreakerConfig));
|
||||
|
||||
return CircuitBreakerRegistry.custom()
|
||||
.withCircuitBreakerConfig(CircuitBreakerConfig.ofDefaults())
|
||||
.withRegistryStore(stores)
|
||||
.build();
|
||||
|
||||
}
|
||||
|
||||
}
|
|
@ -0,0 +1,20 @@
|
|||
package com.myoa.engineering.crawl.shopping.configuration.slack;
|
||||
|
||||
import com.myoa.engineering.crawl.shopping.configuration.slack.properties.SlackSecretProperties;
|
||||
import com.slack.api.Slack;
|
||||
import com.slack.api.methods.MethodsClient;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
|
||||
@Configuration
|
||||
public class SlackConfiguration {
|
||||
|
||||
private static final String DEFAULT_BOT_UNIT_NAME = "shopping-crawler";
|
||||
|
||||
@Bean
|
||||
public MethodsClient slackMethodsClient(SlackSecretProperties slackSecretProperties) {
|
||||
String token = slackSecretProperties.find(DEFAULT_BOT_UNIT_NAME).getToken();
|
||||
return Slack.getInstance().methods(token);
|
||||
}
|
||||
|
||||
}
|
|
@ -0,0 +1,28 @@
|
|||
package com.myoa.engineering.crawl.shopping.configuration.web;
|
||||
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.http.converter.HttpMessageConverter;
|
||||
import org.springframework.web.servlet.config.annotation.WebMvcConfigurer;
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.List;
|
||||
|
||||
@Configuration
|
||||
public class WebConfiguration implements WebMvcConfigurer {
|
||||
/*
|
||||
@Override
|
||||
public void configureMessageConverters(List<HttpMessageConverter<?>> converters) {
|
||||
WebMvcConfigurer.super.configureMessageConverters(converters);
|
||||
addJsonFormUrlEncodedConverter(converters);
|
||||
}
|
||||
|
||||
private void addJsonFormUrlEncodedConverter(List<HttpMessageConverter<?>> converters) {
|
||||
JsonFormUrlEncodedConverter<?> converter = new JsonFormUrlEncodedConverter<>();
|
||||
MediaType mediaType = new MediaType(MediaType.APPLICATION_FORM_URLENCODED, StandardCharsets.UTF_8);
|
||||
converter.setSupportedMediaTypes(List.of(mediaType));
|
||||
converters.add(converter);
|
||||
}
|
||||
*/
|
||||
|
||||
}
|
|
@ -0,0 +1,65 @@
|
|||
package com.myoa.engineering.crawl.shopping.controller;
|
||||
|
||||
import com.myoa.engineering.crawl.shopping.dto.slack.v2.SlackEventSubscriptionDTO;
|
||||
import com.myoa.engineering.crawl.shopping.dto.slack.v2.SlackSlashCommandDTO;
|
||||
import com.myoa.engineering.crawl.shopping.service.slack.RegisterCommandHandler;
|
||||
import com.myoa.engineering.crawl.shopping.service.slack.ShoppingCommandHandler;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Slf4j
|
||||
@RestController
|
||||
@RequestMapping("/api/v1/slack")
|
||||
public class SlackEventSubscriptionAPIController {
|
||||
|
||||
private final ShoppingCommandHandler shoppingCommandHandler;
|
||||
private final RegisterCommandHandler registerCommandHandler;
|
||||
|
||||
public SlackEventSubscriptionAPIController(ShoppingCommandHandler shoppingCommandHandler,
|
||||
RegisterCommandHandler registerCommandHandler) {
|
||||
this.shoppingCommandHandler = shoppingCommandHandler;
|
||||
this.registerCommandHandler = registerCommandHandler;
|
||||
}
|
||||
|
||||
@PostMapping(value = {"/subscriptions"})
|
||||
private ResponseEntity<String> slackEventSubscription(@RequestBody SlackEventSubscriptionDTO requestDTO) {
|
||||
log.info("requestDTO: {}", requestDTO);
|
||||
return ResponseEntity.ok(requestDTO.getChallenge());
|
||||
}
|
||||
|
||||
@PostMapping(value = {"/commands/help"}, consumes = {"application/x-www-form-urlencoded"})
|
||||
private ResponseEntity<String> handleCommandHelp(SlackSlashCommandDTO requestDTO) {
|
||||
log.info("requestDTO: {}", requestDTO);
|
||||
return ResponseEntity.ok("""
|
||||
/쇼핑 목록
|
||||
/쇼핑 키워드 추가 (쇼핑몰명) (키워드명)
|
||||
/쇼핑 키워드 삭제 (쇼핑몰명) (키워드명)
|
||||
/쇼핑 키워드 목록
|
||||
/등록 (사용자명)
|
||||
""");
|
||||
}
|
||||
|
||||
@PostMapping(value = {"/commands/shopping"}, consumes = {"application/x-www-form-urlencoded"})
|
||||
private ResponseEntity<String> handleAddSubscription(SlackSlashCommandDTO requestDTO) {
|
||||
log.info("requestDTO: {}", requestDTO);
|
||||
|
||||
List<String> results = shoppingCommandHandler.handle(requestDTO);
|
||||
|
||||
return ResponseEntity.ok(String.join("\n", results));
|
||||
}
|
||||
|
||||
@PostMapping(value = {"/commands/register"}, consumes = {"application/x-www-form-urlencoded"})
|
||||
private ResponseEntity<String> handleRegisterUser(SlackSlashCommandDTO requestDTO) {
|
||||
log.info("requestDTO: {}", requestDTO);
|
||||
|
||||
List<String> results = registerCommandHandler.handle(requestDTO);
|
||||
|
||||
return ResponseEntity.ok(String.join("\n", results));
|
||||
}
|
||||
}
|
|
@ -1,22 +1,64 @@
|
|||
package com.myoa.engineering.crawl.shopping.controller;
|
||||
|
||||
import com.myoa.engineering.crawl.shopping.crawlhandler.PpomppuCrawlDomesticHandler;
|
||||
import com.myoa.engineering.crawl.shopping.crawlhandler.CrawlHandler;
|
||||
import com.myoa.engineering.crawl.shopping.infra.client.fmkorea.FmkoreaBoardClientV2;
|
||||
import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget;
|
||||
import com.slack.api.methods.MethodsClient;
|
||||
import com.slack.api.methods.SlackApiException;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
@Slf4j
|
||||
@RestController
|
||||
@RequestMapping("/api/v1/exploit")
|
||||
public class TestAPIController {
|
||||
|
||||
private final PpomppuCrawlDomesticHandler ppomppuCrawlDomesticHandler;
|
||||
private final MethodsClient methodsClient;
|
||||
private final List<CrawlHandler> crawlHandlers;
|
||||
private final FmkoreaBoardClientV2 fmkoreaBoardClientV2;
|
||||
|
||||
public TestAPIController(PpomppuCrawlDomesticHandler ppomppuCrawlDomesticHandler) {
|
||||
this.ppomppuCrawlDomesticHandler = ppomppuCrawlDomesticHandler;
|
||||
public TestAPIController(MethodsClient methodsClient,
|
||||
List<CrawlHandler> crawlHandlers,
|
||||
FmkoreaBoardClientV2 fmkoreaBoardClientV2) {
|
||||
this.methodsClient = methodsClient;
|
||||
this.crawlHandlers = crawlHandlers;
|
||||
this.fmkoreaBoardClientV2 = fmkoreaBoardClientV2;
|
||||
}
|
||||
|
||||
@GetMapping("/triggers")
|
||||
public void triggerExploit(@RequestParam("crawlTarget") CrawlTarget crawlTarget) {
|
||||
crawlHandlers
|
||||
.stream().filter(e -> e.getCrawlTarget().equals(crawlTarget))
|
||||
.forEach(CrawlHandler::handle);
|
||||
}
|
||||
|
||||
@GetMapping("/ratelimiter")
|
||||
public void triggerExploit() {
|
||||
ppomppuCrawlDomesticHandler.handle();
|
||||
log.info("will be called page 1");
|
||||
fmkoreaBoardClientV2.getBoardHtml(1, null);
|
||||
log.info("called page 1");
|
||||
|
||||
}
|
||||
|
||||
private Map<String, String> generateRequestParams(int pageId) {
|
||||
Map<String, String> params = new HashMap<>();
|
||||
params.put("mid", "hotdeal");
|
||||
params.put("page", String.valueOf(pageId));
|
||||
return params;
|
||||
}
|
||||
|
||||
@GetMapping("/test-message")
|
||||
public void testMessage() throws SlackApiException, IOException {
|
||||
methodsClient.chatPostMessage(req -> req
|
||||
.channel("notify_shopping")
|
||||
.text("Hello, World!"));
|
||||
}
|
||||
}
|
||||
|
|
|
@ -0,0 +1,22 @@
|
|||
package com.myoa.engineering.crawl.shopping.crawlhandler;
|
||||
|
||||
import com.myoa.engineering.crawl.shopping.infra.client.fmkorea.FmkoreaBoardClientV2;
|
||||
import org.springframework.cache.annotation.Cacheable;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
|
||||
@Service
|
||||
public class FmkoreaCookieService {
|
||||
|
||||
private final FmkoreaBoardClientV2 fmkoreaBoardClientV2;
|
||||
|
||||
public FmkoreaCookieService(FmkoreaBoardClientV2 fmkoreaBoardClientV2) {
|
||||
this.fmkoreaBoardClientV2 = fmkoreaBoardClientV2;
|
||||
}
|
||||
|
||||
@Cacheable(cacheNames = "crawltarget.fmkorea", key = "#root.methodName")
|
||||
public String getCookie() {
|
||||
String fakeHtml = fmkoreaBoardClientV2.getBoardHtml(1, null);
|
||||
return FmkoreaFake430Resolver.resolveFake430(fakeHtml);
|
||||
}
|
||||
}
|
|
@ -1,12 +1,33 @@
|
|||
package com.myoa.engineering.crawl.shopping.crawlhandler;
|
||||
|
||||
import com.myoa.engineering.crawl.shopping.crawlhandler.parser.FmkoreaArticleParser;
|
||||
import com.myoa.engineering.crawl.shopping.domain.entity.v2.Article;
|
||||
import com.myoa.engineering.crawl.shopping.infra.client.fmkorea.FmkoreaBoardClientV2;
|
||||
import com.myoa.engineering.crawl.shopping.service.ArticleCommandService;
|
||||
import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
@Slf4j
|
||||
@Component
|
||||
public class FmkoreaCrawlHandler implements CrawlHandler {
|
||||
|
||||
private final FmkoreaBoardClientV2 fmkoreaBoardClientV2;
|
||||
private final FmkoreaArticleParser fmkoreaArticleParser;
|
||||
private final ArticleCommandService articleCommandService;
|
||||
private final FmkoreaCookieService fmkoreaCookieService;
|
||||
|
||||
public FmkoreaCrawlHandler(FmkoreaBoardClientV2 fmkoreaBoardClientV2,
|
||||
FmkoreaArticleParser fmkoreaArticleParser, ArticleCommandService articleCommandService, FmkoreaCookieService fmkoreaCookieService) {
|
||||
this.fmkoreaBoardClientV2 = fmkoreaBoardClientV2;
|
||||
this.fmkoreaArticleParser = fmkoreaArticleParser;
|
||||
this.articleCommandService = articleCommandService;
|
||||
this.fmkoreaCookieService = fmkoreaCookieService;
|
||||
}
|
||||
|
||||
@Override
|
||||
public CrawlTarget getCrawlTarget() {
|
||||
return CrawlTarget.FMKOREA;
|
||||
|
@ -14,5 +35,20 @@ public class FmkoreaCrawlHandler implements CrawlHandler {
|
|||
|
||||
@Override
|
||||
public void handle() {
|
||||
String cookie = fmkoreaCookieService.getCookie();
|
||||
|
||||
String boardHtmlPage1 = fmkoreaBoardClientV2.getBoardHtml(1, cookie);
|
||||
List<Article> parsedPage1 = fmkoreaArticleParser.parse(boardHtmlPage1);
|
||||
|
||||
String boardHtmlPage2 = fmkoreaBoardClientV2.getBoardHtml(2, cookie);
|
||||
List<Article> parsedPage2 = fmkoreaArticleParser.parse(boardHtmlPage2);
|
||||
|
||||
List<Article> merged = Stream.of(parsedPage1, parsedPage2)
|
||||
.flatMap(List::stream)
|
||||
.map(e -> e.updateCrawlTarget(getCrawlTarget()))
|
||||
.toList();
|
||||
|
||||
articleCommandService.upsert(merged);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -0,0 +1,88 @@
|
|||
package com.myoa.engineering.crawl.shopping.crawlhandler;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.databind.type.CollectionType;
|
||||
import com.myoa.engineering.crawl.shopping.util.ObjectMapperFactory;
|
||||
import org.jsoup.Jsoup;
|
||||
import org.jsoup.nodes.Document;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.util.Base64;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
public final class FmkoreaFake430Resolver {
|
||||
|
||||
private static final Pattern PATTERN_DOCUMENT = Pattern.compile("var .+?\\s*=\\s*(\\[.+?\\];)");
|
||||
private static final Pattern PATTERN_COOKIE = Pattern.compile("escape\\('(.+?)'\\)");
|
||||
private static final ObjectMapper MAPPER = ObjectMapperFactory.DEFAULT_MAPPER;
|
||||
private static final CollectionType COLLECTION_TYPE = MAPPER.getTypeFactory().constructCollectionType(List.class, String.class);
|
||||
|
||||
private static final DateTimeFormatter DATE_TIME_FORMATTER_COOKIE_LIFE = DateTimeFormatter.ofPattern("EEE, dd MMM yyyy HH:mm:ss 'GMT'", Locale.ENGLISH);
|
||||
|
||||
private FmkoreaFake430Resolver() {
|
||||
}
|
||||
|
||||
public static String resolveFake430(String fakeHtml) {
|
||||
Document parse = Jsoup.parse(fakeHtml);
|
||||
String javascript = parse.select("script").html();
|
||||
String cookieHtml = extractEncodedCookieHtml(javascript);
|
||||
|
||||
try {
|
||||
List<String> chunks = MAPPER.readValue(cookieHtml, COLLECTION_TYPE);
|
||||
String decodedhtml = decodeHtmlChunks(chunks);
|
||||
String cookieHexValue = extractCookieHexValue(decodedhtml);
|
||||
return generateLiteTimeCookie(cookieHexValue);
|
||||
} catch (Exception e) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private static String extractEncodedCookieHtml(String javascript) {
|
||||
final Matcher matcher = PATTERN_DOCUMENT.matcher(javascript);
|
||||
|
||||
if (matcher.find()) {
|
||||
return matcher.group(1);
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
private static String decodeHtmlChunks(List<String> chunks) {
|
||||
return chunks.stream()
|
||||
.map(e -> new String(Base64.getDecoder().decode(e.substring(3, e.length() - 3))))
|
||||
.map(e -> (char) (e.charAt(0) - 3 + 256) % 256)
|
||||
.collect(StringBuilder::new, StringBuilder::appendCodePoint, StringBuilder::append).toString();
|
||||
}
|
||||
|
||||
private static String extractCookieHexValue(String decodedHtml) {
|
||||
final Matcher matcher = PATTERN_COOKIE.matcher(decodedHtml);
|
||||
if (matcher.find()) {
|
||||
return matcher.group(1);
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
private static String escape(String input) {
|
||||
StringBuilder result = new StringBuilder();
|
||||
for (char ch : input.toCharArray()) {
|
||||
if (Character.isLetterOrDigit(ch) || ch == '-' || ch == '_' || ch == '.' || ch == '~') {
|
||||
result.append(ch);
|
||||
} else {
|
||||
result.append(String.format("%%%02X", (int) ch));
|
||||
}
|
||||
}
|
||||
return result.toString();
|
||||
}
|
||||
|
||||
public static String generateLiteTimeCookie(String cookieHexValue) {
|
||||
LocalDateTime ldt = LocalDateTime.now().plusDays(1L);
|
||||
|
||||
String cookie = "lite_year=" + escape(cookieHexValue) +
|
||||
"; expires=" + ldt.format(DATE_TIME_FORMATTER_COOKIE_LIFE) + "; path=/";
|
||||
return cookie;
|
||||
}
|
||||
|
||||
}
|
|
@ -0,0 +1,58 @@
|
|||
package com.myoa.engineering.crawl.shopping.crawlhandler;
|
||||
|
||||
import com.myoa.engineering.crawl.shopping.crawlhandler.parser.PpomppuArticleParserV2;
|
||||
import com.myoa.engineering.crawl.shopping.domain.entity.v2.Article;
|
||||
import com.myoa.engineering.crawl.shopping.infra.client.ppomppu.PpomppuBoardClientV2;
|
||||
import com.myoa.engineering.crawl.shopping.service.ArticleCommandService;
|
||||
import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
@Component
|
||||
public class PpomppuCrawlOverseaHandler implements CrawlHandler {
|
||||
|
||||
private final PpomppuBoardClientV2 ppomppuBoardClient;
|
||||
private final PpomppuArticleParserV2 ppomppuArticleParserV2;
|
||||
private final ArticleCommandService articleCommandService;
|
||||
|
||||
public PpomppuCrawlOverseaHandler(PpomppuBoardClientV2 ppomppuBoardClient,
|
||||
PpomppuArticleParserV2 ppomppuArticleParserV2,
|
||||
ArticleCommandService articleCommandService) {
|
||||
this.ppomppuBoardClient = ppomppuBoardClient;
|
||||
this.ppomppuArticleParserV2 = ppomppuArticleParserV2;
|
||||
this.articleCommandService = articleCommandService;
|
||||
}
|
||||
|
||||
@Override
|
||||
public CrawlTarget getCrawlTarget() {
|
||||
return CrawlTarget.PPOMPPU_OVERSEA;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void handle() {
|
||||
|
||||
String boardHtmlPage1 = ppomppuBoardClient.getBoardHtml("/zboard/zboard.php", generateRequestParams(1));
|
||||
List<Article> parsedPage1 = ppomppuArticleParserV2.parse(boardHtmlPage1);
|
||||
|
||||
String boardHtmlPage2 = ppomppuBoardClient.getBoardHtml("/zboard/zboard.php", generateRequestParams(2));
|
||||
List<Article> parsedPage2 = ppomppuArticleParserV2.parse(boardHtmlPage2);
|
||||
|
||||
List<Article> merged = Stream.of(parsedPage1, parsedPage2)
|
||||
.flatMap(List::stream)
|
||||
.map(e -> e.updateCrawlTarget(getCrawlTarget()))
|
||||
.toList();
|
||||
|
||||
articleCommandService.upsert(merged);
|
||||
}
|
||||
|
||||
private Map<String, String> generateRequestParams(int pageId) {
|
||||
Map<String, String> params = new HashMap<>();
|
||||
params.put("id", "ppomppu4");
|
||||
params.put("page", String.valueOf(pageId));
|
||||
return params;
|
||||
}
|
||||
}
|
|
@ -0,0 +1,68 @@
|
|||
package com.myoa.engineering.crawl.shopping.crawlhandler.parser;
|
||||
|
||||
import com.myoa.engineering.crawl.shopping.domain.entity.v2.Article;
|
||||
import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget;
|
||||
import com.myoa.engineering.crawl.shopping.util.DateTimeUtils;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.jsoup.Jsoup;
|
||||
import org.jsoup.nodes.Document;
|
||||
import org.jsoup.nodes.Element;
|
||||
import org.jsoup.select.Elements;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.time.ZonedDateTime;
|
||||
import java.util.List;
|
||||
|
||||
@Slf4j
|
||||
@Component
|
||||
public class FmkoreaArticleParser {
|
||||
|
||||
private static final String FMKOREA_URL = "https://www.fmkorea.com";
|
||||
|
||||
public List<Article> parse(String html) {
|
||||
Elements liElements = converHtmlToTrElements(html);
|
||||
return liElements.stream()
|
||||
// .filter(this::isRealArticle)
|
||||
.map(this::parse)
|
||||
.toList();
|
||||
}
|
||||
|
||||
private Elements converHtmlToTrElements(String html) {
|
||||
Document document = Jsoup.parse(html);
|
||||
Element liTable = document.getElementsByClass("fm_best_widget").first();
|
||||
return liTable.select("li.li_best2_pop0");
|
||||
}
|
||||
|
||||
private Article parse(Element item) {
|
||||
String link = item.select("h3.title a").attr("href");
|
||||
Long articleId = Long.parseLong(link.replace("/", ""));
|
||||
|
||||
// https://www.fmkorea.com/7023440365
|
||||
String articleUrl = FMKOREA_URL + link;
|
||||
String boardName = item.select("span.category a").text().trim();
|
||||
|
||||
String title = item.select("h3.title a").text();
|
||||
String itemPrice = item.select("div.hotdeal_info span:contains(가격) a").text();
|
||||
String deliveryPrice = item.select("div.hotdeal_info span:contains(배송) a").text();
|
||||
title = title + " 가격: " + itemPrice + " 배송: " + deliveryPrice;
|
||||
String registeredAtString = item.select("span.regdate").text().trim();
|
||||
ZonedDateTime registeredAt = DateTimeUtils.parse(registeredAtString, DateTimeUtils.FORMATTER_HHMM, DateTimeUtils.FORMATTER_YYMMDD_DOT);
|
||||
|
||||
Element recommendationElement = item.selectFirst("a.pc_voted_count");
|
||||
Integer recommended = null;
|
||||
if (recommendationElement != null) {
|
||||
recommended = Integer.parseInt(recommendationElement.selectFirst("span.count").text());
|
||||
}
|
||||
|
||||
return Article.builder()
|
||||
.articleId(articleId)
|
||||
.title(title)
|
||||
.boardName(boardName)
|
||||
.articleUrl(articleUrl)
|
||||
.recommended(recommended)
|
||||
.registeredAt(registeredAt)
|
||||
.build();
|
||||
}
|
||||
|
||||
|
||||
}
|
|
@ -21,14 +21,11 @@ import java.util.regex.Pattern;
|
|||
|
||||
@Slf4j
|
||||
@Component
|
||||
public final class PpomppuArticleParserV2 {
|
||||
public class PpomppuArticleParserV2 {
|
||||
|
||||
private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yy.MM.dd HH:mm:ss")
|
||||
.withZone(ZoneId.of("Asia/Seoul"));
|
||||
|
||||
private PpomppuArticleParserV2() {
|
||||
}
|
||||
|
||||
public List<Article> parse(String html) {
|
||||
Elements trElements = converHtmlToTrElements(html);
|
||||
return trElements.stream()
|
||||
|
@ -38,8 +35,8 @@ public final class PpomppuArticleParserV2 {
|
|||
|
||||
}
|
||||
|
||||
private Elements converHtmlToTrElements(String data) {
|
||||
Document document = Jsoup.parse(data);
|
||||
private Elements converHtmlToTrElements(String html) {
|
||||
Document document = Jsoup.parse(html);
|
||||
Elements trList = document.getElementById("revolution_main_table").getElementsByTag("tr");
|
||||
return trList;
|
||||
}
|
||||
|
@ -72,7 +69,7 @@ public final class PpomppuArticleParserV2 {
|
|||
String boardName = parseBoardName(title);
|
||||
Integer recommended = parseRecommended(tdList.get(4));
|
||||
Integer hit = NumberUtils.parseInt(tdList.get(5).text(), 0);
|
||||
ZonedDateTime registeredAt = DateTimeUtils.parse(tdList.get(3).text());
|
||||
ZonedDateTime registeredAt = DateTimeUtils.parse(tdList.get(3).text(), DateTimeUtils.FORMATTER_HHMMss, DateTimeUtils.FORMATTER_YYMMDD_SLASH);
|
||||
|
||||
return Article.builder()
|
||||
.articleId(articleId)
|
||||
|
|
|
@ -12,7 +12,7 @@ import lombok.NoArgsConstructor;
|
|||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
@Entity
|
||||
@Table
|
||||
@Table(name = "app_user")
|
||||
public class AppUser extends Auditable {
|
||||
@Id
|
||||
@GeneratedValue(strategy = GenerationType.IDENTITY)
|
||||
|
|
|
@ -15,7 +15,7 @@ import java.time.ZonedDateTime;
|
|||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
@Entity
|
||||
@Table
|
||||
@Table(name = "article")
|
||||
public class Article extends Auditable {
|
||||
|
||||
@Id
|
||||
|
|
|
@ -13,7 +13,7 @@ import lombok.NoArgsConstructor;
|
|||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
@Entity
|
||||
@Table
|
||||
@Table(name = "subscribed_keyword")
|
||||
public class SubscribedKeyword extends Auditable {
|
||||
|
||||
@Id
|
||||
|
|
|
@ -1,9 +1,11 @@
|
|||
package com.myoa.engineering.crawl.shopping.domain.model;
|
||||
|
||||
import com.myoa.engineering.crawl.shopping.domain.model.v2.ArticleModel;
|
||||
import com.slack.api.methods.response.chat.ChatPostMessageResponse;
|
||||
import lombok.*;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
|
||||
@ToString
|
||||
|
@ -12,13 +14,20 @@ import java.util.List;
|
|||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class UserNotifyModel {
|
||||
private String userId;
|
||||
private String slackId;
|
||||
private List<ArticleModel> articles;
|
||||
private ChatPostMessageResponse chatPostMessageResponse;
|
||||
|
||||
public static UserNotifyModel of(String userId, List<ArticleModel> articles) {
|
||||
return UserNotifyModel.builder()
|
||||
.userId(userId)
|
||||
.articles(articles)
|
||||
.build();
|
||||
public String toCompositedMessage() {
|
||||
return wrapUserId() + "\n" +
|
||||
articles.stream()
|
||||
.map(ArticleModel::convertArticletoMessage)
|
||||
.collect(Collectors.joining("\n"));
|
||||
}
|
||||
|
||||
private String wrapUserId() {
|
||||
return "<@" + slackId + ">";
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
|
@ -22,4 +22,7 @@ public class ArticleModel {
|
|||
private Integer recommended;
|
||||
private ZonedDateTime registeredAt;
|
||||
|
||||
public String convertArticletoMessage() {
|
||||
return "• <" + this.getArticleUrl() + "|" + this.getTitle() + ">";
|
||||
}
|
||||
}
|
||||
|
|
|
@ -0,0 +1,19 @@
|
|||
package com.myoa.engineering.crawl.shopping.dto;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.Setter;
|
||||
|
||||
@Getter
|
||||
@Setter
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class ExceptionMessage {
|
||||
private String timestamp;
|
||||
private int status;
|
||||
private String error;
|
||||
private String message;
|
||||
private String path;
|
||||
|
||||
}
|
|
@ -1,4 +1,4 @@
|
|||
package com.myoa.engineering.crawl.shopping.dto;
|
||||
package com.myoa.engineering.crawl.shopping.dto.feed.v1;
|
||||
|
||||
import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName;
|
||||
import lombok.Builder;
|
|
@ -1,4 +1,4 @@
|
|||
package com.myoa.engineering.crawl.shopping.dto;
|
||||
package com.myoa.engineering.crawl.shopping.dto.feed.v1;
|
||||
|
||||
import com.myoa.engineering.crawl.shopping.domain.entity.v1.PpomppuArticle;
|
||||
import com.myoa.engineering.crawl.shopping.support.util.DateUtil;
|
|
@ -1,4 +1,4 @@
|
|||
package com.myoa.engineering.crawl.shopping.dto;
|
||||
package com.myoa.engineering.crawl.shopping.dto.feed.v1;
|
||||
|
||||
import com.myoa.engineering.crawl.shopping.domain.entity.v1.PpomppuArticle;
|
||||
import com.myoa.engineering.crawl.shopping.support.dto.BlockMessageDTO;
|
|
@ -0,0 +1,20 @@
|
|||
package com.myoa.engineering.crawl.shopping.dto.feed.v1;
|
||||
|
||||
import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget;
|
||||
import lombok.*;
|
||||
|
||||
@Getter
|
||||
@Builder
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class SubscribedKeywordDTO {
|
||||
|
||||
private String userId;
|
||||
private CrawlTarget crawlTarget;
|
||||
private String keyword;
|
||||
|
||||
|
||||
public String toMessage() {
|
||||
return crawlTarget.getAlias() + " | " + keyword;
|
||||
}
|
||||
}
|
|
@ -1,4 +1,4 @@
|
|||
package com.myoa.engineering.crawl.shopping.dto.slack;
|
||||
package com.myoa.engineering.crawl.shopping.dto.slack.v1;
|
||||
|
||||
import java.io.Serializable;
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
package com.myoa.engineering.crawl.shopping.dto.slack;
|
||||
package com.myoa.engineering.crawl.shopping.dto.slack.v1;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
package com.myoa.engineering.crawl.shopping.dto.slack;
|
||||
package com.myoa.engineering.crawl.shopping.dto.slack.v1;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
|
@ -1,4 +1,4 @@
|
|||
package com.myoa.engineering.crawl.shopping.dto.slack;
|
||||
package com.myoa.engineering.crawl.shopping.dto.slack.v1;
|
||||
|
||||
import java.io.Serializable;
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
package com.myoa.engineering.crawl.shopping.dto.slack;
|
||||
package com.myoa.engineering.crawl.shopping.dto.slack.v1;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Getter;
|
|
@ -1,4 +1,4 @@
|
|||
package com.myoa.engineering.crawl.shopping.dto.slack;
|
||||
package com.myoa.engineering.crawl.shopping.dto.slack.v1;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
|
@ -1,4 +1,4 @@
|
|||
package com.myoa.engineering.crawl.shopping.dto.slack;
|
||||
package com.myoa.engineering.crawl.shopping.dto.slack.v1;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||
import lombok.Builder;
|
|
@ -0,0 +1,35 @@
|
|||
package com.myoa.engineering.crawl.shopping.dto.slack.v2;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Getter;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.ToString;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
@ToString
|
||||
@Getter
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class SlackEventSubscriptionDTO {
|
||||
|
||||
private String token;
|
||||
private String challenge;
|
||||
private String type;
|
||||
|
||||
@JsonProperty("team_id")
|
||||
private String teamId;
|
||||
|
||||
@JsonProperty("api_app_id")
|
||||
private String apiAppId;
|
||||
|
||||
private Map<String, Object> event;
|
||||
|
||||
@JsonProperty("event_id")
|
||||
private String eventId;
|
||||
|
||||
@JsonProperty("event_time")
|
||||
private long eventTime;
|
||||
|
||||
}
|
|
@ -0,0 +1,22 @@
|
|||
package com.myoa.engineering.crawl.shopping.dto.slack.v2;
|
||||
|
||||
import lombok.*;
|
||||
|
||||
@ToString
|
||||
@Getter
|
||||
@Setter
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class SlackSlashCommandDTO {
|
||||
|
||||
private String token;
|
||||
private String command;
|
||||
private String text;
|
||||
private String response_url;
|
||||
private String trigger_id;
|
||||
private String user_id;
|
||||
private String user_name;
|
||||
private String team_id;
|
||||
private String api_app_id;
|
||||
|
||||
}
|
|
@ -7,14 +7,17 @@ import com.myoa.engineering.crawl.shopping.domain.model.v2.SubscribedKeywordAggr
|
|||
import com.myoa.engineering.crawl.shopping.event.ArticleUpsertEvent;
|
||||
import com.myoa.engineering.crawl.shopping.service.AppUserQueryService;
|
||||
import com.myoa.engineering.crawl.shopping.service.SubscribedKeywordCacheService;
|
||||
import com.myoa.engineering.crawl.shopping.service.UserNotifyService;
|
||||
import com.myoa.engineering.crawl.shopping.service.slack.UserNotifyService;
|
||||
import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget;
|
||||
import com.myoa.engineering.crawl.shopping.util.SlackMessageUtils;
|
||||
import com.slack.api.methods.request.chat.ChatPostMessageRequest;
|
||||
import com.slack.api.methods.response.chat.ChatPostMessageResponse;
|
||||
import com.slack.api.model.block.Blocks;
|
||||
import org.springframework.context.event.EventListener;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.function.Function;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Component
|
||||
|
@ -38,49 +41,74 @@ public class ArticleUpsertEventListener {
|
|||
Map<CrawlTarget, List<ArticleModel>> articleMap =
|
||||
((List<ArticleModel>) event.getSource()).stream()
|
||||
.collect(Collectors.groupingBy(ArticleModel::getCrawlTarget));
|
||||
Map<CrawlTarget, ChatPostMessageResponse> allArticleNotifiedResultMap =
|
||||
articleMap.entrySet()
|
||||
.stream()
|
||||
.collect(Collectors.toMap(Map.Entry::getKey, e -> notifyMessage(e.getKey(), e.getValue())));
|
||||
|
||||
List<AppUserModel> appUsers = appUserQueryService.findAll();
|
||||
|
||||
appUsers.stream()
|
||||
.filter(AppUserModel::getEnabled)
|
||||
.map(user -> {
|
||||
List<ArticleModel> filteredArticles = handleAhoCorasick(articleMap)
|
||||
.apply(subscribedKeywordCacheService.getSubscribedKeywordsCached(user.getName()));
|
||||
return UserNotifyModel.of(user.getName(), filteredArticles);
|
||||
.flatMap(user -> {
|
||||
Map<CrawlTarget, SubscribedKeywordAggregatedModel> subscribedKeywords =
|
||||
subscribedKeywordCacheService.getSubscribedKeywordsCached(user.getSlackId());
|
||||
return subscribedKeywords.entrySet()
|
||||
.stream()
|
||||
.map(entry -> {
|
||||
List<ArticleModel> filtered = doAhocorasick(articleMap.get(entry.getKey()), entry.getValue());
|
||||
return UserNotifyModel.builder()
|
||||
.slackId(user.getSlackId())
|
||||
.articles(filtered)
|
||||
.chatPostMessageResponse(allArticleNotifiedResultMap.get(entry.getKey()))
|
||||
.build();
|
||||
});
|
||||
})
|
||||
.forEach(this::notifyMessage);
|
||||
|
||||
}
|
||||
|
||||
private Function<Map<CrawlTarget, SubscribedKeywordAggregatedModel>, List<ArticleModel>> handleAhoCorasick(
|
||||
Map<CrawlTarget, List<ArticleModel>> articleMap) {
|
||||
return userTrieModel -> {
|
||||
return userTrieModel
|
||||
.entrySet()
|
||||
.stream().filter(e -> articleMap.containsKey(e.getKey()))
|
||||
.map((entry) -> filterAhocorasick(articleMap.get(entry.getKey()), entry.getValue()))
|
||||
.flatMap(List::stream)
|
||||
.toList();
|
||||
// return UserNotifyModel.of(userTrieModel.values().stream().findFirst().get().getUserId(),
|
||||
// filteredArticle);
|
||||
};
|
||||
}
|
||||
|
||||
private List<ArticleModel> filterAhocorasick(List<ArticleModel> articles,
|
||||
SubscribedKeywordAggregatedModel trieModel) {
|
||||
private List<ArticleModel> doAhocorasick(List<ArticleModel> articles,
|
||||
SubscribedKeywordAggregatedModel trieModel) {
|
||||
if (articles == null || articles.isEmpty()) {
|
||||
return List.of();
|
||||
}
|
||||
return articles.stream()
|
||||
.filter(article -> !trieModel.getAhoCorasickTrie()
|
||||
.parseText(article.getTitle())
|
||||
.isEmpty())
|
||||
.toList();
|
||||
//ArticleUpsertEventListener::printArticle
|
||||
}
|
||||
|
||||
private void notifyMessage(UserNotifyModel article) {
|
||||
System.out.println("article = " + article);
|
||||
if (article.getArticles().isEmpty()){
|
||||
private ChatPostMessageResponse notifyMessage(CrawlTarget crawlTarget, List<ArticleModel> articles) {
|
||||
String composited = articles.stream()
|
||||
.map(ArticleModel::convertArticletoMessage)
|
||||
.collect(Collectors.joining("\n"));
|
||||
|
||||
ChatPostMessageRequest request =
|
||||
userNotifyService.generateMessage()
|
||||
.blocks(Blocks.asBlocks(
|
||||
SlackMessageUtils.ofHeader(crawlTarget.getAlias()),
|
||||
SlackMessageUtils.ofSection(composited),
|
||||
SlackMessageUtils.ofDivider()
|
||||
))
|
||||
.build();
|
||||
|
||||
return userNotifyService.notify(request);
|
||||
}
|
||||
|
||||
private void notifyMessage(UserNotifyModel userNotifyModel) {
|
||||
System.out.println("article = " + userNotifyModel);
|
||||
if (userNotifyModel.getArticles().isEmpty()) {
|
||||
return;
|
||||
}
|
||||
userNotifyService.notify("안녕 " + article.getUserId() + "\n" + article.getArticles());
|
||||
|
||||
ChatPostMessageRequest request =
|
||||
userNotifyService.generateMessage()
|
||||
.blocks(Blocks.asBlocks(
|
||||
SlackMessageUtils.ofSection(userNotifyModel.toCompositedMessage())
|
||||
))
|
||||
.build();
|
||||
userNotifyService.notify(request);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -1,12 +0,0 @@
|
|||
package com.myoa.engineering.crawl.shopping.infra.client.fmkorea;
|
||||
|
||||
import org.springframework.cloud.openfeign.FeignClient;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
|
||||
@FeignClient(value = "fmkorea-board-client", url = "https://fmkorea.com")
|
||||
public interface FmkoreaBoardClient {
|
||||
|
||||
@GetMapping("{boardLink}")
|
||||
String getBoardHtml(@PathVariable("boardLink") String boardLink);
|
||||
}
|
|
@ -0,0 +1,47 @@
|
|||
package com.myoa.engineering.crawl.shopping.infra.client.fmkorea;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.http.HttpHeaders;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.web.reactive.function.client.WebClient;
|
||||
|
||||
@Slf4j
|
||||
@Component
|
||||
public class FmkoreaBoardClientV2 {
|
||||
|
||||
private static final String URI_BOARD = "/index.php";
|
||||
|
||||
private final WebClient webClient;
|
||||
|
||||
public FmkoreaBoardClientV2() {
|
||||
webClient = WebClient.builder()
|
||||
.codecs(configurer -> configurer.defaultCodecs()
|
||||
.maxInMemorySize(2 * 1024 * 1024))
|
||||
.baseUrl("https://www.fmkorea.com")
|
||||
.defaultHeader(HttpHeaders.USER_AGENT, "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3")
|
||||
.defaultHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7\n")
|
||||
.defaultHeader(HttpHeaders.REFERER, "https://www.fmkorea.com")
|
||||
.defaultHeader(HttpHeaders.ACCEPT_ENCODING, "gzip, deflate, br")
|
||||
.defaultHeader(HttpHeaders.ACCEPT_LANGUAGE, "ko-KR,ko;q=0.9,en-US;q=0.8,en;q=0.7")
|
||||
.defaultHeader(HttpHeaders.CACHE_CONTROL, "max-age=0")
|
||||
.defaultHeader("Sec-Ch-Ua", "\"Whale\";v=\"3\", \"Not-A.Brand\";v=\"8\", \"Chromium\";v=\"122\"")
|
||||
.defaultHeader("Sec-Ch-Ua-Mobile", "?0")
|
||||
.defaultHeader("Sec-Ch-Ua-Platform", "\"macOS\"")
|
||||
.defaultHeader("Sec-Fetch-Dest", "document")
|
||||
.defaultHeader("Sec-Fetch-Mode", "navigate")
|
||||
.defaultHeader("Sec-Fetch-Site", "same-origin")
|
||||
.defaultHeader("Upgrade-Insecure-Requests", "1")
|
||||
.build();
|
||||
}
|
||||
|
||||
public String getBoardHtml(Integer page, String cookie) {
|
||||
return webClient.get()
|
||||
.uri(builder -> builder.path(URI_BOARD)
|
||||
.queryParam("page", page)
|
||||
.queryParam("mid", "hotdeal")
|
||||
.build())
|
||||
.header("Cookie", cookie)
|
||||
.exchangeToMono(clientResponse -> clientResponse.bodyToMono(String.class))
|
||||
.block();
|
||||
}
|
||||
}
|
|
@ -1,38 +0,0 @@
|
|||
package com.myoa.engineering.crawl.shopping.infra.client.ppomppu;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/**
|
||||
* PpomppuBoardFeedRetriever
|
||||
*
|
||||
* @author Shin Woo-jin (woozu.shin@kakaoent.com)
|
||||
* @since 2021-09-08
|
||||
*/
|
||||
@Slf4j
|
||||
@Component
|
||||
public class PpomppuBoardClient {
|
||||
/*
|
||||
private final WebClient webClient;
|
||||
|
||||
public PpomppuBoardClient(WebClient.Builder webClientBuilder) {
|
||||
this.webClient = webClientBuilder.baseUrl(PpomppuBoardName.PPOMPPU_URL)
|
||||
.exchangeStrategies(WebFluxExchangeStragiesFactory.ofTextHtml())
|
||||
.filter(WebClientFilterFactory.logRequest())
|
||||
.filter(WebClientFilterFactory.logResponse())
|
||||
.build();
|
||||
}
|
||||
|
||||
public Mono<String> getHtml(String uri) {
|
||||
return webClient.get()
|
||||
.uri(uri)
|
||||
.exchangeToMono(e -> e.bodyToMono(String.class))
|
||||
.publishOn(Schedulers.boundedElastic())
|
||||
.onErrorResume(WebClientRequestException.class, t -> {
|
||||
log.info("Exception occured, ignoring. : {}", t.getClass().getSimpleName());
|
||||
return Mono.empty();
|
||||
});
|
||||
// .doOnNext(e -> log.info("[getHtml] {}", e));
|
||||
}
|
||||
*/
|
||||
}
|
|
@ -1,17 +0,0 @@
|
|||
package com.myoa.engineering.crawl.shopping.infra.client.slack;
|
||||
|
||||
import com.myoa.engineering.crawl.shopping.configuration.FeignDefaultConfig;
|
||||
import com.myoa.engineering.crawl.shopping.dto.slack.SlackMessageDTO;
|
||||
import org.springframework.cloud.openfeign.FeignClient;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.bind.annotation.RequestHeader;
|
||||
|
||||
@FeignClient(value = "slack-api-client", url = "https://slack.com/api",
|
||||
configuration = FeignDefaultConfig.class)
|
||||
public interface SlackAPIClient {
|
||||
|
||||
@PostMapping("/chat.postMessage")
|
||||
String sendMessage(@RequestBody SlackMessageDTO message,
|
||||
@RequestHeader("Authorization") String token);
|
||||
}
|
|
@ -4,6 +4,9 @@ import com.myoa.engineering.crawl.shopping.domain.entity.v2.AppUser;
|
|||
import org.springframework.data.jpa.repository.JpaRepository;
|
||||
import org.springframework.stereotype.Repository;
|
||||
|
||||
import java.util.Optional;
|
||||
|
||||
@Repository
|
||||
public interface AppUserRepository extends JpaRepository<AppUser, Long> {
|
||||
Optional<AppUser> findBySlackId(String userId);
|
||||
}
|
||||
|
|
|
@ -6,6 +6,7 @@ import org.springframework.data.jpa.repository.JpaRepository;
|
|||
import org.springframework.stereotype.Repository;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
@Repository
|
||||
public interface SubscribedKeywordRepository extends JpaRepository<SubscribedKeyword, Long> {
|
||||
|
@ -20,4 +21,6 @@ public interface SubscribedKeywordRepository extends JpaRepository<SubscribedKey
|
|||
List<SubscribedKeyword> findByUserIdAndCrawlTarget(String userId, CrawlTarget crawlTarget);
|
||||
|
||||
List<SubscribedKeyword> findByUserId(String userId);
|
||||
|
||||
Optional<SubscribedKeyword> findByUserIdAndCrawlTargetAndKeyword(String userId, CrawlTarget crawlTarget, String keyword);
|
||||
}
|
||||
|
|
|
@ -1,15 +1,20 @@
|
|||
package com.myoa.engineering.crawl.shopping.scheduler;
|
||||
|
||||
import com.myoa.engineering.crawl.shopping.crawlhandler.CrawlHandler;
|
||||
import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.context.annotation.Profile;
|
||||
import org.springframework.scheduling.annotation.EnableScheduling;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
|
||||
@Slf4j
|
||||
@Component
|
||||
@EnableScheduling
|
||||
@Profile("!local")
|
||||
public class ParseEventEmitter {
|
||||
private final List<CrawlHandler> crawlHandlers;
|
||||
|
||||
|
@ -18,10 +23,11 @@ public class ParseEventEmitter {
|
|||
this.crawlHandlers = crawlHandlers;
|
||||
}
|
||||
|
||||
// @Scheduled(cron = "0 0/5 * * * ?")
|
||||
@Scheduled(cron = "0 0/15 * * * ?")
|
||||
public void emit() {
|
||||
log.info("[emitDomesticBoard] trigger fired!");
|
||||
crawlHandlers.forEach(CrawlHandler::handle);
|
||||
|
||||
crawlHandlers
|
||||
.stream()
|
||||
.filter(e -> e.getCrawlTarget().isAvailable())
|
||||
.forEach(CrawlHandler::handle);
|
||||
}
|
||||
}
|
||||
|
|
|
@ -0,0 +1,31 @@
|
|||
package com.myoa.engineering.crawl.shopping.service;
|
||||
|
||||
import com.myoa.engineering.crawl.shopping.domain.entity.v2.AppUser;
|
||||
import com.myoa.engineering.crawl.shopping.infra.repository.v2.AppUserRepository;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
|
||||
@Service
|
||||
public class AppUserCommandService {
|
||||
private final AppUserRepository appUserRepository;
|
||||
|
||||
public AppUserCommandService(AppUserRepository appUserRepository) {
|
||||
this.appUserRepository = appUserRepository;
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public void register(String slackId, String userName) {
|
||||
appUserRepository.findBySlackId(slackId)
|
||||
.ifPresent(user -> {
|
||||
throw new IllegalArgumentException("이미 등록된 사용자입니다.");
|
||||
});
|
||||
|
||||
AppUser appUser = AppUser.builder()
|
||||
.enabled(true)
|
||||
.slackId(slackId)
|
||||
.name(userName)
|
||||
.build();
|
||||
|
||||
appUserRepository.save(appUser);
|
||||
}
|
||||
}
|
|
@ -3,6 +3,7 @@ package com.myoa.engineering.crawl.shopping.service;
|
|||
import com.myoa.engineering.crawl.shopping.domain.model.v2.AppUserModel;
|
||||
import com.myoa.engineering.crawl.shopping.infra.repository.v2.AppUserRepository;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
|
@ -15,6 +16,7 @@ public class AppUserQueryService {
|
|||
this.appUserRepository = appUserRepository;
|
||||
}
|
||||
|
||||
@Transactional(readOnly = true)
|
||||
public List<AppUserModel> findAll() {
|
||||
return appUserRepository.findAll()
|
||||
.stream()
|
||||
|
@ -22,4 +24,11 @@ public class AppUserQueryService {
|
|||
.toList();
|
||||
}
|
||||
|
||||
@Transactional(readOnly = true)
|
||||
public AppUserModel findByUserId(String userId) {
|
||||
return appUserRepository.findBySlackId(userId)
|
||||
.map(AppUserModel::from)
|
||||
.orElseThrow();
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -19,23 +19,23 @@ public class SubscribedKeywordCacheService {
|
|||
this.subscribedKeywordQueryService = subscribedKeywordQueryService;
|
||||
}
|
||||
|
||||
@Cacheable(cacheNames = "subscribe.keywords", key = "#userId + '_' + #crawlTarget.name()")
|
||||
public SubscribedKeywordAggregatedModel getSubscribedKeywordsCached(String userId, CrawlTarget crawlTarget) {
|
||||
@Cacheable(cacheNames = "subscribe.keywords", key = "#slackId + '_' + #crawlTarget.name()")
|
||||
public SubscribedKeywordAggregatedModel getSubscribedKeywordsCached(String slackId, CrawlTarget crawlTarget) {
|
||||
System.out.println("getSubscribedKeywordsCached");
|
||||
List<String> keywords = subscribedKeywordQueryService.findByUserWithTarget(userId, crawlTarget)
|
||||
List<String> keywords = subscribedKeywordQueryService.findByUserWithTarget(slackId, crawlTarget)
|
||||
.stream().map(SubscribedKeyword::getKeyword).toList();
|
||||
return SubscribedKeywordAggregatedModel.of(userId, crawlTarget, keywords);
|
||||
return SubscribedKeywordAggregatedModel.of(slackId, crawlTarget, keywords);
|
||||
}
|
||||
|
||||
@Cacheable(cacheNames = "subscribe.keywords", key = "#userId")
|
||||
public Map<CrawlTarget, SubscribedKeywordAggregatedModel> getSubscribedKeywordsCached(String userId) {
|
||||
@Cacheable(cacheNames = "subscribe.keywords", key = "#slackId")
|
||||
public Map<CrawlTarget, SubscribedKeywordAggregatedModel> getSubscribedKeywordsCached(String slackId) {
|
||||
System.out.println("getSubscribedKeywordsCached");
|
||||
return subscribedKeywordQueryService.findByUser(userId)
|
||||
return subscribedKeywordQueryService.findByUser(slackId)
|
||||
.stream()
|
||||
.collect(Collectors.groupingBy(SubscribedKeyword::getCrawlTarget,
|
||||
Collectors.mapping(SubscribedKeyword::getKeyword, Collectors.toList())))
|
||||
.entrySet().stream()
|
||||
.collect(Collectors.toMap(Map.Entry::getKey, e -> SubscribedKeywordAggregatedModel.of(userId, e.getKey(), e.getValue())));
|
||||
.collect(Collectors.toMap(Map.Entry::getKey, e -> SubscribedKeywordAggregatedModel.of(slackId, e.getKey(), e.getValue())));
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -0,0 +1,46 @@
|
|||
package com.myoa.engineering.crawl.shopping.service;
|
||||
|
||||
import com.myoa.engineering.crawl.shopping.domain.entity.v2.SubscribedKeyword;
|
||||
import com.myoa.engineering.crawl.shopping.dto.feed.v1.SubscribedKeywordDTO;
|
||||
import com.myoa.engineering.crawl.shopping.infra.repository.v2.SubscribedKeywordRepository;
|
||||
import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Service
|
||||
public class SubscribedKeywordCommandService {
|
||||
|
||||
private final SubscribedKeywordRepository subscribedKeywordRepository;
|
||||
|
||||
public SubscribedKeywordCommandService(SubscribedKeywordRepository subscribedKeywordRepository) {
|
||||
this.subscribedKeywordRepository = subscribedKeywordRepository;
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public void add(SubscribedKeyword subscribedKeyword) {
|
||||
subscribedKeywordRepository.save(subscribedKeyword);
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public void delete(String userId, CrawlTarget crawlTarget, String keyword) {
|
||||
SubscribedKeyword subscribedKeyword = subscribedKeywordRepository.findByUserIdAndCrawlTargetAndKeyword(userId, crawlTarget, keyword)
|
||||
.orElseThrow(() -> new IllegalArgumentException("Not found keyword: " + keyword));
|
||||
|
||||
subscribedKeywordRepository.delete(subscribedKeyword);
|
||||
}
|
||||
|
||||
@Transactional(readOnly = true)
|
||||
public List<SubscribedKeywordDTO> list(String userId) {
|
||||
return subscribedKeywordRepository.findByUserId(userId)
|
||||
.stream()
|
||||
.map(e -> SubscribedKeywordDTO.builder()
|
||||
.keyword(e.getKeyword())
|
||||
.userId(e.getUserId())
|
||||
.crawlTarget(e.getCrawlTarget())
|
||||
.build())
|
||||
.toList();
|
||||
}
|
||||
|
||||
}
|
|
@ -1,34 +0,0 @@
|
|||
package com.myoa.engineering.crawl.shopping.service;
|
||||
|
||||
import com.myoa.engineering.crawl.shopping.configuration.slack.properties.SlackSecretProperties;
|
||||
import com.myoa.engineering.crawl.shopping.dto.slack.SlackMessageDTO;
|
||||
import com.myoa.engineering.crawl.shopping.infra.client.slack.SlackAPIClient;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
@Service
|
||||
public class UserNotifyService {
|
||||
|
||||
private static final String SLACK_PROPERTIES_UNIT_NAME = "shopping-crawler";
|
||||
// private static final String NOTIFY_CHANNEL_ID = "notify_shopping";
|
||||
// private static final String NOTIFY_ICON_EMOJI = ":monge_big:";
|
||||
// private static final String NOTIFY_BOT_NAME = "몽이 탈호구봇";
|
||||
|
||||
private final SlackAPIClient slackAPIClient;
|
||||
private final SlackSecretProperties.SlackSecretPropertiesUnit slackSecretProperties;
|
||||
|
||||
public UserNotifyService(SlackAPIClient slackAPIClient,
|
||||
SlackSecretProperties slackSecretProperties) {
|
||||
this.slackAPIClient = slackAPIClient;
|
||||
this.slackSecretProperties = slackSecretProperties.find(SLACK_PROPERTIES_UNIT_NAME);
|
||||
}
|
||||
|
||||
public void notify(String message) {
|
||||
SlackMessageDTO slackMessageDTO = SlackMessageDTO.builder()
|
||||
.channel(slackSecretProperties.getChannel())
|
||||
.text(message)
|
||||
.iconEmoji(slackSecretProperties.getIconEmoji())
|
||||
.username(slackSecretProperties.getUsername())
|
||||
.build();
|
||||
slackAPIClient.sendMessage(slackMessageDTO, slackSecretProperties.getToken());
|
||||
}
|
||||
}
|
|
@ -0,0 +1,33 @@
|
|||
package com.myoa.engineering.crawl.shopping.service.slack;
|
||||
|
||||
import com.myoa.engineering.crawl.shopping.dto.slack.v2.SlackSlashCommandDTO;
|
||||
import com.myoa.engineering.crawl.shopping.service.AppUserCommandService;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Service
|
||||
public class RegisterCommandHandler implements SlackCommandHandler {
|
||||
|
||||
private final AppUserCommandService appUserCommandService;
|
||||
|
||||
public RegisterCommandHandler(AppUserCommandService appUserCommandService) {
|
||||
this.appUserCommandService = appUserCommandService;
|
||||
}
|
||||
|
||||
public List<String> handle(SlackSlashCommandDTO requestDTO) {
|
||||
if (!requestDTO.getCommand().equals("/등록")) {
|
||||
throw new IllegalArgumentException("Invalid command: " + requestDTO.getCommand());
|
||||
}
|
||||
|
||||
final String userName = requestDTO.getText();
|
||||
|
||||
try {
|
||||
appUserCommandService.register(requestDTO.getUser_id(), userName);
|
||||
return List.of("등록 완료");
|
||||
} catch (Exception e) {
|
||||
return List.of(e.getMessage());
|
||||
}
|
||||
|
||||
}
|
||||
}
|
|
@ -0,0 +1,34 @@
|
|||
package com.myoa.engineering.crawl.shopping.service.slack;
|
||||
|
||||
import com.myoa.engineering.crawl.shopping.dto.slack.v2.SlackSlashCommandDTO;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Service
|
||||
public class ShoppingCommandHandler implements SlackCommandHandler {
|
||||
|
||||
private final List<ShoppingCommandProcessor> processors;
|
||||
|
||||
public ShoppingCommandHandler(List<ShoppingCommandProcessor> processors) {
|
||||
this.processors = processors;
|
||||
}
|
||||
|
||||
public List<String> handle(SlackSlashCommandDTO requestDTO) {
|
||||
String[] commands = requestDTO.getText().split("\\s+", 2);
|
||||
|
||||
if (!requestDTO.getCommand().equals("/쇼핑")) {
|
||||
throw new IllegalArgumentException("Invalid command: " + requestDTO.getCommand());
|
||||
}
|
||||
|
||||
final String action = commands[0];
|
||||
|
||||
return processors.stream()
|
||||
.filter(processor -> processor.isAssignable(action))
|
||||
.map(processor -> processor.process(requestDTO))
|
||||
.toList();
|
||||
|
||||
}
|
||||
|
||||
|
||||
}
|
|
@ -0,0 +1,10 @@
|
|||
package com.myoa.engineering.crawl.shopping.service.slack;
|
||||
|
||||
import com.myoa.engineering.crawl.shopping.dto.slack.v2.SlackSlashCommandDTO;
|
||||
|
||||
public interface ShoppingCommandProcessor {
|
||||
|
||||
boolean isAssignable(String menuContext);
|
||||
|
||||
String process(SlackSlashCommandDTO requestDTO);
|
||||
}
|
|
@ -0,0 +1,4 @@
|
|||
package com.myoa.engineering.crawl.shopping.service.slack;
|
||||
|
||||
public interface SlackCommandHandler {
|
||||
}
|
|
@ -0,0 +1,41 @@
|
|||
package com.myoa.engineering.crawl.shopping.service.slack;
|
||||
|
||||
import com.myoa.engineering.crawl.shopping.configuration.slack.properties.SlackSecretProperties;
|
||||
import com.slack.api.methods.MethodsClient;
|
||||
import com.slack.api.methods.request.chat.ChatPostMessageRequest;
|
||||
import com.slack.api.methods.response.chat.ChatPostMessageResponse;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
public class UserNotifyService {
|
||||
|
||||
private static final String SLACK_PROPERTIES_UNIT_NAME = "shopping-crawler";
|
||||
|
||||
private final SlackSecretProperties.SlackSecretPropertiesUnit slackSecretProperties;
|
||||
private final MethodsClient methodsClient;
|
||||
|
||||
public UserNotifyService(SlackSecretProperties slackSecretProperties,
|
||||
MethodsClient methodsClient) {
|
||||
this.slackSecretProperties = slackSecretProperties.find(SLACK_PROPERTIES_UNIT_NAME);
|
||||
this.methodsClient = methodsClient;
|
||||
}
|
||||
|
||||
public ChatPostMessageResponse notify(ChatPostMessageRequest request) {
|
||||
try {
|
||||
return methodsClient.chatPostMessage(request);
|
||||
} catch (Exception e) {
|
||||
log.warn("Failed. message: {}", request, e);
|
||||
ChatPostMessageResponse response = new ChatPostMessageResponse();
|
||||
response.setOk(false);
|
||||
return response;
|
||||
}
|
||||
}
|
||||
|
||||
public ChatPostMessageRequest.ChatPostMessageRequestBuilder generateMessage() {
|
||||
return ChatPostMessageRequest.builder()
|
||||
.channel(slackSecretProperties.getChannel())
|
||||
.username(slackSecretProperties.getUsername());
|
||||
}
|
||||
}
|
|
@ -0,0 +1,67 @@
|
|||
package com.myoa.engineering.crawl.shopping.service.slack.shopping;
|
||||
|
||||
import com.myoa.engineering.crawl.shopping.domain.entity.v2.SubscribedKeyword;
|
||||
import com.myoa.engineering.crawl.shopping.dto.feed.v1.SubscribedKeywordDTO;
|
||||
import com.myoa.engineering.crawl.shopping.dto.slack.v2.SlackSlashCommandDTO;
|
||||
import com.myoa.engineering.crawl.shopping.service.SubscribedKeywordCommandService;
|
||||
import com.myoa.engineering.crawl.shopping.service.slack.ShoppingCommandProcessor;
|
||||
import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Component
|
||||
public class KeywordShoppingCommandProcessor implements ShoppingCommandProcessor {
|
||||
|
||||
private final SubscribedKeywordCommandService subscribedKeywordCommandService;
|
||||
|
||||
public KeywordShoppingCommandProcessor(SubscribedKeywordCommandService subscribedKeywordCommandService) {
|
||||
this.subscribedKeywordCommandService = subscribedKeywordCommandService;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isAssignable(String menuContext) {
|
||||
return menuContext.equals("키워드");
|
||||
}
|
||||
|
||||
@Override
|
||||
public String process(SlackSlashCommandDTO requestDTO) {
|
||||
// "키워드 추가 (쇼핑몰명) (키워드명)"
|
||||
final String[] splited = requestDTO.getText().split("\\s+", 4);
|
||||
|
||||
final String action = splited[1];
|
||||
|
||||
switch (action) {
|
||||
case "추가" -> {
|
||||
final CrawlTarget crawlTarget = CrawlTarget.fromAlias(splited[2]);
|
||||
final String keyword = splited[3];
|
||||
SubscribedKeyword subscribedKeyword =
|
||||
SubscribedKeyword.builder()
|
||||
.crawlTarget(crawlTarget)
|
||||
.userId(requestDTO.getUser_id())
|
||||
.keyword(keyword)
|
||||
.build();
|
||||
subscribedKeywordCommandService.add(subscribedKeyword);
|
||||
return "keyword: [" + keyword + "] 추가완료";
|
||||
}
|
||||
case "삭제" -> {
|
||||
final CrawlTarget crawlTarget = CrawlTarget.fromAlias(splited[2]);
|
||||
final String keyword = splited[3];
|
||||
subscribedKeywordCommandService.delete(requestDTO.getUser_id(), crawlTarget, keyword);
|
||||
return "keyword: [" + keyword + "] 삭제완료";
|
||||
}
|
||||
case "목록" -> {
|
||||
List<SubscribedKeywordDTO> keywords = subscribedKeywordCommandService.list(requestDTO.getUser_id());
|
||||
if (keywords.isEmpty()) {
|
||||
return "등록된 키워드가 없습니다.";
|
||||
}
|
||||
return keywords.stream().map(SubscribedKeywordDTO::toMessage)
|
||||
.collect(Collectors.joining("\n"));
|
||||
}
|
||||
default -> {
|
||||
return "지원하지 않는 명령어입니다.";
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
|
@ -0,0 +1,25 @@
|
|||
package com.myoa.engineering.crawl.shopping.service.slack.shopping;
|
||||
|
||||
import com.myoa.engineering.crawl.shopping.dto.slack.v2.SlackSlashCommandDTO;
|
||||
import com.myoa.engineering.crawl.shopping.service.slack.ShoppingCommandProcessor;
|
||||
import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
@Component
|
||||
public class ListShoppingCommandProcessor implements ShoppingCommandProcessor {
|
||||
|
||||
@Override
|
||||
public boolean isAssignable(String menuContext) {
|
||||
return menuContext.equals("목록");
|
||||
}
|
||||
|
||||
@Override
|
||||
public String process(SlackSlashCommandDTO requestDTO) {
|
||||
return Stream.of(CrawlTarget.values())
|
||||
.map(e -> e.getAlias() + " | " + e.isAvailable())
|
||||
.collect(Collectors.joining("\n"));
|
||||
}
|
||||
}
|
|
@ -1,27 +1,38 @@
|
|||
package com.myoa.engineering.crawl.shopping.util;
|
||||
|
||||
import org.springframework.lang.Nullable;
|
||||
|
||||
import java.time.*;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
|
||||
public final class DateTimeUtils {
|
||||
|
||||
private static final DateTimeFormatter FORMATTER_HHMMss = DateTimeFormatter.ofPattern("HH:mm:ss");
|
||||
public static final DateTimeFormatter FORMATTER_HHMMss = DateTimeFormatter.ofPattern("HH:mm:ss");
|
||||
public static final DateTimeFormatter FORMATTER_YYMMDD_SLASH = DateTimeFormatter.ofPattern("yy/MM/dd");
|
||||
public static final DateTimeFormatter FORMATTER_YYMMDD_DOT = DateTimeFormatter.ofPattern("yyyy.MM.dd");
|
||||
public static final DateTimeFormatter FORMATTER_HHMM = DateTimeFormatter.ofPattern("HH:mm");
|
||||
private static final ZoneId ZONE_ASIA_SEOUL = ZoneId.of("Asia/Seoul");
|
||||
|
||||
private DateTimeUtils() {
|
||||
}
|
||||
|
||||
public static ZonedDateTime parse(String HHMMss) {
|
||||
public static ZonedDateTime parse(String dateTimeString, DateTimeFormatter formatter, @Nullable DateTimeFormatter fallback) {
|
||||
try {
|
||||
LocalTime time = LocalTime.parse(HHMMss, FORMATTER_HHMMss);
|
||||
LocalTime time = LocalTime.parse(dateTimeString, formatter);
|
||||
LocalDateTime dateTime = LocalDateTime.of(LocalDate.now(), time);
|
||||
if (dateTime.isAfter(LocalDateTime.now())) {
|
||||
dateTime = dateTime.minusDays(1);
|
||||
}
|
||||
return dateTime.atZone(ZONE_ASIA_SEOUL);
|
||||
} catch (Exception e) {
|
||||
return null;
|
||||
} catch (Exception ignored) {
|
||||
}
|
||||
|
||||
try {
|
||||
LocalDate date = LocalDate.parse(dateTimeString, fallback);
|
||||
return date.atStartOfDay(ZONE_ASIA_SEOUL);
|
||||
} catch (Exception ignored) {
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -0,0 +1,33 @@
|
|||
package com.myoa.engineering.crawl.shopping.util;
|
||||
|
||||
import com.slack.api.model.block.DividerBlock;
|
||||
import com.slack.api.model.block.HeaderBlock;
|
||||
import com.slack.api.model.block.SectionBlock;
|
||||
import com.slack.api.model.block.composition.MarkdownTextObject;
|
||||
import com.slack.api.model.block.composition.PlainTextObject;
|
||||
|
||||
public final class SlackMessageUtils {
|
||||
|
||||
private SlackMessageUtils() {
|
||||
}
|
||||
|
||||
public static HeaderBlock ofHeader(String message) {
|
||||
return HeaderBlock.builder()
|
||||
.text(PlainTextObject.builder()
|
||||
.text(message)
|
||||
.build())
|
||||
.build();
|
||||
}
|
||||
|
||||
public static DividerBlock ofDivider() {
|
||||
return DividerBlock.builder().build();
|
||||
}
|
||||
|
||||
public static SectionBlock ofSection(String message) {
|
||||
return SectionBlock.builder()
|
||||
.text(MarkdownTextObject.builder()
|
||||
.text(message)
|
||||
.build())
|
||||
.build();
|
||||
}
|
||||
}
|
|
@ -1,12 +0,0 @@
|
|||
spring:
|
||||
config:
|
||||
activate:
|
||||
on-profile: development
|
||||
import:
|
||||
- "configserver:http://192.168.0.100:11080"
|
||||
|
||||
|
||||
server:
|
||||
port: 20081
|
||||
|
||||
# import: optional:configserver:http://localhost:11080 # can be start up even config server was not found.
|
|
@ -0,0 +1,23 @@
|
|||
spring:
|
||||
config:
|
||||
activate:
|
||||
on-profile: prod
|
||||
import:
|
||||
- classpath:/datasource/prod.yml
|
||||
- classpath:/slack/prod.yml
|
||||
devtools:
|
||||
livereload:
|
||||
enabled: false
|
||||
restart:
|
||||
enabled: false
|
||||
|
||||
server:
|
||||
port: 20080
|
||||
|
||||
# import: optional:configserver:http://localhost:11080 # can be start up even config server was not found.
|
||||
|
||||
feign:
|
||||
client:
|
||||
config:
|
||||
default:
|
||||
loggerLevel: FULL
|
|
@ -1,6 +0,0 @@
|
|||
spring:
|
||||
config:
|
||||
activate:
|
||||
on-profile: production
|
||||
import:
|
||||
- "configserver:http://ppn-config-server:20080"
|
|
@ -7,8 +7,7 @@ spring:
|
|||
active: ${SPRING_ACTIVE_PROFILE:local}
|
||||
group:
|
||||
local: "local,datasource-local,webclient-local"
|
||||
development: "development,datasource-development,webclient-development"
|
||||
production: "production,datasource-production,webclient-production"
|
||||
prod: "prod"
|
||||
freemarker:
|
||||
enabled: false
|
||||
cloud:
|
||||
|
|
|
@ -2,14 +2,14 @@
|
|||
<configuration>
|
||||
<springProperty name="DEFAULT_LEVEL_CONFIG" source="log.defaultLevel" />
|
||||
<springProfile name="local">
|
||||
<include resource="logback/logback-development.xml" />
|
||||
<include resource="logback/logback-dev.xml" />
|
||||
<logger name="org.apache.kafka" level="INFO" />
|
||||
</springProfile>
|
||||
<springProfile name="development">
|
||||
<include resource="logback/logback-development.xml" />
|
||||
<springProfile name="dev">
|
||||
<include resource="logback/logback-dev.xml" />
|
||||
<logger name="org.apache.kafka" level="INFO" />
|
||||
</springProfile>
|
||||
<springProfile name="production">
|
||||
<include resource="logback/logback-production.xml" />
|
||||
<springProfile name="prod">
|
||||
<include resource="logback/logback-prod.xml" />
|
||||
</springProfile>
|
||||
</configuration>
|
|
@ -2,6 +2,10 @@
|
|||
<included>
|
||||
<!-- =========== property BETA ========= -->
|
||||
<property name="DEFAULT_LEVEL" value="${DEFAULT_LEVEL_CONFIG:-INFO}"/>
|
||||
<!--file-->
|
||||
<property name="IMMEDIATE_FLUSH" value="true"/>
|
||||
<!--nelo2-->
|
||||
<property name="NELO2_LEVEL" value="WARN"/>
|
||||
<!-- =========== include appender =========== -->
|
||||
<include resource="org/springframework/boot/logging/logback/defaults.xml"/>
|
||||
<include resource="org/springframework/boot/logging/logback/console-appender.xml"/>
|
|
@ -1,19 +0,0 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<included>
|
||||
<!-- =========== property BETA ========= -->
|
||||
<property name="DEFAULT_LEVEL" value="${DEFAULT_LEVEL_CONFIG:-INFO}"/>
|
||||
<!--file-->
|
||||
<property name="DIRECTORY" value="/home1/www/logs/supervisor"/>
|
||||
<property name="IMMEDIATE_FLUSH" value="true"/>
|
||||
<!--nelo2-->
|
||||
<property name="NELO2_LEVEL" value="WARN"/>
|
||||
<!-- =========== include appender =========== -->
|
||||
<include resource="org/springframework/boot/logging/logback/defaults.xml"/>
|
||||
<include resource="org/springframework/boot/logging/logback/console-appender.xml"/>
|
||||
<include resource="logback/component/logback-nelo2.xml"/>
|
||||
<include resource="logback/component/logback-datachain.xml"/>
|
||||
<!-- =========== root logger ============== -->
|
||||
<root level="${DEFAULT_LEVEL}">
|
||||
<appender-ref ref="CONSOLE"/>
|
||||
</root>
|
||||
</included>
|
|
@ -3,15 +3,12 @@
|
|||
<!-- =========== property RELEASE ========= -->
|
||||
<property name="DEFAULT_LEVEL" value="${DEFAULT_LEVEL_CONFIG:-INFO}"/>
|
||||
<!--file-->
|
||||
<property name="DIRECTORY" value="/home1/www/logs/supervisor"/>
|
||||
<property name="IMMEDIATE_FLUSH" value="true"/>
|
||||
<!--nelo2-->
|
||||
<property name="NELO2_LEVEL" value="WARN"/>
|
||||
<!-- =========== include appender =========== -->
|
||||
<include resource="org/springframework/boot/logging/logback/defaults.xml"/>
|
||||
<include resource="org/springframework/boot/logging/logback/console-appender.xml"/>
|
||||
<include resource="logback/component/logback-nelo2.xml"/>
|
||||
<include resource="logback/component/logback-datachain.xml"/>
|
||||
<!-- =========== root logger ============== -->
|
||||
<root level="${DEFAULT_LEVEL}">
|
||||
<appender-ref ref="CONSOLE"/>
|
|
@ -0,0 +1,33 @@
|
|||
package com.myoa.engineering.crawl.shopping.controller;
|
||||
|
||||
import org.junit.jupiter.api.Assertions;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
class SlackEventSubscriptionAPIControllerTest {
|
||||
|
||||
@Test
|
||||
void test_handleAddSubscription_1() {
|
||||
// given
|
||||
String sut = "/쇼핑 키워드 추가 키워드1";
|
||||
|
||||
// when
|
||||
String actual = sut.replaceFirst("/.+? ", "");
|
||||
|
||||
// then
|
||||
Assertions.assertEquals("키워드 추가 키워드1", actual);
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_handleAddSubscription_2() {
|
||||
// given
|
||||
String sut = "키워드 추가 키워드 1입니 다";
|
||||
|
||||
// when
|
||||
String[] actual = sut.split("\\s+", 2);
|
||||
|
||||
// then
|
||||
Assertions.assertEquals(2, actual.length);
|
||||
Assertions.assertEquals("키워드", actual[0]);
|
||||
Assertions.assertEquals("추가 키워드 1입니 다", actual[1]);
|
||||
}
|
||||
}
|
|
@ -0,0 +1,63 @@
|
|||
package com.myoa.engineering.crawl.shopping.controller;
|
||||
|
||||
import com.slack.api.methods.MethodsClient;
|
||||
import com.slack.api.methods.SlackApiException;
|
||||
import com.slack.api.methods.request.chat.ChatPostMessageRequest;
|
||||
import com.slack.api.model.block.DividerBlock;
|
||||
import com.slack.api.model.block.HeaderBlock;
|
||||
import com.slack.api.model.block.LayoutBlock;
|
||||
import com.slack.api.model.block.SectionBlock;
|
||||
import com.slack.api.model.block.composition.PlainTextObject;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.test.context.SpringBootTest;
|
||||
import org.springframework.test.context.ActiveProfiles;
|
||||
import org.springframework.test.context.junit.jupiter.SpringExtension;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
@ActiveProfiles("local")
|
||||
@ExtendWith(SpringExtension.class)
|
||||
@SpringBootTest
|
||||
class TestAPIControllerTest {
|
||||
|
||||
@Autowired
|
||||
MethodsClient methodsClient;
|
||||
|
||||
@Test
|
||||
void test1() throws SlackApiException, IOException {
|
||||
List<LayoutBlock> blocks = new ArrayList<>();
|
||||
HeaderBlock asdf = HeaderBlock.builder()
|
||||
.text(PlainTextObject.builder()
|
||||
.text("asdf")
|
||||
.build())
|
||||
.build();
|
||||
|
||||
DividerBlock dividerBlock = DividerBlock.builder().build();
|
||||
SectionBlock section = SectionBlock.builder()
|
||||
.text(PlainTextObject.builder()
|
||||
.text("• asdf")
|
||||
.build())
|
||||
.build();
|
||||
blocks.add(asdf);
|
||||
blocks.add(section);
|
||||
blocks.add(section);
|
||||
blocks.add(section);
|
||||
blocks.add(dividerBlock);
|
||||
|
||||
ChatPostMessageRequest request = ChatPostMessageRequest.builder()
|
||||
.channel("notify_shopping")
|
||||
.blocks(blocks)
|
||||
.build();
|
||||
methodsClient.chatPostMessage(request);
|
||||
|
||||
}
|
||||
|
||||
private void notifyMessage() {
|
||||
|
||||
}
|
||||
|
||||
}
|
|
@ -0,0 +1,16 @@
|
|||
package com.myoa.engineering.crawl.shopping.crawlhandler;
|
||||
|
||||
import com.myoa.engineering.crawl.shopping.util.TestDataUtils;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
class FmkoreaCrawlHandlerTest {
|
||||
|
||||
|
||||
@Test
|
||||
void resolve_fake430() {
|
||||
String fakeHtml = TestDataUtils.fileToString("testdata/fmkorea/fake430.html");
|
||||
FmkoreaFake430Resolver.resolveFake430(fakeHtml);
|
||||
}
|
||||
|
||||
|
||||
}
|
|
@ -0,0 +1,31 @@
|
|||
package com.myoa.engineering.crawl.shopping.crawlhandler.parser;
|
||||
|
||||
import com.myoa.engineering.crawl.shopping.domain.entity.v2.Article;
|
||||
import com.myoa.engineering.crawl.shopping.util.TestDataUtils;
|
||||
import org.junit.jupiter.api.Assertions;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
class FmkoreaArticleParserTest {
|
||||
|
||||
private FmkoreaArticleParser sut;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
sut = new FmkoreaArticleParser();
|
||||
}
|
||||
|
||||
@Test
|
||||
void parse() {
|
||||
// given
|
||||
String boardHtml = TestDataUtils.fileToString("testdata/fmkorea/file1.html");
|
||||
|
||||
// when
|
||||
List<Article> actual = sut.parse(boardHtml);
|
||||
|
||||
// then
|
||||
Assertions.assertEquals(20, actual.size());
|
||||
}
|
||||
}
|
|
@ -1,6 +0,0 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<configuratiown>
|
||||
<springProperty name="DEFAULT_LEVEL_CONFIG" source="log.defaultLevel"/>
|
||||
|
||||
<include resource="logback-development.xml"/>
|
||||
</configuratiown>
|
File diff suppressed because one or more lines are too long
File diff suppressed because it is too large
Load Diff
|
@ -0,0 +1,17 @@
|
|||
curl 'https://www.fmkorea.com/hotdeal' \
|
||||
-H 'accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7' \
|
||||
-H 'accept-language: ko-KR,ko;q=0.9,en-US;q=0.8,en;q=0.7' \
|
||||
-H 'cache-control: max-age=0' \
|
||||
-H 'cookie: PHPSESSID=gkpqv1r1odheqkqpal2eh24e4j; _ga=GA1.1.1510788361.1715523165; use_np=use_np; readed_documents=7021938967.7023825588.7023440365; _ga_GFCL6FWBKV=GS1.1.1715523165.1.1.1715526874.51.0.0' \
|
||||
-H 'dnt: 1' \
|
||||
-H 'priority: u=0, i' \
|
||||
-H 'referer: https://www.fmkorea.com/index.php?mid=hotdeal&page=10' \
|
||||
-H 'sec-ch-ua: "Chromium";v="124", "Google Chrome";v="124", "Not-A.Brand";v="99"' \
|
||||
-H 'sec-ch-ua-mobile: ?0' \
|
||||
-H 'sec-ch-ua-platform: "macOS"' \
|
||||
-H 'sec-fetch-dest: document' \
|
||||
-H 'sec-fetch-mode: navigate' \
|
||||
-H 'sec-fetch-site: same-origin' \
|
||||
-H 'sec-fetch-user: ?1' \
|
||||
-H 'upgrade-insecure-requests: 1' \
|
||||
-H 'user-agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36'
|
|
@ -6,8 +6,20 @@ import lombok.Getter;
|
|||
@Getter
|
||||
@AllArgsConstructor
|
||||
public enum CrawlTarget {
|
||||
PPOMPPU_DOMESTIC,
|
||||
PPOMPPU_OVERSEA,
|
||||
FMKOREA,
|
||||
PPOMPPU_DOMESTIC("뽐뿌국내", true),
|
||||
PPOMPPU_OVERSEA("뽐뿌해외", true),
|
||||
FMKOREA("펨코", false),
|
||||
;
|
||||
|
||||
private final String alias;
|
||||
private final boolean available;
|
||||
|
||||
public static CrawlTarget fromAlias(String alias) {
|
||||
for (CrawlTarget crawlTarget : values()) {
|
||||
if (crawlTarget.getAlias().equals(alias)) {
|
||||
return crawlTarget;
|
||||
}
|
||||
}
|
||||
throw new IllegalArgumentException("Not found alias: " + alias);
|
||||
}
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue