Compare commits

...

No commits in common. "develop" and "master" have entirely different histories.

84 changed files with 3602 additions and 542 deletions

View File

@ -3,6 +3,7 @@ plugins {
id 'idea'
id 'org.springframework.boot' version '2.5.4'
id 'io.spring.dependency-management' version '1.0.11.RELEASE'
id 'com.google.cloud.tools.jib' version '3.4.0'
}
group = 'com.myoa.engineering.crawl.shopping'
@ -19,14 +20,16 @@ repositories {
mavenCentral()
}
allprojects {
subprojects {
group = 'com.myoa.engineering.crawl.shopping'
version = '1.1.1'
sourceCompatibility = JavaVersion.VERSION_17
apply plugin: 'java'
apply plugin: 'idea'
apply plugin: 'org.springframework.boot'
apply plugin: 'io.spring.dependency-management'
apply plugin: "com.google.cloud.tools.jib"
repositories {
mavenCentral()
@ -37,8 +40,11 @@ allprojects {
ext {
set('springCloudVersion', "2020.0.4")
set("BASE_IMAGE_REGISTRY_URL", "registry.myoa-universe.com")
}
apply from: "${project.rootDir}/jib.gradle"
dependencyManagement {
imports {
mavenBom "org.springframework.cloud:spring-cloud-dependencies:${springCloudVersion}"

View File

@ -4,3 +4,7 @@ distributionPath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-8.5-bin.zip
zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists
org.gradle.caching=true
org.gradle.parallel=true
org.gradle.parallel.threads=4
org.gradle.daemon=true

28
jib.gradle Normal file
View File

@ -0,0 +1,28 @@
jib.setAllowInsecureRegistries(true)
jib {
from {
image = "eclipse-temurin:17-jdk"
platforms {
platform {
architecture = "amd64"
os = "linux"
}
}
}
to {
image = "${BASE_IMAGE_REGISTRY_URL}/${project.name}"
tags = ["latest"]
}
container {
// entrypoint = ["INHERIT"]
creationTime = "USE_CURRENT_TIMESTAMP"
environment = [
"TZ": "Asia/Seoul",
]
jvmFlags = [
"--add-opens=java.base/java.time=ALL-UNNAMED"
]
}
}

View File

@ -1,70 +0,0 @@
package com.myoa.engineering.crawl.shopping.controller;
import java.util.List;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import com.myoa.engineering.crawl.shopping.processor.domain.PpomppuArticle;
import com.myoa.engineering.crawl.shopping.processor.dto.FeedParsedResult;
import com.myoa.engineering.crawl.shopping.processor.service.MessageSenderService;
import com.myoa.engineering.crawl.shopping.processor.service.PpomppuArticleService;
import com.myoa.engineering.crawl.shopping.processor.service.PpomppuFeedService;
import com.myoa.engineering.crawl.shopping.support.dto.APIResponse;
import com.myoa.engineering.crawl.shopping.support.dto.code.PpomppuBoardName;
import lombok.extern.slf4j.Slf4j;
import reactor.core.publisher.Mono;
/**
* CrawlAPIController
*
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-09-05
*/
@Slf4j
@RestController
@RequestMapping("/api/v1/crawl")
public class CrawlAPIController {
private final PpomppuFeedService ppomppuRSSFeedService;
private final PpomppuArticleService ppomppuArticleService;
private final MessageSenderService messageSenderService;
public CrawlAPIController(PpomppuFeedService ppomppuRSSFeedService,
PpomppuArticleService ppomppuArticleService,
MessageSenderService messageSenderService) {
this.ppomppuRSSFeedService = ppomppuRSSFeedService;
this.ppomppuArticleService = ppomppuArticleService;
this.messageSenderService = messageSenderService;
}
@PostMapping("/boards/{boardName}")
public Mono<APIResponse<FeedParsedResult>> crawlBoard(@PathVariable("boardName") PpomppuBoardName boardName) {
log.info("got request... {}", boardName);
FeedParsedResult result = FeedParsedResult.of(boardName);
Mono<String> publishedMessages =
ppomppuRSSFeedService.getArticles(boardName)
.map(e -> ppomppuArticleService.filterOnlyNewArticles(boardName, e))
.map(e -> ppomppuArticleService.save(boardName, e))
.filter(e -> !e.isEmpty())
.flatMap(e -> messageSenderService.sendBlockMessageToSlack(boardName, e));
return publishedMessages.then(Mono.just(APIResponse.success(result.done())));
}
@PostMapping("/exploit/boards/{boardName}")
public Mono<APIResponse<String>> crawlBoardDryRun(
@PathVariable("boardName") PpomppuBoardName boardName) {
log.info("got request... {}", boardName);
Mono<String> publishedMessages =
ppomppuRSSFeedService.getArticles(boardName)
.flatMap(e -> messageSenderService.sendBlockMessageToSlack(boardName, e));
return publishedMessages.map(APIResponse::success);
}
}

View File

@ -1,78 +0,0 @@
package com.myoa.engineering.crawl.shopping.dto;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import com.myoa.engineering.crawl.shopping.processor.domain.PpomppuArticle;
import com.myoa.engineering.crawl.shopping.support.dto.code.PpomppuBoardName;
/**
* PpomppuArticleTransformer
*
* @author Shin Woo-jin (woozu.shin@kakaoent.com)
* @since 2021-09-08
*/
public final class PpomppuArticleParser {
private PpomppuArticleParser() {}
public static PpomppuArticle toArticle(Elements articleElement) {
final String articleIdString = PpomppuArticleParser.parseArticleId(articleElement.get(0));
final String title = PpomppuArticleParser.parseTitle(articleElement.get(2));
final String articleUrl = PpomppuArticleParser.parseArticleUrl(articleElement.get(2));
final String thumbnailUrl = PpomppuArticleParser.parseThumbnailUrl(articleElement.get(3));
final Integer recommended = PpomppuArticleParser.parseRecommended(articleElement.get(6));
final String hitString = PpomppuArticleParser.parseHit(articleElement.get(7));
final String registeredAtString = PpomppuArticleParser.parseRegisteredAt(articleElement.get(5));
return PpomppuArticleParseDTO.builder()
.articleId(articleIdString)
.title(title)
.articleUrl(articleUrl)
.thumbnailUrl(thumbnailUrl)
.recommended(recommended)
.hit(hitString)
.registeredAt(registeredAtString)
.build()
.convert();
}
public static String parseArticleId(Element td) {
return td.text().trim();
}
public static String parseTitle(Element td) {
return td.getElementsByTag("a").text();
}
public static String parseArticleUrl(Element td) {
return PpomppuBoardName.ofViewPageUrl(td.getElementsByTag("a").attr("href"));
}
public static String parseThumbnailUrl(Element td) {
return "https:" + td.getElementsByTag("img").get(0).attr("src");
}
public static Integer parseRecommended(Element td) {
final String voteString = td.text();
final int recommended;
if (voteString.isEmpty()) {
recommended = 0;
} else {
final int voteUp = Integer.parseInt(td.text().split(" - ")[0]);
final int voteDown = Integer.parseInt(td.text().split(" - ")[1]);
recommended = voteUp - voteDown;
}
return recommended;
}
public static String parseHit(Element td) {
return td.text();
}
public static String parseRegisteredAt(Element td) {
return td.attr("title");
}
}

View File

@ -1,38 +0,0 @@
package com.myoa.engineering.crawl.shopping.receiver.scheduler;
import java.util.Arrays;
import org.springframework.scheduling.annotation.EnableScheduling;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import com.myoa.engineering.crawl.shopping.receiver.service.ProcessorAPIService;
import com.myoa.engineering.crawl.shopping.support.dto.code.PpomppuBoardName;
import lombok.extern.slf4j.Slf4j;
/**
* ParseEventEmitter
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-09-05
*
*/
@Slf4j
@Component
@EnableScheduling
public class ParseEventEmitter {
private final ProcessorAPIService processorAPIService;
public ParseEventEmitter(ProcessorAPIService processorAPIService) {
this.processorAPIService = processorAPIService;
}
@Scheduled(fixedRate = 600 * 1000L)
public void emitBoards() {
log.info("[emitDomesticBoard] trigger fired!");
Arrays.stream(PpomppuBoardName.values())
.filter(PpomppuBoardName::isCrawlWithDefaultTimer)
.forEach(boardName -> processorAPIService.emitParseEvent(boardName).block());
}
}

View File

@ -1,6 +0,0 @@
spring:
config:
activate:
on-profile: production
import:
- "configserver:http://ppn-config-server:20080"

View File

@ -1,25 +0,0 @@
spring:
application:
name: ppn-receiver
main:
allow-bean-definition-overriding: true
profiles:
active: ${SPRING_ACTIVE_PROFILE:local}
group:
local: "local,webclient-local"
development: "development,webclient-development"
production: "production,webclient-production"
freemarker:
enabled: false
server:
port: 20080
error:
whitelabel:
enabled: false
management:
endpoints:
web:
exposure:
include: refresh,health

View File

@ -1,62 +0,0 @@
package com.myoa.engineering.crawl.shopping.controller;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import com.myoa.engineering.crawl.shopping.sender.dto.SlackBaseMessageBlock;
import com.myoa.engineering.crawl.shopping.sender.dto.SlackMessageDTO;
import com.myoa.engineering.crawl.shopping.sender.infrastructure.client.MongeShoppingBotSlackMessageSender;
import com.myoa.engineering.crawl.shopping.support.dto.APIResponse;
import com.myoa.engineering.crawl.shopping.support.dto.BlockMessageDTO;
import com.myoa.engineering.crawl.shopping.support.dto.SimpleMessageDTO;
import com.myoa.engineering.crawl.shopping.support.util.ObjectMapperFactory;
import lombok.extern.slf4j.Slf4j;
import reactor.core.publisher.Mono;
/**
* MessageSenderAPIController
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-11-21
*
*/
@Slf4j
@RestController
@RequestMapping("/api/v1")
public class MessageSenderAPIController {
private final MongeShoppingBotSlackMessageSender sender;
public MessageSenderAPIController(MongeShoppingBotSlackMessageSender sender) {
this.sender = sender;
}
@PostMapping("/messages/sendSimpleMessage/messengers/slack")
public Mono<APIResponse<SimpleMessageDTO>> sendSimpleMessageToSlack(@RequestBody SimpleMessageDTO dto) {
return sender.sendMessage(sender.ofMessage(dto.getBody()))
.then(Mono.just(APIResponse.success(dto)));
}
@PostMapping("/messages/sendBlockMessage/messengers/slack")
public Mono<APIResponse<BlockMessageDTO>> sendBlockMessageToSlack(@RequestBody BlockMessageDTO dto) {
if (dto.getBlocks().isEmpty()) {
return Mono.just(APIResponse.fail(dto, "empty blocks"));
}
return sender.sendMessage(buildSlackMessageDTO(dto))
// .doOnNext(e -> log.info("[sendBlockMessageToSlack] slackMessageDTO: {}",
// ObjectMapperFactory.writeAsString(buildSlackMessageDTO(dto))))
.then(Mono.just(APIResponse.success(dto)));
}
private SlackMessageDTO buildSlackMessageDTO(BlockMessageDTO dto) {
SlackMessageDTO slackMessageDTO = sender.ofBlockMessageBased();
slackMessageDTO.addSectionBlock(dto.getTitle());
dto.getBlocks().forEach(slackMessageDTO::addSectionBlock);
slackMessageDTO.addBlock(SlackBaseMessageBlock.ofDivider());
return slackMessageDTO;
}
}

View File

@ -1,25 +0,0 @@
spring:
application:
name: ppn-sender
main:
allow-bean-definition-overriding: true
profiles:
active: ${SPRING_ACTIVE_PROFILE:local}
group:
local: "local,slackapi-local,webclient-local"
development: "development,slackapi-development,webclient-development"
production: "production,slackapi-production,webclient-production"
freemarker:
enabled: false
server:
port: 20080
error:
whitelabel:
enabled: false
management:
endpoints:
web:
exposure:
include: refresh,health

View File

@ -9,6 +9,8 @@ dependencies {
implementation("org.springframework.boot:spring-boot-starter-web") {
exclude group: "org.springframework.boot", module: "spring-boot-starter-tomcat"
}
implementation("org.springframework.boot:spring-boot-starter-webflux")
implementation("org.springframework.boot:spring-boot-starter-undertow") {
exclude group: "io.undertow", module: "undertow-websockets-jsr"
@ -22,6 +24,15 @@ dependencies {
implementation "org.springframework.cloud:spring-cloud-starter-openfeign"
implementation "io.github.openfeign:feign-hc5"
implementation 'org.ahocorasick:ahocorasick:0.6.3'
implementation "com.slack.api:slack-api-client:1.39.1"
// implementation "io.github.resilience4j:resilience4j-spring-boot3:2.2.0"
implementation 'io.github.resilience4j:resilience4j-all:2.2.0'
implementation "io.github.resilience4j:resilience4j-feign:2.2.0"
implementation "org.springframework.cloud:spring-cloud-starter-circuitbreaker-resilience4j"
// implementation 'io.github.openfeign:feign-okhttp:13.1'
implementation 'io.github.openfeign:feign-jackson:13.2'
annotationProcessor 'org.springframework.boot:spring-boot-configuration-processor'
annotationProcessor 'org.projectlombok:lombok'

View File

@ -12,7 +12,7 @@ import org.springframework.context.event.EventListener;
import java.sql.SQLException;
@Slf4j
@Profile({"datasource-local", "datasource-development"})
@Profile("!prod")
@Configuration
public class H2ConsoleConfiguration {
@ -24,7 +24,7 @@ public class H2ConsoleConfiguration {
@EventListener(ContextRefreshedEvent.class)
public void start() throws SQLException {
log.info("starting h2 console");
this.webServer = Server.createWebServer("-webPort", port, "-tcpAllowOthers").start();
this.webServer = Server.createWebServer("-webPort", port, "-tcpAllowOthers", "-webAllowOthers").start();
}
@EventListener(ContextClosedEvent.class)

View File

@ -0,0 +1,19 @@
package com.myoa.engineering.crawl.shopping.configuration.feign;
import feign.RequestInterceptor;
import feign.RequestTemplate;
import org.apache.http.HttpHeaders;
public class FakeUserAgentInterceptor implements RequestInterceptor {
private static final String USER_AGENT_VALUE = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36";
@Override
public void apply(RequestTemplate template) {
template.header(HttpHeaders.USER_AGENT, USER_AGENT_VALUE);
template.header(HttpHeaders.ACCEPT, "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8");
template.header(HttpHeaders.ACCEPT_LANGUAGE, "ko-KR,ko;q=0.9,en-US;q=0.8,en;q=0.7");
template.header(HttpHeaders.USER_AGENT, USER_AGENT_VALUE);
template.header(HttpHeaders.USER_AGENT, USER_AGENT_VALUE);
}
}

View File

@ -1,8 +1,7 @@
package com.myoa.engineering.crawl.shopping.configuration;
package com.myoa.engineering.crawl.shopping.configuration.feign;
import feign.Logger;
import feign.RequestInterceptor;
import feign.codec.ErrorDecoder;
import lombok.extern.slf4j.Slf4j;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;

View File

@ -0,0 +1,40 @@
package com.myoa.engineering.crawl.shopping.configuration.feign;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.myoa.engineering.crawl.shopping.dto.ExceptionMessage;
import feign.Response;
import feign.codec.ErrorDecoder;
import io.undertow.util.BadRequestException;
import javassist.NotFoundException;
import lombok.extern.slf4j.Slf4j;
import java.io.IOException;
import java.io.InputStream;
@Slf4j
public class FmkoreaClientErrorDecoder implements ErrorDecoder {
private final ErrorDecoder errorDecoder = new Default();
@Override
public Exception decode(String methodsKey, Response response) {
ExceptionMessage message = null;
try (InputStream bodyIs = response.body()
.asInputStream()) {
ObjectMapper mapper = new ObjectMapper();
message = mapper.readValue(bodyIs, ExceptionMessage.class);
} catch (IOException e) {
return new Exception(e.getMessage());
}
switch (response.status()) {
case 400:
return new BadRequestException(message.getMessage() != null ? message.getMessage() : "Bad Request");
case 404:
return new NotFoundException(message.getMessage() != null ? message.getMessage() : "Not found");
default:
return errorDecoder.decode(methodsKey, response);
}
}
}

View File

@ -0,0 +1,52 @@
package com.myoa.engineering.crawl.shopping.configuration.resilience;
import io.github.resilience4j.circuitbreaker.CircuitBreaker;
import io.github.resilience4j.circuitbreaker.CircuitBreakerConfig;
import io.github.resilience4j.circuitbreaker.CircuitBreakerRegistry;
import io.github.resilience4j.core.RegistryStore;
import io.github.resilience4j.core.registry.InMemoryRegistryStore;
import io.github.resilience4j.ratelimiter.RateLimiter;
import io.github.resilience4j.ratelimiter.RateLimiterConfig;
import io.github.resilience4j.ratelimiter.RateLimiterRegistry;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import java.time.Duration;
@Configuration
public class RateLimitConfiguration {
@Bean
public RateLimiterRegistry rateLimiterRegistry() {
RegistryStore<RateLimiter> stores = new InMemoryRegistryStore<>();
// TODO 개별 config 에서 등록하도록 변경
RateLimiterConfig rateLimiterConfig = RateLimiterConfig.custom()
.limitRefreshPeriod(Duration.ofMillis(500)) // 0.5 seconds
.limitForPeriod(1) // number of permits in a refresh period
.build();
stores.putIfAbsent("fmkoreaAvoid429", RateLimiter.of("fmkoreaAvoid429", rateLimiterConfig));
return RateLimiterRegistry.custom()
.withRateLimiterConfig(RateLimiterConfig.ofDefaults())
.withRegistryStore(stores)
.build();
}
@Bean
public CircuitBreakerRegistry circuitBreakerRegistry() {
RegistryStore<CircuitBreaker> stores = new InMemoryRegistryStore<>();
CircuitBreakerConfig circuitBreakerConfig = CircuitBreakerConfig.custom()
.slidingWindowSize(1)
.build();
stores.putIfAbsent("fmkoreaAvoid429", CircuitBreaker.of("fmkoreaAvoid429", circuitBreakerConfig));
return CircuitBreakerRegistry.custom()
.withCircuitBreakerConfig(CircuitBreakerConfig.ofDefaults())
.withRegistryStore(stores)
.build();
}
}

View File

@ -0,0 +1,20 @@
package com.myoa.engineering.crawl.shopping.configuration.slack;
import com.myoa.engineering.crawl.shopping.configuration.slack.properties.SlackSecretProperties;
import com.slack.api.Slack;
import com.slack.api.methods.MethodsClient;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@Configuration
public class SlackConfiguration {
private static final String DEFAULT_BOT_UNIT_NAME = "shopping-crawler";
@Bean
public MethodsClient slackMethodsClient(SlackSecretProperties slackSecretProperties) {
String token = slackSecretProperties.find(DEFAULT_BOT_UNIT_NAME).getToken();
return Slack.getInstance().methods(token);
}
}

View File

@ -0,0 +1,28 @@
package com.myoa.engineering.crawl.shopping.configuration.web;
import org.springframework.context.annotation.Configuration;
import org.springframework.http.MediaType;
import org.springframework.http.converter.HttpMessageConverter;
import org.springframework.web.servlet.config.annotation.WebMvcConfigurer;
import java.nio.charset.StandardCharsets;
import java.util.List;
@Configuration
public class WebConfiguration implements WebMvcConfigurer {
/*
@Override
public void configureMessageConverters(List<HttpMessageConverter<?>> converters) {
WebMvcConfigurer.super.configureMessageConverters(converters);
addJsonFormUrlEncodedConverter(converters);
}
private void addJsonFormUrlEncodedConverter(List<HttpMessageConverter<?>> converters) {
JsonFormUrlEncodedConverter<?> converter = new JsonFormUrlEncodedConverter<>();
MediaType mediaType = new MediaType(MediaType.APPLICATION_FORM_URLENCODED, StandardCharsets.UTF_8);
converter.setSupportedMediaTypes(List.of(mediaType));
converters.add(converter);
}
*/
}

View File

@ -0,0 +1,65 @@
package com.myoa.engineering.crawl.shopping.controller;
import com.myoa.engineering.crawl.shopping.dto.slack.v2.SlackEventSubscriptionDTO;
import com.myoa.engineering.crawl.shopping.dto.slack.v2.SlackSlashCommandDTO;
import com.myoa.engineering.crawl.shopping.service.slack.RegisterCommandHandler;
import com.myoa.engineering.crawl.shopping.service.slack.ShoppingCommandHandler;
import lombok.extern.slf4j.Slf4j;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import java.util.List;
@Slf4j
@RestController
@RequestMapping("/api/v1/slack")
public class SlackEventSubscriptionAPIController {
private final ShoppingCommandHandler shoppingCommandHandler;
private final RegisterCommandHandler registerCommandHandler;
public SlackEventSubscriptionAPIController(ShoppingCommandHandler shoppingCommandHandler,
RegisterCommandHandler registerCommandHandler) {
this.shoppingCommandHandler = shoppingCommandHandler;
this.registerCommandHandler = registerCommandHandler;
}
@PostMapping(value = {"/subscriptions"})
private ResponseEntity<String> slackEventSubscription(@RequestBody SlackEventSubscriptionDTO requestDTO) {
log.info("requestDTO: {}", requestDTO);
return ResponseEntity.ok(requestDTO.getChallenge());
}
@PostMapping(value = {"/commands/help"}, consumes = {"application/x-www-form-urlencoded"})
private ResponseEntity<String> handleCommandHelp(SlackSlashCommandDTO requestDTO) {
log.info("requestDTO: {}", requestDTO);
return ResponseEntity.ok("""
/쇼핑 목록
/쇼핑 키워드 추가 (쇼핑몰명) (키워드명)
/쇼핑 키워드 삭제 (쇼핑몰명) (키워드명)
/쇼핑 키워드 목록
/등록 (사용자명)
""");
}
@PostMapping(value = {"/commands/shopping"}, consumes = {"application/x-www-form-urlencoded"})
private ResponseEntity<String> handleAddSubscription(SlackSlashCommandDTO requestDTO) {
log.info("requestDTO: {}", requestDTO);
List<String> results = shoppingCommandHandler.handle(requestDTO);
return ResponseEntity.ok(String.join("\n", results));
}
@PostMapping(value = {"/commands/register"}, consumes = {"application/x-www-form-urlencoded"})
private ResponseEntity<String> handleRegisterUser(SlackSlashCommandDTO requestDTO) {
log.info("requestDTO: {}", requestDTO);
List<String> results = registerCommandHandler.handle(requestDTO);
return ResponseEntity.ok(String.join("\n", results));
}
}

View File

@ -1,22 +1,64 @@
package com.myoa.engineering.crawl.shopping.controller;
import com.myoa.engineering.crawl.shopping.crawlhandler.PpomppuCrawlDomesticHandler;
import com.myoa.engineering.crawl.shopping.crawlhandler.CrawlHandler;
import com.myoa.engineering.crawl.shopping.infra.client.fmkorea.FmkoreaBoardClientV2;
import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget;
import com.slack.api.methods.MethodsClient;
import com.slack.api.methods.SlackApiException;
import lombok.extern.slf4j.Slf4j;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import java.io.IOException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@Slf4j
@RestController
@RequestMapping("/api/v1/exploit")
public class TestAPIController {
private final PpomppuCrawlDomesticHandler ppomppuCrawlDomesticHandler;
private final MethodsClient methodsClient;
private final List<CrawlHandler> crawlHandlers;
private final FmkoreaBoardClientV2 fmkoreaBoardClientV2;
public TestAPIController(PpomppuCrawlDomesticHandler ppomppuCrawlDomesticHandler) {
this.ppomppuCrawlDomesticHandler = ppomppuCrawlDomesticHandler;
public TestAPIController(MethodsClient methodsClient,
List<CrawlHandler> crawlHandlers,
FmkoreaBoardClientV2 fmkoreaBoardClientV2) {
this.methodsClient = methodsClient;
this.crawlHandlers = crawlHandlers;
this.fmkoreaBoardClientV2 = fmkoreaBoardClientV2;
}
@GetMapping("/triggers")
public void triggerExploit(@RequestParam("crawlTarget") CrawlTarget crawlTarget) {
crawlHandlers
.stream().filter(e -> e.getCrawlTarget().equals(crawlTarget))
.forEach(CrawlHandler::handle);
}
@GetMapping("/ratelimiter")
public void triggerExploit() {
ppomppuCrawlDomesticHandler.handle();
log.info("will be called page 1");
fmkoreaBoardClientV2.getBoardHtml(1, null);
log.info("called page 1");
}
private Map<String, String> generateRequestParams(int pageId) {
Map<String, String> params = new HashMap<>();
params.put("mid", "hotdeal");
params.put("page", String.valueOf(pageId));
return params;
}
@GetMapping("/test-message")
public void testMessage() throws SlackApiException, IOException {
methodsClient.chatPostMessage(req -> req
.channel("notify_shopping")
.text("Hello, World!"));
}
}

View File

@ -0,0 +1,22 @@
package com.myoa.engineering.crawl.shopping.crawlhandler;
import com.myoa.engineering.crawl.shopping.infra.client.fmkorea.FmkoreaBoardClientV2;
import org.springframework.cache.annotation.Cacheable;
import org.springframework.stereotype.Service;
@Service
public class FmkoreaCookieService {
private final FmkoreaBoardClientV2 fmkoreaBoardClientV2;
public FmkoreaCookieService(FmkoreaBoardClientV2 fmkoreaBoardClientV2) {
this.fmkoreaBoardClientV2 = fmkoreaBoardClientV2;
}
@Cacheable(cacheNames = "crawltarget.fmkorea", key = "#root.methodName")
public String getCookie() {
String fakeHtml = fmkoreaBoardClientV2.getBoardHtml(1, null);
return FmkoreaFake430Resolver.resolveFake430(fakeHtml);
}
}

View File

@ -1,12 +1,33 @@
package com.myoa.engineering.crawl.shopping.crawlhandler;
import com.myoa.engineering.crawl.shopping.crawlhandler.parser.FmkoreaArticleParser;
import com.myoa.engineering.crawl.shopping.domain.entity.v2.Article;
import com.myoa.engineering.crawl.shopping.infra.client.fmkorea.FmkoreaBoardClientV2;
import com.myoa.engineering.crawl.shopping.service.ArticleCommandService;
import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import java.util.List;
import java.util.stream.Stream;
@Slf4j
@Component
public class FmkoreaCrawlHandler implements CrawlHandler {
private final FmkoreaBoardClientV2 fmkoreaBoardClientV2;
private final FmkoreaArticleParser fmkoreaArticleParser;
private final ArticleCommandService articleCommandService;
private final FmkoreaCookieService fmkoreaCookieService;
public FmkoreaCrawlHandler(FmkoreaBoardClientV2 fmkoreaBoardClientV2,
FmkoreaArticleParser fmkoreaArticleParser, ArticleCommandService articleCommandService, FmkoreaCookieService fmkoreaCookieService) {
this.fmkoreaBoardClientV2 = fmkoreaBoardClientV2;
this.fmkoreaArticleParser = fmkoreaArticleParser;
this.articleCommandService = articleCommandService;
this.fmkoreaCookieService = fmkoreaCookieService;
}
@Override
public CrawlTarget getCrawlTarget() {
return CrawlTarget.FMKOREA;
@ -14,5 +35,20 @@ public class FmkoreaCrawlHandler implements CrawlHandler {
@Override
public void handle() {
String cookie = fmkoreaCookieService.getCookie();
String boardHtmlPage1 = fmkoreaBoardClientV2.getBoardHtml(1, cookie);
List<Article> parsedPage1 = fmkoreaArticleParser.parse(boardHtmlPage1);
String boardHtmlPage2 = fmkoreaBoardClientV2.getBoardHtml(2, cookie);
List<Article> parsedPage2 = fmkoreaArticleParser.parse(boardHtmlPage2);
List<Article> merged = Stream.of(parsedPage1, parsedPage2)
.flatMap(List::stream)
.map(e -> e.updateCrawlTarget(getCrawlTarget()))
.toList();
articleCommandService.upsert(merged);
}
}

View File

@ -0,0 +1,88 @@
package com.myoa.engineering.crawl.shopping.crawlhandler;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.type.CollectionType;
import com.myoa.engineering.crawl.shopping.util.ObjectMapperFactory;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.Base64;
import java.util.List;
import java.util.Locale;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public final class FmkoreaFake430Resolver {
private static final Pattern PATTERN_DOCUMENT = Pattern.compile("var .+?\\s*=\\s*(\\[.+?\\];)");
private static final Pattern PATTERN_COOKIE = Pattern.compile("escape\\('(.+?)'\\)");
private static final ObjectMapper MAPPER = ObjectMapperFactory.DEFAULT_MAPPER;
private static final CollectionType COLLECTION_TYPE = MAPPER.getTypeFactory().constructCollectionType(List.class, String.class);
private static final DateTimeFormatter DATE_TIME_FORMATTER_COOKIE_LIFE = DateTimeFormatter.ofPattern("EEE, dd MMM yyyy HH:mm:ss 'GMT'", Locale.ENGLISH);
private FmkoreaFake430Resolver() {
}
public static String resolveFake430(String fakeHtml) {
Document parse = Jsoup.parse(fakeHtml);
String javascript = parse.select("script").html();
String cookieHtml = extractEncodedCookieHtml(javascript);
try {
List<String> chunks = MAPPER.readValue(cookieHtml, COLLECTION_TYPE);
String decodedhtml = decodeHtmlChunks(chunks);
String cookieHexValue = extractCookieHexValue(decodedhtml);
return generateLiteTimeCookie(cookieHexValue);
} catch (Exception e) {
return null;
}
}
private static String extractEncodedCookieHtml(String javascript) {
final Matcher matcher = PATTERN_DOCUMENT.matcher(javascript);
if (matcher.find()) {
return matcher.group(1);
}
return "";
}
private static String decodeHtmlChunks(List<String> chunks) {
return chunks.stream()
.map(e -> new String(Base64.getDecoder().decode(e.substring(3, e.length() - 3))))
.map(e -> (char) (e.charAt(0) - 3 + 256) % 256)
.collect(StringBuilder::new, StringBuilder::appendCodePoint, StringBuilder::append).toString();
}
private static String extractCookieHexValue(String decodedHtml) {
final Matcher matcher = PATTERN_COOKIE.matcher(decodedHtml);
if (matcher.find()) {
return matcher.group(1);
}
return "";
}
private static String escape(String input) {
StringBuilder result = new StringBuilder();
for (char ch : input.toCharArray()) {
if (Character.isLetterOrDigit(ch) || ch == '-' || ch == '_' || ch == '.' || ch == '~') {
result.append(ch);
} else {
result.append(String.format("%%%02X", (int) ch));
}
}
return result.toString();
}
public static String generateLiteTimeCookie(String cookieHexValue) {
LocalDateTime ldt = LocalDateTime.now().plusDays(1L);
String cookie = "lite_year=" + escape(cookieHexValue) +
"; expires=" + ldt.format(DATE_TIME_FORMATTER_COOKIE_LIFE) + "; path=/";
return cookie;
}
}

View File

@ -0,0 +1,58 @@
package com.myoa.engineering.crawl.shopping.crawlhandler;
import com.myoa.engineering.crawl.shopping.crawlhandler.parser.PpomppuArticleParserV2;
import com.myoa.engineering.crawl.shopping.domain.entity.v2.Article;
import com.myoa.engineering.crawl.shopping.infra.client.ppomppu.PpomppuBoardClientV2;
import com.myoa.engineering.crawl.shopping.service.ArticleCommandService;
import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget;
import org.springframework.stereotype.Component;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Stream;
@Component
public class PpomppuCrawlOverseaHandler implements CrawlHandler {
private final PpomppuBoardClientV2 ppomppuBoardClient;
private final PpomppuArticleParserV2 ppomppuArticleParserV2;
private final ArticleCommandService articleCommandService;
public PpomppuCrawlOverseaHandler(PpomppuBoardClientV2 ppomppuBoardClient,
PpomppuArticleParserV2 ppomppuArticleParserV2,
ArticleCommandService articleCommandService) {
this.ppomppuBoardClient = ppomppuBoardClient;
this.ppomppuArticleParserV2 = ppomppuArticleParserV2;
this.articleCommandService = articleCommandService;
}
@Override
public CrawlTarget getCrawlTarget() {
return CrawlTarget.PPOMPPU_OVERSEA;
}
@Override
public void handle() {
String boardHtmlPage1 = ppomppuBoardClient.getBoardHtml("/zboard/zboard.php", generateRequestParams(1));
List<Article> parsedPage1 = ppomppuArticleParserV2.parse(boardHtmlPage1);
String boardHtmlPage2 = ppomppuBoardClient.getBoardHtml("/zboard/zboard.php", generateRequestParams(2));
List<Article> parsedPage2 = ppomppuArticleParserV2.parse(boardHtmlPage2);
List<Article> merged = Stream.of(parsedPage1, parsedPage2)
.flatMap(List::stream)
.map(e -> e.updateCrawlTarget(getCrawlTarget()))
.toList();
articleCommandService.upsert(merged);
}
private Map<String, String> generateRequestParams(int pageId) {
Map<String, String> params = new HashMap<>();
params.put("id", "ppomppu4");
params.put("page", String.valueOf(pageId));
return params;
}
}

View File

@ -0,0 +1,68 @@
package com.myoa.engineering.crawl.shopping.crawlhandler.parser;
import com.myoa.engineering.crawl.shopping.domain.entity.v2.Article;
import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget;
import com.myoa.engineering.crawl.shopping.util.DateTimeUtils;
import lombok.extern.slf4j.Slf4j;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.stereotype.Component;
import java.time.ZonedDateTime;
import java.util.List;
@Slf4j
@Component
public class FmkoreaArticleParser {
private static final String FMKOREA_URL = "https://www.fmkorea.com";
public List<Article> parse(String html) {
Elements liElements = converHtmlToTrElements(html);
return liElements.stream()
// .filter(this::isRealArticle)
.map(this::parse)
.toList();
}
private Elements converHtmlToTrElements(String html) {
Document document = Jsoup.parse(html);
Element liTable = document.getElementsByClass("fm_best_widget").first();
return liTable.select("li.li_best2_pop0");
}
private Article parse(Element item) {
String link = item.select("h3.title a").attr("href");
Long articleId = Long.parseLong(link.replace("/", ""));
// https://www.fmkorea.com/7023440365
String articleUrl = FMKOREA_URL + link;
String boardName = item.select("span.category a").text().trim();
String title = item.select("h3.title a").text();
String itemPrice = item.select("div.hotdeal_info span:contains(가격) a").text();
String deliveryPrice = item.select("div.hotdeal_info span:contains(배송) a").text();
title = title + " 가격: " + itemPrice + " 배송: " + deliveryPrice;
String registeredAtString = item.select("span.regdate").text().trim();
ZonedDateTime registeredAt = DateTimeUtils.parse(registeredAtString, DateTimeUtils.FORMATTER_HHMM, DateTimeUtils.FORMATTER_YYMMDD_DOT);
Element recommendationElement = item.selectFirst("a.pc_voted_count");
Integer recommended = null;
if (recommendationElement != null) {
recommended = Integer.parseInt(recommendationElement.selectFirst("span.count").text());
}
return Article.builder()
.articleId(articleId)
.title(title)
.boardName(boardName)
.articleUrl(articleUrl)
.recommended(recommended)
.registeredAt(registeredAt)
.build();
}
}

View File

@ -21,14 +21,11 @@ import java.util.regex.Pattern;
@Slf4j
@Component
public final class PpomppuArticleParserV2 {
public class PpomppuArticleParserV2 {
private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yy.MM.dd HH:mm:ss")
.withZone(ZoneId.of("Asia/Seoul"));
private PpomppuArticleParserV2() {
}
public List<Article> parse(String html) {
Elements trElements = converHtmlToTrElements(html);
return trElements.stream()
@ -38,8 +35,8 @@ public final class PpomppuArticleParserV2 {
}
private Elements converHtmlToTrElements(String data) {
Document document = Jsoup.parse(data);
private Elements converHtmlToTrElements(String html) {
Document document = Jsoup.parse(html);
Elements trList = document.getElementById("revolution_main_table").getElementsByTag("tr");
return trList;
}
@ -72,7 +69,7 @@ public final class PpomppuArticleParserV2 {
String boardName = parseBoardName(title);
Integer recommended = parseRecommended(tdList.get(4));
Integer hit = NumberUtils.parseInt(tdList.get(5).text(), 0);
ZonedDateTime registeredAt = DateTimeUtils.parse(tdList.get(3).text());
ZonedDateTime registeredAt = DateTimeUtils.parse(tdList.get(3).text(), DateTimeUtils.FORMATTER_HHMMss, DateTimeUtils.FORMATTER_YYMMDD_SLASH);
return Article.builder()
.articleId(articleId)

View File

@ -12,7 +12,7 @@ import lombok.NoArgsConstructor;
@NoArgsConstructor
@AllArgsConstructor
@Entity
@Table
@Table(name = "app_user")
public class AppUser extends Auditable {
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)

View File

@ -15,7 +15,7 @@ import java.time.ZonedDateTime;
@NoArgsConstructor
@AllArgsConstructor
@Entity
@Table
@Table(name = "article")
public class Article extends Auditable {
@Id

View File

@ -13,7 +13,7 @@ import lombok.NoArgsConstructor;
@NoArgsConstructor
@AllArgsConstructor
@Entity
@Table
@Table(name = "subscribed_keyword")
public class SubscribedKeyword extends Auditable {
@Id

View File

@ -1,9 +1,11 @@
package com.myoa.engineering.crawl.shopping.domain.model;
import com.myoa.engineering.crawl.shopping.domain.model.v2.ArticleModel;
import com.slack.api.methods.response.chat.ChatPostMessageResponse;
import lombok.*;
import java.util.List;
import java.util.stream.Collectors;
@ToString
@ -12,13 +14,20 @@ import java.util.List;
@NoArgsConstructor
@AllArgsConstructor
public class UserNotifyModel {
private String userId;
private String slackId;
private List<ArticleModel> articles;
private ChatPostMessageResponse chatPostMessageResponse;
public static UserNotifyModel of(String userId, List<ArticleModel> articles) {
return UserNotifyModel.builder()
.userId(userId)
.articles(articles)
.build();
public String toCompositedMessage() {
return wrapUserId() + "\n" +
articles.stream()
.map(ArticleModel::convertArticletoMessage)
.collect(Collectors.joining("\n"));
}
private String wrapUserId() {
return "<@" + slackId + ">";
}
}

View File

@ -22,4 +22,7 @@ public class ArticleModel {
private Integer recommended;
private ZonedDateTime registeredAt;
public String convertArticletoMessage() {
return "• <" + this.getArticleUrl() + "|" + this.getTitle() + ">";
}
}

View File

@ -0,0 +1,19 @@
package com.myoa.engineering.crawl.shopping.dto;
import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
@Getter
@Setter
@NoArgsConstructor
@AllArgsConstructor
public class ExceptionMessage {
private String timestamp;
private int status;
private String error;
private String message;
private String path;
}

View File

@ -1,4 +1,4 @@
package com.myoa.engineering.crawl.shopping.dto;
package com.myoa.engineering.crawl.shopping.dto.feed.v1;
import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName;
import lombok.Builder;

View File

@ -1,4 +1,4 @@
package com.myoa.engineering.crawl.shopping.dto;
package com.myoa.engineering.crawl.shopping.dto.feed.v1;
import com.myoa.engineering.crawl.shopping.domain.entity.v1.PpomppuArticle;
import com.myoa.engineering.crawl.shopping.support.util.DateUtil;

View File

@ -1,4 +1,4 @@
package com.myoa.engineering.crawl.shopping.dto;
package com.myoa.engineering.crawl.shopping.dto.feed.v1;
import com.myoa.engineering.crawl.shopping.domain.entity.v1.PpomppuArticle;
import com.myoa.engineering.crawl.shopping.support.dto.BlockMessageDTO;

View File

@ -0,0 +1,20 @@
package com.myoa.engineering.crawl.shopping.dto.feed.v1;
import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget;
import lombok.*;
@Getter
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class SubscribedKeywordDTO {
private String userId;
private CrawlTarget crawlTarget;
private String keyword;
public String toMessage() {
return crawlTarget.getAlias() + " | " + keyword;
}
}

View File

@ -1,4 +1,4 @@
package com.myoa.engineering.crawl.shopping.dto.slack;
package com.myoa.engineering.crawl.shopping.dto.slack.v1;
import java.io.Serializable;

View File

@ -1,4 +1,4 @@
package com.myoa.engineering.crawl.shopping.dto.slack;
package com.myoa.engineering.crawl.shopping.dto.slack.v1;
import com.fasterxml.jackson.annotation.JsonInclude;

View File

@ -1,4 +1,4 @@
package com.myoa.engineering.crawl.shopping.dto.slack;
package com.myoa.engineering.crawl.shopping.dto.slack.v1;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;

View File

@ -1,4 +1,4 @@
package com.myoa.engineering.crawl.shopping.dto.slack;
package com.myoa.engineering.crawl.shopping.dto.slack.v1;
import java.io.Serializable;

View File

@ -1,4 +1,4 @@
package com.myoa.engineering.crawl.shopping.dto.slack;
package com.myoa.engineering.crawl.shopping.dto.slack.v1;
import lombok.AllArgsConstructor;
import lombok.Getter;

View File

@ -1,4 +1,4 @@
package com.myoa.engineering.crawl.shopping.dto.slack;
package com.myoa.engineering.crawl.shopping.dto.slack.v1;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;

View File

@ -1,4 +1,4 @@
package com.myoa.engineering.crawl.shopping.dto.slack;
package com.myoa.engineering.crawl.shopping.dto.slack.v1;
import com.fasterxml.jackson.annotation.JsonInclude;
import lombok.Builder;

View File

@ -0,0 +1,35 @@
package com.myoa.engineering.crawl.shopping.dto.slack.v2;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.ToString;
import java.util.Map;
@ToString
@Getter
@NoArgsConstructor
@AllArgsConstructor
public class SlackEventSubscriptionDTO {
private String token;
private String challenge;
private String type;
@JsonProperty("team_id")
private String teamId;
@JsonProperty("api_app_id")
private String apiAppId;
private Map<String, Object> event;
@JsonProperty("event_id")
private String eventId;
@JsonProperty("event_time")
private long eventTime;
}

View File

@ -0,0 +1,22 @@
package com.myoa.engineering.crawl.shopping.dto.slack.v2;
import lombok.*;
@ToString
@Getter
@Setter
@NoArgsConstructor
@AllArgsConstructor
public class SlackSlashCommandDTO {
private String token;
private String command;
private String text;
private String response_url;
private String trigger_id;
private String user_id;
private String user_name;
private String team_id;
private String api_app_id;
}

View File

@ -7,14 +7,17 @@ import com.myoa.engineering.crawl.shopping.domain.model.v2.SubscribedKeywordAggr
import com.myoa.engineering.crawl.shopping.event.ArticleUpsertEvent;
import com.myoa.engineering.crawl.shopping.service.AppUserQueryService;
import com.myoa.engineering.crawl.shopping.service.SubscribedKeywordCacheService;
import com.myoa.engineering.crawl.shopping.service.UserNotifyService;
import com.myoa.engineering.crawl.shopping.service.slack.UserNotifyService;
import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget;
import com.myoa.engineering.crawl.shopping.util.SlackMessageUtils;
import com.slack.api.methods.request.chat.ChatPostMessageRequest;
import com.slack.api.methods.response.chat.ChatPostMessageResponse;
import com.slack.api.model.block.Blocks;
import org.springframework.context.event.EventListener;
import org.springframework.stereotype.Component;
import java.util.List;
import java.util.Map;
import java.util.function.Function;
import java.util.stream.Collectors;
@Component
@ -38,49 +41,74 @@ public class ArticleUpsertEventListener {
Map<CrawlTarget, List<ArticleModel>> articleMap =
((List<ArticleModel>) event.getSource()).stream()
.collect(Collectors.groupingBy(ArticleModel::getCrawlTarget));
Map<CrawlTarget, ChatPostMessageResponse> allArticleNotifiedResultMap =
articleMap.entrySet()
.stream()
.collect(Collectors.toMap(Map.Entry::getKey, e -> notifyMessage(e.getKey(), e.getValue())));
List<AppUserModel> appUsers = appUserQueryService.findAll();
appUsers.stream()
.filter(AppUserModel::getEnabled)
.map(user -> {
List<ArticleModel> filteredArticles = handleAhoCorasick(articleMap)
.apply(subscribedKeywordCacheService.getSubscribedKeywordsCached(user.getName()));
return UserNotifyModel.of(user.getName(), filteredArticles);
.flatMap(user -> {
Map<CrawlTarget, SubscribedKeywordAggregatedModel> subscribedKeywords =
subscribedKeywordCacheService.getSubscribedKeywordsCached(user.getSlackId());
return subscribedKeywords.entrySet()
.stream()
.map(entry -> {
List<ArticleModel> filtered = doAhocorasick(articleMap.get(entry.getKey()), entry.getValue());
return UserNotifyModel.builder()
.slackId(user.getSlackId())
.articles(filtered)
.chatPostMessageResponse(allArticleNotifiedResultMap.get(entry.getKey()))
.build();
});
})
.forEach(this::notifyMessage);
}
private Function<Map<CrawlTarget, SubscribedKeywordAggregatedModel>, List<ArticleModel>> handleAhoCorasick(
Map<CrawlTarget, List<ArticleModel>> articleMap) {
return userTrieModel -> {
return userTrieModel
.entrySet()
.stream().filter(e -> articleMap.containsKey(e.getKey()))
.map((entry) -> filterAhocorasick(articleMap.get(entry.getKey()), entry.getValue()))
.flatMap(List::stream)
.toList();
// return UserNotifyModel.of(userTrieModel.values().stream().findFirst().get().getUserId(),
// filteredArticle);
};
}
private List<ArticleModel> filterAhocorasick(List<ArticleModel> articles,
SubscribedKeywordAggregatedModel trieModel) {
private List<ArticleModel> doAhocorasick(List<ArticleModel> articles,
SubscribedKeywordAggregatedModel trieModel) {
if (articles == null || articles.isEmpty()) {
return List.of();
}
return articles.stream()
.filter(article -> !trieModel.getAhoCorasickTrie()
.parseText(article.getTitle())
.isEmpty())
.toList();
//ArticleUpsertEventListener::printArticle
}
private void notifyMessage(UserNotifyModel article) {
System.out.println("article = " + article);
if (article.getArticles().isEmpty()){
private ChatPostMessageResponse notifyMessage(CrawlTarget crawlTarget, List<ArticleModel> articles) {
String composited = articles.stream()
.map(ArticleModel::convertArticletoMessage)
.collect(Collectors.joining("\n"));
ChatPostMessageRequest request =
userNotifyService.generateMessage()
.blocks(Blocks.asBlocks(
SlackMessageUtils.ofHeader(crawlTarget.getAlias()),
SlackMessageUtils.ofSection(composited),
SlackMessageUtils.ofDivider()
))
.build();
return userNotifyService.notify(request);
}
private void notifyMessage(UserNotifyModel userNotifyModel) {
System.out.println("article = " + userNotifyModel);
if (userNotifyModel.getArticles().isEmpty()) {
return;
}
userNotifyService.notify("안녕 " + article.getUserId() + "\n" + article.getArticles());
ChatPostMessageRequest request =
userNotifyService.generateMessage()
.blocks(Blocks.asBlocks(
SlackMessageUtils.ofSection(userNotifyModel.toCompositedMessage())
))
.build();
userNotifyService.notify(request);
}
}

View File

@ -1,12 +0,0 @@
package com.myoa.engineering.crawl.shopping.infra.client.fmkorea;
import org.springframework.cloud.openfeign.FeignClient;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
@FeignClient(value = "fmkorea-board-client", url = "https://fmkorea.com")
public interface FmkoreaBoardClient {
@GetMapping("{boardLink}")
String getBoardHtml(@PathVariable("boardLink") String boardLink);
}

View File

@ -0,0 +1,47 @@
package com.myoa.engineering.crawl.shopping.infra.client.fmkorea;
import lombok.extern.slf4j.Slf4j;
import org.springframework.http.HttpHeaders;
import org.springframework.stereotype.Component;
import org.springframework.web.reactive.function.client.WebClient;
@Slf4j
@Component
public class FmkoreaBoardClientV2 {
private static final String URI_BOARD = "/index.php";
private final WebClient webClient;
public FmkoreaBoardClientV2() {
webClient = WebClient.builder()
.codecs(configurer -> configurer.defaultCodecs()
.maxInMemorySize(2 * 1024 * 1024))
.baseUrl("https://www.fmkorea.com")
.defaultHeader(HttpHeaders.USER_AGENT, "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3")
.defaultHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7\n")
.defaultHeader(HttpHeaders.REFERER, "https://www.fmkorea.com")
.defaultHeader(HttpHeaders.ACCEPT_ENCODING, "gzip, deflate, br")
.defaultHeader(HttpHeaders.ACCEPT_LANGUAGE, "ko-KR,ko;q=0.9,en-US;q=0.8,en;q=0.7")
.defaultHeader(HttpHeaders.CACHE_CONTROL, "max-age=0")
.defaultHeader("Sec-Ch-Ua", "\"Whale\";v=\"3\", \"Not-A.Brand\";v=\"8\", \"Chromium\";v=\"122\"")
.defaultHeader("Sec-Ch-Ua-Mobile", "?0")
.defaultHeader("Sec-Ch-Ua-Platform", "\"macOS\"")
.defaultHeader("Sec-Fetch-Dest", "document")
.defaultHeader("Sec-Fetch-Mode", "navigate")
.defaultHeader("Sec-Fetch-Site", "same-origin")
.defaultHeader("Upgrade-Insecure-Requests", "1")
.build();
}
public String getBoardHtml(Integer page, String cookie) {
return webClient.get()
.uri(builder -> builder.path(URI_BOARD)
.queryParam("page", page)
.queryParam("mid", "hotdeal")
.build())
.header("Cookie", cookie)
.exchangeToMono(clientResponse -> clientResponse.bodyToMono(String.class))
.block();
}
}

View File

@ -1,38 +0,0 @@
package com.myoa.engineering.crawl.shopping.infra.client.ppomppu;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
/**
* PpomppuBoardFeedRetriever
*
* @author Shin Woo-jin (woozu.shin@kakaoent.com)
* @since 2021-09-08
*/
@Slf4j
@Component
public class PpomppuBoardClient {
/*
private final WebClient webClient;
public PpomppuBoardClient(WebClient.Builder webClientBuilder) {
this.webClient = webClientBuilder.baseUrl(PpomppuBoardName.PPOMPPU_URL)
.exchangeStrategies(WebFluxExchangeStragiesFactory.ofTextHtml())
.filter(WebClientFilterFactory.logRequest())
.filter(WebClientFilterFactory.logResponse())
.build();
}
public Mono<String> getHtml(String uri) {
return webClient.get()
.uri(uri)
.exchangeToMono(e -> e.bodyToMono(String.class))
.publishOn(Schedulers.boundedElastic())
.onErrorResume(WebClientRequestException.class, t -> {
log.info("Exception occured, ignoring. : {}", t.getClass().getSimpleName());
return Mono.empty();
});
// .doOnNext(e -> log.info("[getHtml] {}", e));
}
*/
}

View File

@ -1,17 +0,0 @@
package com.myoa.engineering.crawl.shopping.infra.client.slack;
import com.myoa.engineering.crawl.shopping.configuration.FeignDefaultConfig;
import com.myoa.engineering.crawl.shopping.dto.slack.SlackMessageDTO;
import org.springframework.cloud.openfeign.FeignClient;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestHeader;
@FeignClient(value = "slack-api-client", url = "https://slack.com/api",
configuration = FeignDefaultConfig.class)
public interface SlackAPIClient {
@PostMapping("/chat.postMessage")
String sendMessage(@RequestBody SlackMessageDTO message,
@RequestHeader("Authorization") String token);
}

View File

@ -4,6 +4,9 @@ import com.myoa.engineering.crawl.shopping.domain.entity.v2.AppUser;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.stereotype.Repository;
import java.util.Optional;
@Repository
public interface AppUserRepository extends JpaRepository<AppUser, Long> {
Optional<AppUser> findBySlackId(String userId);
}

View File

@ -6,6 +6,7 @@ import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.stereotype.Repository;
import java.util.List;
import java.util.Optional;
@Repository
public interface SubscribedKeywordRepository extends JpaRepository<SubscribedKeyword, Long> {
@ -20,4 +21,6 @@ public interface SubscribedKeywordRepository extends JpaRepository<SubscribedKey
List<SubscribedKeyword> findByUserIdAndCrawlTarget(String userId, CrawlTarget crawlTarget);
List<SubscribedKeyword> findByUserId(String userId);
Optional<SubscribedKeyword> findByUserIdAndCrawlTargetAndKeyword(String userId, CrawlTarget crawlTarget, String keyword);
}

View File

@ -1,15 +1,20 @@
package com.myoa.engineering.crawl.shopping.scheduler;
import com.myoa.engineering.crawl.shopping.crawlhandler.CrawlHandler;
import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget;
import lombok.extern.slf4j.Slf4j;
import org.springframework.context.annotation.Profile;
import org.springframework.scheduling.annotation.EnableScheduling;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import java.util.Arrays;
import java.util.List;
@Slf4j
@Component
@EnableScheduling
@Profile("!local")
public class ParseEventEmitter {
private final List<CrawlHandler> crawlHandlers;
@ -18,10 +23,11 @@ public class ParseEventEmitter {
this.crawlHandlers = crawlHandlers;
}
// @Scheduled(cron = "0 0/5 * * * ?")
@Scheduled(cron = "0 0/15 * * * ?")
public void emit() {
log.info("[emitDomesticBoard] trigger fired!");
crawlHandlers.forEach(CrawlHandler::handle);
crawlHandlers
.stream()
.filter(e -> e.getCrawlTarget().isAvailable())
.forEach(CrawlHandler::handle);
}
}

View File

@ -0,0 +1,31 @@
package com.myoa.engineering.crawl.shopping.service;
import com.myoa.engineering.crawl.shopping.domain.entity.v2.AppUser;
import com.myoa.engineering.crawl.shopping.infra.repository.v2.AppUserRepository;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
@Service
public class AppUserCommandService {
private final AppUserRepository appUserRepository;
public AppUserCommandService(AppUserRepository appUserRepository) {
this.appUserRepository = appUserRepository;
}
@Transactional
public void register(String slackId, String userName) {
appUserRepository.findBySlackId(slackId)
.ifPresent(user -> {
throw new IllegalArgumentException("이미 등록된 사용자입니다.");
});
AppUser appUser = AppUser.builder()
.enabled(true)
.slackId(slackId)
.name(userName)
.build();
appUserRepository.save(appUser);
}
}

View File

@ -3,6 +3,7 @@ package com.myoa.engineering.crawl.shopping.service;
import com.myoa.engineering.crawl.shopping.domain.model.v2.AppUserModel;
import com.myoa.engineering.crawl.shopping.infra.repository.v2.AppUserRepository;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import java.util.List;
@ -15,6 +16,7 @@ public class AppUserQueryService {
this.appUserRepository = appUserRepository;
}
@Transactional(readOnly = true)
public List<AppUserModel> findAll() {
return appUserRepository.findAll()
.stream()
@ -22,4 +24,11 @@ public class AppUserQueryService {
.toList();
}
@Transactional(readOnly = true)
public AppUserModel findByUserId(String userId) {
return appUserRepository.findBySlackId(userId)
.map(AppUserModel::from)
.orElseThrow();
}
}

View File

@ -19,23 +19,23 @@ public class SubscribedKeywordCacheService {
this.subscribedKeywordQueryService = subscribedKeywordQueryService;
}
@Cacheable(cacheNames = "subscribe.keywords", key = "#userId + '_' + #crawlTarget.name()")
public SubscribedKeywordAggregatedModel getSubscribedKeywordsCached(String userId, CrawlTarget crawlTarget) {
@Cacheable(cacheNames = "subscribe.keywords", key = "#slackId + '_' + #crawlTarget.name()")
public SubscribedKeywordAggregatedModel getSubscribedKeywordsCached(String slackId, CrawlTarget crawlTarget) {
System.out.println("getSubscribedKeywordsCached");
List<String> keywords = subscribedKeywordQueryService.findByUserWithTarget(userId, crawlTarget)
List<String> keywords = subscribedKeywordQueryService.findByUserWithTarget(slackId, crawlTarget)
.stream().map(SubscribedKeyword::getKeyword).toList();
return SubscribedKeywordAggregatedModel.of(userId, crawlTarget, keywords);
return SubscribedKeywordAggregatedModel.of(slackId, crawlTarget, keywords);
}
@Cacheable(cacheNames = "subscribe.keywords", key = "#userId")
public Map<CrawlTarget, SubscribedKeywordAggregatedModel> getSubscribedKeywordsCached(String userId) {
@Cacheable(cacheNames = "subscribe.keywords", key = "#slackId")
public Map<CrawlTarget, SubscribedKeywordAggregatedModel> getSubscribedKeywordsCached(String slackId) {
System.out.println("getSubscribedKeywordsCached");
return subscribedKeywordQueryService.findByUser(userId)
return subscribedKeywordQueryService.findByUser(slackId)
.stream()
.collect(Collectors.groupingBy(SubscribedKeyword::getCrawlTarget,
Collectors.mapping(SubscribedKeyword::getKeyword, Collectors.toList())))
.entrySet().stream()
.collect(Collectors.toMap(Map.Entry::getKey, e -> SubscribedKeywordAggregatedModel.of(userId, e.getKey(), e.getValue())));
.collect(Collectors.toMap(Map.Entry::getKey, e -> SubscribedKeywordAggregatedModel.of(slackId, e.getKey(), e.getValue())));
}
}

View File

@ -0,0 +1,46 @@
package com.myoa.engineering.crawl.shopping.service;
import com.myoa.engineering.crawl.shopping.domain.entity.v2.SubscribedKeyword;
import com.myoa.engineering.crawl.shopping.dto.feed.v1.SubscribedKeywordDTO;
import com.myoa.engineering.crawl.shopping.infra.repository.v2.SubscribedKeywordRepository;
import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import java.util.List;
@Service
public class SubscribedKeywordCommandService {
private final SubscribedKeywordRepository subscribedKeywordRepository;
public SubscribedKeywordCommandService(SubscribedKeywordRepository subscribedKeywordRepository) {
this.subscribedKeywordRepository = subscribedKeywordRepository;
}
@Transactional
public void add(SubscribedKeyword subscribedKeyword) {
subscribedKeywordRepository.save(subscribedKeyword);
}
@Transactional
public void delete(String userId, CrawlTarget crawlTarget, String keyword) {
SubscribedKeyword subscribedKeyword = subscribedKeywordRepository.findByUserIdAndCrawlTargetAndKeyword(userId, crawlTarget, keyword)
.orElseThrow(() -> new IllegalArgumentException("Not found keyword: " + keyword));
subscribedKeywordRepository.delete(subscribedKeyword);
}
@Transactional(readOnly = true)
public List<SubscribedKeywordDTO> list(String userId) {
return subscribedKeywordRepository.findByUserId(userId)
.stream()
.map(e -> SubscribedKeywordDTO.builder()
.keyword(e.getKeyword())
.userId(e.getUserId())
.crawlTarget(e.getCrawlTarget())
.build())
.toList();
}
}

View File

@ -1,34 +0,0 @@
package com.myoa.engineering.crawl.shopping.service;
import com.myoa.engineering.crawl.shopping.configuration.slack.properties.SlackSecretProperties;
import com.myoa.engineering.crawl.shopping.dto.slack.SlackMessageDTO;
import com.myoa.engineering.crawl.shopping.infra.client.slack.SlackAPIClient;
import org.springframework.stereotype.Service;
@Service
public class UserNotifyService {
private static final String SLACK_PROPERTIES_UNIT_NAME = "shopping-crawler";
// private static final String NOTIFY_CHANNEL_ID = "notify_shopping";
// private static final String NOTIFY_ICON_EMOJI = ":monge_big:";
// private static final String NOTIFY_BOT_NAME = "몽이 탈호구봇";
private final SlackAPIClient slackAPIClient;
private final SlackSecretProperties.SlackSecretPropertiesUnit slackSecretProperties;
public UserNotifyService(SlackAPIClient slackAPIClient,
SlackSecretProperties slackSecretProperties) {
this.slackAPIClient = slackAPIClient;
this.slackSecretProperties = slackSecretProperties.find(SLACK_PROPERTIES_UNIT_NAME);
}
public void notify(String message) {
SlackMessageDTO slackMessageDTO = SlackMessageDTO.builder()
.channel(slackSecretProperties.getChannel())
.text(message)
.iconEmoji(slackSecretProperties.getIconEmoji())
.username(slackSecretProperties.getUsername())
.build();
slackAPIClient.sendMessage(slackMessageDTO, slackSecretProperties.getToken());
}
}

View File

@ -0,0 +1,33 @@
package com.myoa.engineering.crawl.shopping.service.slack;
import com.myoa.engineering.crawl.shopping.dto.slack.v2.SlackSlashCommandDTO;
import com.myoa.engineering.crawl.shopping.service.AppUserCommandService;
import org.springframework.stereotype.Service;
import java.util.List;
@Service
public class RegisterCommandHandler implements SlackCommandHandler {
private final AppUserCommandService appUserCommandService;
public RegisterCommandHandler(AppUserCommandService appUserCommandService) {
this.appUserCommandService = appUserCommandService;
}
public List<String> handle(SlackSlashCommandDTO requestDTO) {
if (!requestDTO.getCommand().equals("/등록")) {
throw new IllegalArgumentException("Invalid command: " + requestDTO.getCommand());
}
final String userName = requestDTO.getText();
try {
appUserCommandService.register(requestDTO.getUser_id(), userName);
return List.of("등록 완료");
} catch (Exception e) {
return List.of(e.getMessage());
}
}
}

View File

@ -0,0 +1,34 @@
package com.myoa.engineering.crawl.shopping.service.slack;
import com.myoa.engineering.crawl.shopping.dto.slack.v2.SlackSlashCommandDTO;
import org.springframework.stereotype.Service;
import java.util.List;
@Service
public class ShoppingCommandHandler implements SlackCommandHandler {
private final List<ShoppingCommandProcessor> processors;
public ShoppingCommandHandler(List<ShoppingCommandProcessor> processors) {
this.processors = processors;
}
public List<String> handle(SlackSlashCommandDTO requestDTO) {
String[] commands = requestDTO.getText().split("\\s+", 2);
if (!requestDTO.getCommand().equals("/쇼핑")) {
throw new IllegalArgumentException("Invalid command: " + requestDTO.getCommand());
}
final String action = commands[0];
return processors.stream()
.filter(processor -> processor.isAssignable(action))
.map(processor -> processor.process(requestDTO))
.toList();
}
}

View File

@ -0,0 +1,10 @@
package com.myoa.engineering.crawl.shopping.service.slack;
import com.myoa.engineering.crawl.shopping.dto.slack.v2.SlackSlashCommandDTO;
public interface ShoppingCommandProcessor {
boolean isAssignable(String menuContext);
String process(SlackSlashCommandDTO requestDTO);
}

View File

@ -0,0 +1,4 @@
package com.myoa.engineering.crawl.shopping.service.slack;
public interface SlackCommandHandler {
}

View File

@ -0,0 +1,41 @@
package com.myoa.engineering.crawl.shopping.service.slack;
import com.myoa.engineering.crawl.shopping.configuration.slack.properties.SlackSecretProperties;
import com.slack.api.methods.MethodsClient;
import com.slack.api.methods.request.chat.ChatPostMessageRequest;
import com.slack.api.methods.response.chat.ChatPostMessageResponse;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
@Slf4j
@Service
public class UserNotifyService {
private static final String SLACK_PROPERTIES_UNIT_NAME = "shopping-crawler";
private final SlackSecretProperties.SlackSecretPropertiesUnit slackSecretProperties;
private final MethodsClient methodsClient;
public UserNotifyService(SlackSecretProperties slackSecretProperties,
MethodsClient methodsClient) {
this.slackSecretProperties = slackSecretProperties.find(SLACK_PROPERTIES_UNIT_NAME);
this.methodsClient = methodsClient;
}
public ChatPostMessageResponse notify(ChatPostMessageRequest request) {
try {
return methodsClient.chatPostMessage(request);
} catch (Exception e) {
log.warn("Failed. message: {}", request, e);
ChatPostMessageResponse response = new ChatPostMessageResponse();
response.setOk(false);
return response;
}
}
public ChatPostMessageRequest.ChatPostMessageRequestBuilder generateMessage() {
return ChatPostMessageRequest.builder()
.channel(slackSecretProperties.getChannel())
.username(slackSecretProperties.getUsername());
}
}

View File

@ -0,0 +1,67 @@
package com.myoa.engineering.crawl.shopping.service.slack.shopping;
import com.myoa.engineering.crawl.shopping.domain.entity.v2.SubscribedKeyword;
import com.myoa.engineering.crawl.shopping.dto.feed.v1.SubscribedKeywordDTO;
import com.myoa.engineering.crawl.shopping.dto.slack.v2.SlackSlashCommandDTO;
import com.myoa.engineering.crawl.shopping.service.SubscribedKeywordCommandService;
import com.myoa.engineering.crawl.shopping.service.slack.ShoppingCommandProcessor;
import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget;
import org.springframework.stereotype.Component;
import java.util.List;
import java.util.stream.Collectors;
@Component
public class KeywordShoppingCommandProcessor implements ShoppingCommandProcessor {
private final SubscribedKeywordCommandService subscribedKeywordCommandService;
public KeywordShoppingCommandProcessor(SubscribedKeywordCommandService subscribedKeywordCommandService) {
this.subscribedKeywordCommandService = subscribedKeywordCommandService;
}
@Override
public boolean isAssignable(String menuContext) {
return menuContext.equals("키워드");
}
@Override
public String process(SlackSlashCommandDTO requestDTO) {
// "키워드 추가 (쇼핑몰명) (키워드명)"
final String[] splited = requestDTO.getText().split("\\s+", 4);
final String action = splited[1];
switch (action) {
case "추가" -> {
final CrawlTarget crawlTarget = CrawlTarget.fromAlias(splited[2]);
final String keyword = splited[3];
SubscribedKeyword subscribedKeyword =
SubscribedKeyword.builder()
.crawlTarget(crawlTarget)
.userId(requestDTO.getUser_id())
.keyword(keyword)
.build();
subscribedKeywordCommandService.add(subscribedKeyword);
return "keyword: [" + keyword + "] 추가완료";
}
case "삭제" -> {
final CrawlTarget crawlTarget = CrawlTarget.fromAlias(splited[2]);
final String keyword = splited[3];
subscribedKeywordCommandService.delete(requestDTO.getUser_id(), crawlTarget, keyword);
return "keyword: [" + keyword + "] 삭제완료";
}
case "목록" -> {
List<SubscribedKeywordDTO> keywords = subscribedKeywordCommandService.list(requestDTO.getUser_id());
if (keywords.isEmpty()) {
return "등록된 키워드가 없습니다.";
}
return keywords.stream().map(SubscribedKeywordDTO::toMessage)
.collect(Collectors.joining("\n"));
}
default -> {
return "지원하지 않는 명령어입니다.";
}
}
}
}

View File

@ -0,0 +1,25 @@
package com.myoa.engineering.crawl.shopping.service.slack.shopping;
import com.myoa.engineering.crawl.shopping.dto.slack.v2.SlackSlashCommandDTO;
import com.myoa.engineering.crawl.shopping.service.slack.ShoppingCommandProcessor;
import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget;
import org.springframework.stereotype.Component;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@Component
public class ListShoppingCommandProcessor implements ShoppingCommandProcessor {
@Override
public boolean isAssignable(String menuContext) {
return menuContext.equals("목록");
}
@Override
public String process(SlackSlashCommandDTO requestDTO) {
return Stream.of(CrawlTarget.values())
.map(e -> e.getAlias() + " | " + e.isAvailable())
.collect(Collectors.joining("\n"));
}
}

View File

@ -1,27 +1,38 @@
package com.myoa.engineering.crawl.shopping.util;
import org.springframework.lang.Nullable;
import java.time.*;
import java.time.format.DateTimeFormatter;
public final class DateTimeUtils {
private static final DateTimeFormatter FORMATTER_HHMMss = DateTimeFormatter.ofPattern("HH:mm:ss");
public static final DateTimeFormatter FORMATTER_HHMMss = DateTimeFormatter.ofPattern("HH:mm:ss");
public static final DateTimeFormatter FORMATTER_YYMMDD_SLASH = DateTimeFormatter.ofPattern("yy/MM/dd");
public static final DateTimeFormatter FORMATTER_YYMMDD_DOT = DateTimeFormatter.ofPattern("yyyy.MM.dd");
public static final DateTimeFormatter FORMATTER_HHMM = DateTimeFormatter.ofPattern("HH:mm");
private static final ZoneId ZONE_ASIA_SEOUL = ZoneId.of("Asia/Seoul");
private DateTimeUtils() {
}
public static ZonedDateTime parse(String HHMMss) {
public static ZonedDateTime parse(String dateTimeString, DateTimeFormatter formatter, @Nullable DateTimeFormatter fallback) {
try {
LocalTime time = LocalTime.parse(HHMMss, FORMATTER_HHMMss);
LocalTime time = LocalTime.parse(dateTimeString, formatter);
LocalDateTime dateTime = LocalDateTime.of(LocalDate.now(), time);
if (dateTime.isAfter(LocalDateTime.now())) {
dateTime = dateTime.minusDays(1);
}
return dateTime.atZone(ZONE_ASIA_SEOUL);
} catch (Exception e) {
return null;
} catch (Exception ignored) {
}
try {
LocalDate date = LocalDate.parse(dateTimeString, fallback);
return date.atStartOfDay(ZONE_ASIA_SEOUL);
} catch (Exception ignored) {
}
return null;
}
}

View File

@ -0,0 +1,33 @@
package com.myoa.engineering.crawl.shopping.util;
import com.slack.api.model.block.DividerBlock;
import com.slack.api.model.block.HeaderBlock;
import com.slack.api.model.block.SectionBlock;
import com.slack.api.model.block.composition.MarkdownTextObject;
import com.slack.api.model.block.composition.PlainTextObject;
public final class SlackMessageUtils {
private SlackMessageUtils() {
}
public static HeaderBlock ofHeader(String message) {
return HeaderBlock.builder()
.text(PlainTextObject.builder()
.text(message)
.build())
.build();
}
public static DividerBlock ofDivider() {
return DividerBlock.builder().build();
}
public static SectionBlock ofSection(String message) {
return SectionBlock.builder()
.text(MarkdownTextObject.builder()
.text(message)
.build())
.build();
}
}

View File

@ -1,12 +0,0 @@
spring:
config:
activate:
on-profile: development
import:
- "configserver:http://192.168.0.100:11080"
server:
port: 20081
# import: optional:configserver:http://localhost:11080 # can be start up even config server was not found.

View File

@ -0,0 +1,23 @@
spring:
config:
activate:
on-profile: prod
import:
- classpath:/datasource/prod.yml
- classpath:/slack/prod.yml
devtools:
livereload:
enabled: false
restart:
enabled: false
server:
port: 20080
# import: optional:configserver:http://localhost:11080 # can be start up even config server was not found.
feign:
client:
config:
default:
loggerLevel: FULL

View File

@ -1,6 +0,0 @@
spring:
config:
activate:
on-profile: production
import:
- "configserver:http://ppn-config-server:20080"

View File

@ -7,8 +7,7 @@ spring:
active: ${SPRING_ACTIVE_PROFILE:local}
group:
local: "local,datasource-local,webclient-local"
development: "development,datasource-development,webclient-development"
production: "production,datasource-production,webclient-production"
prod: "prod"
freemarker:
enabled: false
cloud:

View File

@ -2,14 +2,14 @@
<configuration>
<springProperty name="DEFAULT_LEVEL_CONFIG" source="log.defaultLevel" />
<springProfile name="local">
<include resource="logback/logback-development.xml" />
<include resource="logback/logback-dev.xml" />
<logger name="org.apache.kafka" level="INFO" />
</springProfile>
<springProfile name="development">
<include resource="logback/logback-development.xml" />
<springProfile name="dev">
<include resource="logback/logback-dev.xml" />
<logger name="org.apache.kafka" level="INFO" />
</springProfile>
<springProfile name="production">
<include resource="logback/logback-production.xml" />
<springProfile name="prod">
<include resource="logback/logback-prod.xml" />
</springProfile>
</configuration>

View File

@ -2,6 +2,10 @@
<included>
<!-- =========== property BETA ========= -->
<property name="DEFAULT_LEVEL" value="${DEFAULT_LEVEL_CONFIG:-INFO}"/>
<!--file-->
<property name="IMMEDIATE_FLUSH" value="true"/>
<!--nelo2-->
<property name="NELO2_LEVEL" value="WARN"/>
<!-- =========== include appender =========== -->
<include resource="org/springframework/boot/logging/logback/defaults.xml"/>
<include resource="org/springframework/boot/logging/logback/console-appender.xml"/>

View File

@ -1,19 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<included>
<!-- =========== property BETA ========= -->
<property name="DEFAULT_LEVEL" value="${DEFAULT_LEVEL_CONFIG:-INFO}"/>
<!--file-->
<property name="DIRECTORY" value="/home1/www/logs/supervisor"/>
<property name="IMMEDIATE_FLUSH" value="true"/>
<!--nelo2-->
<property name="NELO2_LEVEL" value="WARN"/>
<!-- =========== include appender =========== -->
<include resource="org/springframework/boot/logging/logback/defaults.xml"/>
<include resource="org/springframework/boot/logging/logback/console-appender.xml"/>
<include resource="logback/component/logback-nelo2.xml"/>
<include resource="logback/component/logback-datachain.xml"/>
<!-- =========== root logger ============== -->
<root level="${DEFAULT_LEVEL}">
<appender-ref ref="CONSOLE"/>
</root>
</included>

View File

@ -3,15 +3,12 @@
<!-- =========== property RELEASE ========= -->
<property name="DEFAULT_LEVEL" value="${DEFAULT_LEVEL_CONFIG:-INFO}"/>
<!--file-->
<property name="DIRECTORY" value="/home1/www/logs/supervisor"/>
<property name="IMMEDIATE_FLUSH" value="true"/>
<!--nelo2-->
<property name="NELO2_LEVEL" value="WARN"/>
<!-- =========== include appender =========== -->
<include resource="org/springframework/boot/logging/logback/defaults.xml"/>
<include resource="org/springframework/boot/logging/logback/console-appender.xml"/>
<include resource="logback/component/logback-nelo2.xml"/>
<include resource="logback/component/logback-datachain.xml"/>
<!-- =========== root logger ============== -->
<root level="${DEFAULT_LEVEL}">
<appender-ref ref="CONSOLE"/>

View File

@ -0,0 +1,33 @@
package com.myoa.engineering.crawl.shopping.controller;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
class SlackEventSubscriptionAPIControllerTest {
@Test
void test_handleAddSubscription_1() {
// given
String sut = "/쇼핑 키워드 추가 키워드1";
// when
String actual = sut.replaceFirst("/.+? ", "");
// then
Assertions.assertEquals("키워드 추가 키워드1", actual);
}
@Test
void test_handleAddSubscription_2() {
// given
String sut = "키워드 추가 키워드 1입니 다";
// when
String[] actual = sut.split("\\s+", 2);
// then
Assertions.assertEquals(2, actual.length);
Assertions.assertEquals("키워드", actual[0]);
Assertions.assertEquals("추가 키워드 1입니 다", actual[1]);
}
}

View File

@ -0,0 +1,63 @@
package com.myoa.engineering.crawl.shopping.controller;
import com.slack.api.methods.MethodsClient;
import com.slack.api.methods.SlackApiException;
import com.slack.api.methods.request.chat.ChatPostMessageRequest;
import com.slack.api.model.block.DividerBlock;
import com.slack.api.model.block.HeaderBlock;
import com.slack.api.model.block.LayoutBlock;
import com.slack.api.model.block.SectionBlock;
import com.slack.api.model.block.composition.PlainTextObject;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.ActiveProfiles;
import org.springframework.test.context.junit.jupiter.SpringExtension;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
@ActiveProfiles("local")
@ExtendWith(SpringExtension.class)
@SpringBootTest
class TestAPIControllerTest {
@Autowired
MethodsClient methodsClient;
@Test
void test1() throws SlackApiException, IOException {
List<LayoutBlock> blocks = new ArrayList<>();
HeaderBlock asdf = HeaderBlock.builder()
.text(PlainTextObject.builder()
.text("asdf")
.build())
.build();
DividerBlock dividerBlock = DividerBlock.builder().build();
SectionBlock section = SectionBlock.builder()
.text(PlainTextObject.builder()
.text("• asdf")
.build())
.build();
blocks.add(asdf);
blocks.add(section);
blocks.add(section);
blocks.add(section);
blocks.add(dividerBlock);
ChatPostMessageRequest request = ChatPostMessageRequest.builder()
.channel("notify_shopping")
.blocks(blocks)
.build();
methodsClient.chatPostMessage(request);
}
private void notifyMessage() {
}
}

View File

@ -0,0 +1,16 @@
package com.myoa.engineering.crawl.shopping.crawlhandler;
import com.myoa.engineering.crawl.shopping.util.TestDataUtils;
import org.junit.jupiter.api.Test;
class FmkoreaCrawlHandlerTest {
@Test
void resolve_fake430() {
String fakeHtml = TestDataUtils.fileToString("testdata/fmkorea/fake430.html");
FmkoreaFake430Resolver.resolveFake430(fakeHtml);
}
}

View File

@ -0,0 +1,31 @@
package com.myoa.engineering.crawl.shopping.crawlhandler.parser;
import com.myoa.engineering.crawl.shopping.domain.entity.v2.Article;
import com.myoa.engineering.crawl.shopping.util.TestDataUtils;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import java.util.List;
class FmkoreaArticleParserTest {
private FmkoreaArticleParser sut;
@BeforeEach
void setUp() {
sut = new FmkoreaArticleParser();
}
@Test
void parse() {
// given
String boardHtml = TestDataUtils.fileToString("testdata/fmkorea/file1.html");
// when
List<Article> actual = sut.parse(boardHtml);
// then
Assertions.assertEquals(20, actual.size());
}
}

View File

@ -1,6 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<configuratiown>
<springProperty name="DEFAULT_LEVEL_CONFIG" source="log.defaultLevel"/>
<include resource="logback-development.xml"/>
</configuratiown>

File diff suppressed because one or more lines are too long

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,17 @@
curl 'https://www.fmkorea.com/hotdeal' \
-H 'accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7' \
-H 'accept-language: ko-KR,ko;q=0.9,en-US;q=0.8,en;q=0.7' \
-H 'cache-control: max-age=0' \
-H 'cookie: PHPSESSID=gkpqv1r1odheqkqpal2eh24e4j; _ga=GA1.1.1510788361.1715523165; use_np=use_np; readed_documents=7021938967.7023825588.7023440365; _ga_GFCL6FWBKV=GS1.1.1715523165.1.1.1715526874.51.0.0' \
-H 'dnt: 1' \
-H 'priority: u=0, i' \
-H 'referer: https://www.fmkorea.com/index.php?mid=hotdeal&page=10' \
-H 'sec-ch-ua: "Chromium";v="124", "Google Chrome";v="124", "Not-A.Brand";v="99"' \
-H 'sec-ch-ua-mobile: ?0' \
-H 'sec-ch-ua-platform: "macOS"' \
-H 'sec-fetch-dest: document' \
-H 'sec-fetch-mode: navigate' \
-H 'sec-fetch-site: same-origin' \
-H 'sec-fetch-user: ?1' \
-H 'upgrade-insecure-requests: 1' \
-H 'user-agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36'

View File

@ -6,8 +6,20 @@ import lombok.Getter;
@Getter
@AllArgsConstructor
public enum CrawlTarget {
PPOMPPU_DOMESTIC,
PPOMPPU_OVERSEA,
FMKOREA,
PPOMPPU_DOMESTIC("뽐뿌국내", true),
PPOMPPU_OVERSEA("뽐뿌해외", true),
FMKOREA("펨코", false),
;
private final String alias;
private final boolean available;
public static CrawlTarget fromAlias(String alias) {
for (CrawlTarget crawlTarget : values()) {
if (crawlTarget.getAlias().equals(alias)) {
return crawlTarget;
}
}
throw new IllegalArgumentException("Not found alias: " + alias);
}
}