MyoaEngineering
/
ShoppingCrawler
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

[NO-ISSUE] Fix bug

This commit is contained in:
woo-jin.shin 2021-12-01 01:16:15 +09:00
parent b204b70b79
commit 7b230fdb74
25 changed files with 219 additions and 119 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
build.gradle
View File

@ -6,7 +6,7 @@ plugins {
}
group = 'com.myoa.engineering.crawl.ppomppu'
version = '1.0.1'
version = '1.0.3'
sourceCompatibility = '11'
configurations {
@ -21,7 +21,7 @@ repositories {
allprojects {
group = 'com.myoa.engineering.crawl.ppomppu'
version = '1.0.1'
version = '1.0.3'
apply plugin: 'java'
apply plugin: 'idea'

3
copy.bat Normal file
View File

@ -0,0 +1,3 @@
xcopy /y .\processor\build\libs\*.jar .\
xcopy /y .\receiver\build\libs\*.jar .\
xcopy /y .\sender\build\libs\*.jar .\

4
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/controller/CrawlAPIController.java
View File

@ -1,6 +1,5 @@
package com.myoa.engineering.crawl.ppomppu.processor.controller;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
@ -48,7 +47,8 @@ public class CrawlAPIController {
ppomppuRSSFeedService.getArticles(boardName)
.map(e -> ppomppuArticleService.filterOnlyNewArticles(boardName, e))
.map(e -> ppomppuArticleService.save(boardName, e))
.flatMap(messageSenderService::sendMessageToSlack);
.filter(e -> !e.isEmpty())
.flatMap(messageSenderService::sendBlockMessageToSlack);
return publishedMessages.then(Mono.just(APIResponse.success(result.done())));
}

28
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/PpomppuArticleTransformer.java
View File

@ -8,6 +8,7 @@ import java.util.function.Function;
import java.util.stream.Collectors;
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
import com.myoa.engineering.crawl.ppomppu.support.dto.BlockMessageDTO;
import com.myoa.engineering.crawl.ppomppu.support.dto.SimpleMessageDTO;
/**
@ -20,21 +21,22 @@ public final class PpomppuArticleTransformer {
private PpomppuArticleTransformer() {}
private static final String MESSAGE_FORMAT_V1 = "%s)) `%s` <%s:LINK>";
private static final String MESSAGE_FORMAT_V1 = "%s)) <%s|LINK> `%s` ";
private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")
.withZone(ZoneId.of("Asia/Seoul"));
public static final Function<PpomppuArticle, SimpleMessageDTO> TRANSFORM_TO_MESSAGE_DTO = entity ->
public static final Function<PpomppuArticle, SimpleMessageDTO> TRANSFORM_TO_MESSAGE_DTO = article ->
SimpleMessageDTO.builder()
.requestedAt(Instant.now())
.publishedAt(entity.getRegisteredAt())
.title(String.format(MESSAGE_FORMAT_V1, entity.getBoardName().getMenuName(), entity.getTitle()))
.body(entity.getArticleUrl())
.publishedAt(article.getRegisteredAt())
.title(String.format(MESSAGE_FORMAT_V1,
article.getBoardName().getMenuName(), article.getArticleUrl(), article.getTitle()))
.body(article.getArticleUrl())
.build();
// https://stackoverflow.com/questions/24882927/using-streams-to-convert-a-list-of-objects-into-a-string-obtained-from-the-tostr
public static SimpleMessageDTO transform(List<PpomppuArticle> articles) {
public static SimpleMessageDTO transformToSimpleMessage(List<PpomppuArticle> articles) {
Instant requestedAt = Instant.now();
String body = articles.stream()
.map(PpomppuArticleTransformer::convertToInlineMessage)
@ -46,8 +48,20 @@ public final class PpomppuArticleTransformer {
.build();
}
public static BlockMessageDTO transformToBlockMessage(List<PpomppuArticle> articles) {
Instant requestedAt = Instant.now();
List<String> body = articles.stream()
.map(PpomppuArticleTransformer::convertToInlineMessage)
.collect(Collectors.toList());
return BlockMessageDTO.builder()
.requestedAt(requestedAt)
.title(DATE_TIME_FORMATTER.format(requestedAt))
.blocks(body)
.build();
}
public static String convertToInlineMessage(PpomppuArticle article) {
return String.format(MESSAGE_FORMAT_V1,
article.getBoardName().getMenuName(), article.getTitle(), article.getArticleUrl());
article.getBoardName().getMenuName(), article.getArticleUrl(), article.getTitle());
}
}

18
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/client/MessageSenderAPIClient.java
View File

@ -8,6 +8,7 @@ import org.springframework.web.reactive.function.client.WebClient;
import org.springframework.web.reactive.function.client.WebClientRequestException;
import com.myoa.engineering.crawl.ppomppu.processor.dto.constant.WebClientPropertiesUnitName;
import com.myoa.engineering.crawl.ppomppu.support.dto.BlockMessageDTO;
import com.myoa.engineering.crawl.ppomppu.support.dto.SimpleMessageDTO;
import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebClientFilterFactory;
import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebFluxExchangeStragiesFactory;
@ -42,9 +43,22 @@ public class MessageSenderAPIClient {
.build();
}
public Mono<String> sendMessageToSlack(SimpleMessageDTO dto) {
public Mono<String> sendSimpleMessageToSlack(SimpleMessageDTO dto) {
return webClient.post()
.uri("/api/v1/messages/sendMessage/messengers/slack")
.uri("/api/v1/messages/sendSimpleMessage/messengers/slack")
.bodyValue(dto)
.exchangeToMono(e -> e.bodyToMono(new ParameterizedTypeReference<String>() {}))
.publishOn(Schedulers.boundedElastic())
.onErrorResume(WebClientRequestException.class, t -> {
log.info("Exception occured, ignoring. : {}", t.getClass().getSimpleName());
return Mono.empty();
});
}
public Mono<String> sendBlockMessageToSlack(BlockMessageDTO dto) {
return webClient.post()
.uri("/api/v1/messages/sendBlockMessage/messengers/slack")
.bodyValue(dto)
.exchangeToMono(e -> e.bodyToMono(new ParameterizedTypeReference<String>() {}))
.publishOn(Schedulers.boundedElastic())

12
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/MessageSenderService.java
View File

@ -27,12 +27,16 @@ public class MessageSenderService {
this.messageSenderAPIClient = messageSenderAPIClient;
}
public Mono<String> sendMessageToSlack(PpomppuArticle article) {
return messageSenderAPIClient.sendMessageToSlack(PpomppuArticleTransformer.TRANSFORM_TO_MESSAGE_DTO.apply(article));
public Mono<String> sendSimpleMessageToSlack(PpomppuArticle article) {
return messageSenderAPIClient.sendSimpleMessageToSlack(PpomppuArticleTransformer.TRANSFORM_TO_MESSAGE_DTO.apply(article));
}
public Mono<String> sendMessageToSlack(List<PpomppuArticle> articles) {
return messageSenderAPIClient.sendMessageToSlack(PpomppuArticleTransformer.transform(articles));
public Mono<String> sendSimpleMessageToSlack(List<PpomppuArticle> articles) {
return messageSenderAPIClient.sendSimpleMessageToSlack(PpomppuArticleTransformer.transformToSimpleMessage(articles));
}
public Mono<String> sendBlockMessageToSlack(List<PpomppuArticle> articles) {
return messageSenderAPIClient.sendBlockMessageToSlack(PpomppuArticleTransformer.transformToBlockMessage(articles));
}
}

5
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/PpomppuArticleService.java
View File

@ -50,7 +50,10 @@ public class PpomppuArticleService {
// save PpomppuBoardFeedStatus
Optional<PpomppuBoardFeedStatus> boardFeedStatus = ppomppuBoardFeedStatusRepository.findByBoardName(boardName);
log.info("boardName: {}, isPresent?: {}", boardName, boardFeedStatus.isPresent());
log.info("[save] boardName: {}, isPresent?: {}, latestArticleId: {}",
boardName, boardFeedStatus.isPresent(), latestArticleId);
log.info("[save] articles count: {}, article ids: {}",
articles.size(), articles.stream().map(PpomppuArticle::getArticleId).toArray());
boardFeedStatus.ifPresentOrElse(e -> {
if (latestArticleId.longValue() > 0L) {
e.updateArticleId(latestArticleId);

3
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/PpomppuFeedService.java
View File

@ -4,6 +4,8 @@ import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
import com.myoa.engineering.crawl.ppomppu.processor.dto.PpomppuArticleParser;
import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.client.PpomppuBoardFeedRetriever;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import java.util.Comparator;
import java.util.List;
import lombok.extern.slf4j.Slf4j;
import org.jsoup.Jsoup;
@ -34,6 +36,7 @@ public class PpomppuFeedService {
// .doOnNext(e -> log.info("pre tbody - {}", e.html()));
return extractArticlesFromTbody(tbody).map(this::convertFromElement)
.map(e -> e.updateBoardName(boardName))
.sort(Comparator.comparing(PpomppuArticle::getArticleId))
// .doOnNext(e -> log.info("parsed Result: {}", e))
.collectList();
}

2
processor/src/main/resources/application-development.yml
View File

@ -3,7 +3,7 @@ spring:
activate:
on-profile: development
import:
- "configserver:http://192.168.0.100:20085"
- "configserver:http://192.168.0.100:11080"
server:

4
processor/src/main/resources/application.yml
View File

@ -8,7 +8,7 @@ spring:
group:
local: "local,datasource-local,webclient-local"
development: "development,datasource-development,webclient-development"
production: "production, datasource-production,webclient-production"
production: "production,datasource-production,webclient-production"
freemarker:
enabled: false
@ -22,4 +22,4 @@ management:
endpoints:
web:
exposure:
include: refresh
include: refresh,health

8
receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/scheduler/ParseEventEmitter.java
View File

@ -1,5 +1,7 @@
package com.myoa.engineering.crawl.ppomppu.receiver.scheduler;
import java.util.Arrays;
import org.springframework.scheduling.annotation.EnableScheduling;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
@ -29,8 +31,8 @@ public class ParseEventEmitter {
@Scheduled(fixedRate = 300 * 1000L)
public void emitBoards() {
log.info("[emitDomesticBoard] trigger fired!");
for (PpomppuBoardName boardName : PpomppuBoardName.values()) {
processorAPIService.emitParseEvent(boardName).block();
}
Arrays.stream(PpomppuBoardName.values())
.filter(PpomppuBoardName::isCrawlWithDefaultTimer)
.forEach(boardName -> processorAPIService.emitParseEvent(boardName).block());
}
}

1
receiver/src/main/resources/application-production.yml
View File

@ -3,5 +3,4 @@ spring:
activate:
on-profile: production
import:
- classpath:/production/webclient.yml
- "configserver:http://ppn-config-server:20080"

2
receiver/src/main/resources/application.yml
View File

@ -22,4 +22,4 @@ management:
endpoints:
web:
exposure:
include: refresh
include: refresh,health

34
sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/controller/MessageSenderAPIController.java
View File

@ -1,13 +1,20 @@
package com.myoa.engineering.crawl.ppomppu.sender.controller;
import com.myoa.engineering.crawl.ppomppu.sender.infrastructure.client.MongeShoppingBotSlackMessageSender;
import com.myoa.engineering.crawl.ppomppu.support.dto.APIResponse;
import com.myoa.engineering.crawl.ppomppu.support.dto.SimpleMessageDTO;
import lombok.extern.slf4j.Slf4j;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import com.myoa.engineering.crawl.ppomppu.sender.dto.SlackBaseMessageBlock;
import com.myoa.engineering.crawl.ppomppu.sender.dto.SlackMessageDTO;
import com.myoa.engineering.crawl.ppomppu.sender.dto.SlackSectionMessageBlock;
import com.myoa.engineering.crawl.ppomppu.sender.infrastructure.client.MongeShoppingBotSlackMessageSender;
import com.myoa.engineering.crawl.ppomppu.support.dto.APIResponse;
import com.myoa.engineering.crawl.ppomppu.support.dto.BlockMessageDTO;
import com.myoa.engineering.crawl.ppomppu.support.dto.SimpleMessageDTO;
import com.myoa.engineering.crawl.ppomppu.support.util.ObjectMapperFactory;
import lombok.extern.slf4j.Slf4j;
import reactor.core.publisher.Mono;
/**
@ -27,9 +34,24 @@ public class MessageSenderAPIController {
this.sender = sender;
}
@PostMapping("/messages/sendMessage/messengers/slack")
public Mono<APIResponse<SimpleMessageDTO>> sendMessageToSlack(@RequestBody SimpleMessageDTO dto) {
@PostMapping("/messages/sendSimpleMessage/messengers/slack")
public Mono<APIResponse<SimpleMessageDTO>> sendSimpleMessageToSlack(@RequestBody SimpleMessageDTO dto) {
return sender.sendMessage(sender.ofMessage(dto.getBody()))
.then(Mono.just(APIResponse.success(dto)));
}
@PostMapping("/messages/sendBlockMessage/messengers/slack")
public Mono<APIResponse<BlockMessageDTO>> sendBlockMessageToSlack(@RequestBody BlockMessageDTO dto) {
if (dto.getBlocks().isEmpty()) {
return Mono.just(APIResponse.fail(dto, "empty blocks"));
}
SlackMessageDTO slackMessageDTO = sender.ofBlockMessageBased();
dto.getBlocks().forEach(slackMessageDTO::addBlock);
slackMessageDTO.addBlock(SlackBaseMessageBlock.ofDivider());
return sender.sendMessage(slackMessageDTO)
.doOnNext(e -> log.info("[sendBlockMessageToSlack] slackMessageDTO: {}",
ObjectMapperFactory.writeAsString(slackMessageDTO)))
.then(Mono.just(APIResponse.success(dto)));
}
}

24
sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/dto/SlackBaseMessageBlock.java
View File

@ -1,7 +1,5 @@
package com.myoa.engineering.crawl.ppomppu.sender.dto;
import java.io.Serializable;
import com.fasterxml.jackson.annotation.JsonInclude;
import lombok.Builder;
@ -17,29 +15,29 @@ import lombok.NoArgsConstructor;
@Getter
@NoArgsConstructor
@JsonInclude(JsonInclude.Include.NON_NULL)
public class SlackMessageBlock implements Serializable {
public class SlackBaseMessageBlock implements SlackMessageBlock {
private static final long serialVersionUID = 1597984001727808419L;
private SlackMessageBlockType type;
private String text;
@Builder
private SlackMessageBlock(SlackMessageBlockType type, String text) {
private SlackBaseMessageBlock(SlackMessageBlockType type, String text) {
this.type = type;
this.text = text;
}
public static SlackMessageBlock ofMarkDown(String message) {
return SlackMessageBlock.builder()
.type(SlackMessageBlockType.MARKDOWN)
.text(message)
.build();
public static SlackBaseMessageBlock ofMarkDown(String message) {
return SlackBaseMessageBlock.builder()
.type(SlackMessageBlockType.MARKDOWN)
.text(message)
.build();
}
public static SlackMessageBlock ofDivider() {
return SlackMessageBlock.builder()
.type(SlackMessageBlockType.DIVIDER)
.build();
public static SlackBaseMessageBlock ofDivider() {
return SlackBaseMessageBlock.builder()
.type(SlackMessageBlockType.DIVIDER)
.build();
}
public String getType() {

8
sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/dto/SlackMessageBlock.java
View File

@ -1,8 +1,12 @@
package com.myoa.engineering.crawl.ppomppu.sender.dto;/**
package com.myoa.engineering.crawl.ppomppu.sender.dto;
import java.io.Serializable;
/**
* SlackMessageBlock
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-12-01
*
*/
public interface SlackMessageBlock {
public interface SlackMessageBlock extends Serializable {
}

2
sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/dto/SlackMessageBlockType.java
View File

@ -11,7 +11,7 @@ import lombok.Getter;
*/
@Getter
@AllArgsConstructor
public enum BlockType {
public enum SlackMessageBlockType {
SECTION("section"),
MARKDOWN("mrkdwn"),
DIVIDER("divider"),

18
sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/dto/SlackMessageDTO.java
View File

@ -1,5 +1,8 @@
package com.myoa.engineering.crawl.ppomppu.sender.dto;
import java.util.List;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.Builder;
@ -14,6 +17,7 @@ import lombok.NoArgsConstructor;
*/
@Getter
@NoArgsConstructor
@JsonInclude(JsonInclude.Include.NON_NULL)
public class SlackMessageDTO implements MessageDTO {
private final static long serialVersionUID = 4737608709660494713L;
@ -21,19 +25,31 @@ public class SlackMessageDTO implements MessageDTO {
private String text;
private String channel;
private String username;
private List<SlackMessageBlock> blocks;
@JsonProperty("icon_emoji")
private String iconEmoji;
@Builder
public SlackMessageDTO(String text, String channel, String username, String iconEmoji) {
public SlackMessageDTO(String text, String channel, String username,
List<SlackMessageBlock> blocks, String iconEmoji) {
this.text = text;
this.channel = channel;
this.username = username;
this.blocks = blocks;
this.iconEmoji = iconEmoji;
}
public void applyText(String text) {
this.text = text;
}
public void addBlock(String blockRawMessage) {
addBlock(SlackSectionMessageBlock.ofMarkDown(blockRawMessage));
}
public void addBlock(SlackMessageBlock block) {
blocks.add(block);
}
}

25
sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/dto/SlackSectionMessageBlock.java
View File

@ -1,7 +1,5 @@
package com.myoa.engineering.crawl.ppomppu.sender.dto;
import java.io.Serializable;
import com.fasterxml.jackson.annotation.JsonInclude;
import lombok.Builder;
@ -17,30 +15,23 @@ import lombok.NoArgsConstructor;
@Getter
@NoArgsConstructor
@JsonInclude(JsonInclude.Include.NON_NULL)
public class SlackSectionBlock implements Serializable {
public class SlackSectionMessageBlock implements SlackMessageBlock {
private static final long serialVersionUID = -7600944576753160168L;
private SlackMessageBlockType type;
private SlackMessageBlock text;
private SlackBaseMessageBlock text;
@Builder
private SlackSectionBlock(SlackMessageBlockType type, SlackMessageBlock text) {
private SlackSectionMessageBlock(SlackMessageBlockType type, SlackBaseMessageBlock text) {
this.type = type;
this.text = text;
}
public static SlackSectionBlock ofMarkDown(String message) {
return SlackSectionBlock.builder()
.type(SlackMessageBlockType.SECTION)
.text(SlackMessageBlock.ofMarkDown(message))
.build();
}
public static SlackSectionBlock ofDivider() {
return SlackSectionBlock.builder()
.type(SlackMessageBlockType.SECTION)
.text(SlackMessageBlock.ofDivider())
.build();
public static SlackSectionMessageBlock ofMarkDown(String message) {
return SlackSectionMessageBlock.builder()
.type(SlackMessageBlockType.SECTION)
.text(SlackBaseMessageBlock.ofMarkDown(message))
.build();
}
public String getType() {

16
sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/infrastructure/client/MongeShoppingBotSlackMessageSender.java
View File

@ -1,10 +1,14 @@
package com.myoa.engineering.crawl.ppomppu.sender.infrastructure.client;
import java.util.ArrayList;
import org.springframework.stereotype.Component;
import com.myoa.engineering.crawl.ppomppu.sender.configuration.properties.SlackSecretProperties;
import com.myoa.engineering.crawl.ppomppu.sender.configuration.properties.SlackSecretProperties.SlackSecretPropertiesUnit;
import com.myoa.engineering.crawl.ppomppu.sender.dto.SlackMessageDTO;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
@Slf4j
@Component
@ -35,4 +39,14 @@ public class MongeShoppingBotSlackMessageSender extends SlackMessageSender {
.text(text)
.build();
}
public SlackMessageDTO ofBlockMessageBased() {
return SlackMessageDTO.builder()
.channel(slackProperties.getChannel())
.iconEmoji(slackProperties.getIconEmoji())
.username(slackProperties.getUsername())
.blocks(new ArrayList<>())
.build();
}
}

4
sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/infrastructure/client/SlackMessageSender.java
View File

@ -33,6 +33,7 @@ public class SlackMessageSender implements MessageSender<SlackMessageDTO> {
.exchangeStrategies(WebFluxExchangeStragiesFactory.ofDefault())
.defaultHeader(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_UTF8_VALUE)
.defaultHeader(HttpHeaders.ACCEPT_CHARSET, "UTF-8")
.filter(WebClientFilterFactory.logRequest())
.filter(WebClientFilterFactory.logResponse())
.build();
}
@ -47,7 +48,8 @@ public class SlackMessageSender implements MessageSender<SlackMessageDTO> {
.onErrorResume(WebClientRequestException.class, t -> {
log.info("Exception occured, ignoring. : {}", t.getClass().getSimpleName());
return Mono.empty();
});
})
.doOnNext(e -> log.info("[sendMessage] {}", e));
}
}

2
sender/src/main/resources/application.yml
View File

@ -22,4 +22,4 @@ management:
endpoints:
web:
exposure:
include: refresh, env
include: refresh,health

51
support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/dto/BlockMessageDTO.java
View File

@ -2,36 +2,37 @@
import java.io.Serializable;
import java.time.Instant;
import java.util.List;
import lombok.Builder;
import lombok.Getter;
import lombok.NoArgsConstructor;
/**
* SimpleMessageDTO
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-11-21
*
*/
@Getter
@NoArgsConstructor
public class SimpleMessageDTO implements Serializable {
/**
* SimpleMessageDTO
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-11-21
*
*/
@Getter
@NoArgsConstructor
public class BlockMessageDTO implements Serializable {
private static final long serialVersionUID = 2203955567672404428L;
private static final long serialVersionUID = -6992039884035135523L;
private String title;
private String body;
private String url;
private Instant publishedAt;
private Instant requestedAt;
private String title;
private List<String> blocks;
private String url;
private Instant publishedAt;
private Instant requestedAt;
@Builder
public SimpleMessageDTO(String title, String body, String url, Instant publishedAt, Instant requestedAt) {
this.title = title;
this.body = body;
this.url = url;
this.publishedAt = publishedAt;
this.requestedAt = requestedAt;
}
}
@Builder
public BlockMessageDTO(String title, List<String> blocks, String url, Instant publishedAt,
Instant requestedAt) {
this.title = title;
this.blocks = blocks;
this.url = url;
this.publishedAt = publishedAt;
this.requestedAt = requestedAt;
}
}

48
support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/dto/code/PpomppuBoardName.java
View File

@ -12,30 +12,30 @@ import lombok.Getter;
@Getter
@AllArgsConstructor
public enum PpomppuBoardName {
PPOMPPU_DOMESTIC_ALL("/zboard/zboard.php?id=ppomppu", "전체", false),
PPOMPPU_DOMESTIC_ETC("/zboard/zboard.php?id=ppomppu&category=1", "기타", true),
PPOMPPU_DOMESTIC_COMPUTER("/zboard/zboard.php?id=ppomppu&category=4", "컴퓨터", true),
PPOMPPU_DOMESTIC_DIGITAL("/zboard/zboard.php?id=ppomppu&category=5", "디지털", true),
PPOMPPU_DOMESTIC_FOOD("/zboard/zboard.php?id=ppomppu&category=6", "식품/건강", true),
PPOMPPU_DOMESTIC_BOOK("/zboard/zboard.php?id=ppomppu&category=8", "서적", true),
PPOMPPU_DOMESTIC_APPLIANCES("/zboard/zboard.php?id=ppomppu&category=9", "가전/가구", true),
PPOMPPU_DOMESTIC_PARENTING("/zboard/zboard.php?id=ppomppu&category=10", "육아", true),
PPOMPPU_DOMESTIC_GIFTCARD("/zboard/zboard.php?id=ppomppu&category=11", "상품권", true),
PPOMPPU_DOMESTIC_CLOTHES("/zboard/zboard.php?id=ppomppu&category=12", "의류/잡화", true),
PPOMPPU_DOMESTIC_COSMETIC("/zboard/zboard.php?id=ppomppu&category=13", "화장품", true),
PPOMPPU_DOMESTIC_OUTDOOR("/zboard/zboard.php?id=ppomppu&category=15", "등산/캠핑", true),
PPOMPPU_OVERSEA_ALL("/zboard/zboard.php?id=ppomppu4", "전체", false),
PPOMPPU_OVERSEA_ETC("/zboard/zboard.php?id=ppomppu4&category=1", "기타", true),
PPOMPPU_OVERSEA_APPLIANCES("/zboard/zboard.php?id=ppomppu4&category=7", "가전", true),
PPOMPPU_OVERSEA_TVAV("/zboard/zboard.php?id=ppomppu4&category=8", "TV/영상", true),
PPOMPPU_OVERSEA_COMPUTER("/zboard/zboard.php?id=ppomppu4&category=3", "컴퓨터", true),
PPOMPPU_OVERSEA_DIGITAL("/zboard/zboard.php?id=ppomppu4&category=4", "디지털", true),
PPOMPPU_OVERSEA_MOBILEACCESSORY("/zboard/zboard.php?id=ppomppu4&category=9", "액세서리", true),
PPOMPPU_OVERSEA_CLOTHES("/zboard/zboard.php?id=ppomppu4&category=5", "의류/잡화", true),
PPOMPPU_OVERSEA_WATCH("/zboard/zboard.php?id=ppomppu4&category=2", "시계", true),
PPOMPPU_OVERSEA_SHOES("/zboard/zboard.php?id=ppomppu4&category=11", "신발", true),
PPOMPPU_OVERSEA_FOOD("/zboard/zboard.php?id=ppomppu4&category=10", "식품/건강", true),
PPOMPPU_OVERSEA_PARENTING("/zboard/zboard.php?id=ppomppu4&category=6", "육아", true),
PPOMPPU_DOMESTIC_ALL("/zboard/zboard.php?id=ppomppu", "국내-전체", true),
PPOMPPU_DOMESTIC_ETC("/zboard/zboard.php?id=ppomppu&category=1", "국내-기타", false),
PPOMPPU_DOMESTIC_COMPUTER("/zboard/zboard.php?id=ppomppu&category=4", "국내-컴퓨터", false),
PPOMPPU_DOMESTIC_DIGITAL("/zboard/zboard.php?id=ppomppu&category=5", "국내-디지털", false),
PPOMPPU_DOMESTIC_FOOD("/zboard/zboard.php?id=ppomppu&category=6", "국내-식품/건강", false),
PPOMPPU_DOMESTIC_BOOK("/zboard/zboard.php?id=ppomppu&category=8", "국내-서적", false),
PPOMPPU_DOMESTIC_APPLIANCES("/zboard/zboard.php?id=ppomppu&category=9", "국내-가전/가구", false),
PPOMPPU_DOMESTIC_PARENTING("/zboard/zboard.php?id=ppomppu&category=10", "국내-육아", false),
PPOMPPU_DOMESTIC_GIFTCARD("/zboard/zboard.php?id=ppomppu&category=11", "국내-상품권", false),
PPOMPPU_DOMESTIC_CLOTHES("/zboard/zboard.php?id=ppomppu&category=12", "국내-의류/잡화", false),
PPOMPPU_DOMESTIC_COSMETIC("/zboard/zboard.php?id=ppomppu&category=13", "국내-화장품", false),
PPOMPPU_DOMESTIC_OUTDOOR("/zboard/zboard.php?id=ppomppu&category=15", "국내-등산/캠핑", false),
PPOMPPU_OVERSEA_ALL("/zboard/zboard.php?id=ppomppu4", "해외-전체", true),
PPOMPPU_OVERSEA_ETC("/zboard/zboard.php?id=ppomppu4&category=1", "해외-기타", false),
PPOMPPU_OVERSEA_APPLIANCES("/zboard/zboard.php?id=ppomppu4&category=7", "해외-가전", false),
PPOMPPU_OVERSEA_TVAV("/zboard/zboard.php?id=ppomppu4&category=8", "해외-TV/영상", false),
PPOMPPU_OVERSEA_COMPUTER("/zboard/zboard.php?id=ppomppu4&category=3", "해외-컴퓨터", false),
PPOMPPU_OVERSEA_DIGITAL("/zboard/zboard.php?id=ppomppu4&category=4", "해외-디지털", false),
PPOMPPU_OVERSEA_MOBILEACCESSORY("/zboard/zboard.php?id=ppomppu4&category=9", "해외-액세서리", false),
PPOMPPU_OVERSEA_CLOTHES("/zboard/zboard.php?id=ppomppu4&category=5", "해외-의류/잡화", false),
PPOMPPU_OVERSEA_WATCH("/zboard/zboard.php?id=ppomppu4&category=2", "해외-시계", false),
PPOMPPU_OVERSEA_SHOES("/zboard/zboard.php?id=ppomppu4&category=11", "해외-신발", false),
PPOMPPU_OVERSEA_FOOD("/zboard/zboard.php?id=ppomppu4&category=10", "해외-식품/건강", false),
PPOMPPU_OVERSEA_PARENTING("/zboard/zboard.php?id=ppomppu4&category=6", "해외-육아", false),
;
private String resourcePath;

10
support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/util/ObjectMapperFactory.java
View File

@ -2,6 +2,7 @@ package com.myoa.engineering.crawl.ppomppu.support.util;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.core.JsonParser.Feature;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
@ -58,6 +59,15 @@ public final class ObjectMapperFactory {
return objectMapper;
}
public static String writeAsString(Object o) {
try {
return defaultMapper().writeValueAsString(o);
} catch (JsonProcessingException e) {
e.printStackTrace();
return null;
}
}
/**
* Copy from {@link GenericJackson2JsonRedisSerializer.NullValueSerializer}.