MyoaEngineering
/
ShoppingCrawler
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Prettify section message

This commit is contained in:
woo-jin.shin 2021-12-04 01:29:32 +09:00
parent 520a651a70
commit 8eb431a812
8 changed files with 60 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/controller/CrawlAPIController.java
View File

@ -51,7 +51,7 @@ public class CrawlAPIController {
.map(e -> ppomppuArticleService.filterOnlyNewArticles(boardName, e)) .map(e -> ppomppuArticleService.filterOnlyNewArticles(boardName, e))
.map(e -> ppomppuArticleService.save(boardName, e)) .map(e -> ppomppuArticleService.save(boardName, e))
.filter(e -> !e.isEmpty()) .filter(e -> !e.isEmpty())
.flatMap(messageSenderService::sendBlockMessageToSlack); .flatMap(e -> messageSenderService.sendBlockMessageToSlack(boardName, e));
return publishedMessages.then(Mono.just(APIResponse.success(result.done()))); return publishedMessages.then(Mono.just(APIResponse.success(result.done())));
} }
@ -62,7 +62,7 @@ public class CrawlAPIController {
log.info("got request... {}", boardName); log.info("got request... {}", boardName);
Mono<String> publishedMessages = Mono<String> publishedMessages =
ppomppuRSSFeedService.getArticles(boardName) ppomppuRSSFeedService.getArticles(boardName)
.flatMap(messageSenderService::sendBlockMessageToSlack); .flatMap(e -> messageSenderService.sendBlockMessageToSlack(boardName, e));
return publishedMessages.map(APIResponse::success); return publishedMessages.map(APIResponse::success);
} }

2
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/PpomppuArticleParser.java
View File

@ -48,7 +48,7 @@ public final class PpomppuArticleParser {
} }
public static String parseTitle(Element td) { public static String parseTitle(Element td) {
return td.getElementsByTag("a").text(); // TODO cdn image extracting return td.getElementsByTag("a").text();
} }
public static String parseArticleUrl(Element td) { public static String parseArticleUrl(Element td) {

11
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/PpomppuArticleTransformer.java
View File

@ -10,6 +10,7 @@ import java.util.stream.Collectors;
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle; import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
import com.myoa.engineering.crawl.ppomppu.support.dto.BlockMessageDTO; import com.myoa.engineering.crawl.ppomppu.support.dto.BlockMessageDTO;
import com.myoa.engineering.crawl.ppomppu.support.dto.SimpleMessageDTO; import com.myoa.engineering.crawl.ppomppu.support.dto.SimpleMessageDTO;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
/** /**
* PpomppuArticleTransformer * PpomppuArticleTransformer
@ -22,6 +23,8 @@ public final class PpomppuArticleTransformer {
private PpomppuArticleTransformer() {} private PpomppuArticleTransformer() {}
private static final String MESSAGE_FORMAT_V1 = "%s)) <%s|LINK> `%s` "; private static final String MESSAGE_FORMAT_V1 = "%s)) <%s|LINK> `%s` ";
private static final String MESSAGE_FORMAT_V2 = "%s *<%s|LINK>*\n%s";
private static final String TITLE_FORMAT_V1 = "_*:hearts: %s | %s*_";
private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss") private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")
.withZone(ZoneId.of("Asia/Seoul")); .withZone(ZoneId.of("Asia/Seoul"));
@ -49,7 +52,7 @@ public final class PpomppuArticleTransformer {
.build(); .build();
} }
public static BlockMessageDTO transformToBlockMessage(List<PpomppuArticle> articles) { public static BlockMessageDTO transformToBlockMessage(PpomppuBoardName boardName, List<PpomppuArticle> articles) {
Instant requestedAt = Instant.now(); Instant requestedAt = Instant.now();
List<BlockMessageDTO.Block> body = articles.stream() List<BlockMessageDTO.Block> body = articles.stream()
.map(e -> BlockMessageDTO.createBlock(convertToInlineMessage(e), .map(e -> BlockMessageDTO.createBlock(convertToInlineMessage(e),
@ -57,13 +60,15 @@ public final class PpomppuArticleTransformer {
.collect(Collectors.toList()); .collect(Collectors.toList());
return BlockMessageDTO.builder() return BlockMessageDTO.builder()
.requestedAt(requestedAt) .requestedAt(requestedAt)
.title(DATE_TIME_FORMATTER.format(requestedAt)) .title(String.format(TITLE_FORMAT_V1,
boardName.getMenuName(),
DATE_TIME_FORMATTER.format(requestedAt)))
.blocks(body) .blocks(body)
.build(); .build();
} }
public static String convertToInlineMessage(PpomppuArticle article) { public static String convertToInlineMessage(PpomppuArticle article) {
return String.format(MESSAGE_FORMAT_V1, return String.format(MESSAGE_FORMAT_V2,
article.getBoardName().getMenuName(), article.getArticleUrl(), article.getTitle()); article.getBoardName().getMenuName(), article.getArticleUrl(), article.getTitle());
} }
} }

6
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/MessageSenderService.java
View File

@ -7,6 +7,7 @@ import org.springframework.stereotype.Service;
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle; import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
import com.myoa.engineering.crawl.ppomppu.processor.dto.PpomppuArticleTransformer; import com.myoa.engineering.crawl.ppomppu.processor.dto.PpomppuArticleTransformer;
import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.client.MessageSenderAPIClient; import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.client.MessageSenderAPIClient;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import reactor.core.publisher.Mono; import reactor.core.publisher.Mono;
@ -35,8 +36,9 @@ public class MessageSenderService {
return messageSenderAPIClient.sendSimpleMessageToSlack(PpomppuArticleTransformer.transformToSimpleMessage(articles)); return messageSenderAPIClient.sendSimpleMessageToSlack(PpomppuArticleTransformer.transformToSimpleMessage(articles));
} }
public Mono<String> sendBlockMessageToSlack(List<PpomppuArticle> articles) { public Mono<String> sendBlockMessageToSlack(PpomppuBoardName boardName, List<PpomppuArticle> articles) {
return messageSenderAPIClient.sendBlockMessageToSlack(PpomppuArticleTransformer.transformToBlockMessage(articles)); return messageSenderAPIClient.sendBlockMessageToSlack(
PpomppuArticleTransformer.transformToBlockMessage(boardName, articles));
} }
} }

18
sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/controller/MessageSenderAPIController.java
View File

@ -44,13 +44,19 @@ public class MessageSenderAPIController {
if (dto.getBlocks().isEmpty()) { if (dto.getBlocks().isEmpty()) {
return Mono.just(APIResponse.fail(dto, "empty blocks")); return Mono.just(APIResponse.fail(dto, "empty blocks"));
} }
SlackMessageDTO slackMessageDTO = sender.ofBlockMessageBased();
dto.getBlocks().forEach(slackMessageDTO::addSectionBlock);
slackMessageDTO.addBlock(SlackBaseMessageBlock.ofDivider());
return sender.sendMessage(slackMessageDTO)
.doOnNext(e -> log.info("[sendBlockMessageToSlack] slackMessageDTO: {}", return sender.sendMessage(buildSlackMessageDTO(dto))
ObjectMapperFactory.writeAsString(slackMessageDTO))) // .doOnNext(e -> log.info("[sendBlockMessageToSlack] slackMessageDTO: {}",
// ObjectMapperFactory.writeAsString(buildSlackMessageDTO(dto))))
.then(Mono.just(APIResponse.success(dto))); .then(Mono.just(APIResponse.success(dto)));
} }
private SlackMessageDTO buildSlackMessageDTO(BlockMessageDTO dto) {
SlackMessageDTO slackMessageDTO = sender.ofBlockMessageBased();
slackMessageDTO.addSectionBlock(dto.getTitle());
dto.getBlocks().forEach(slackMessageDTO::addSectionBlock);
slackMessageDTO.addBlock(SlackBaseMessageBlock.ofDivider());
return slackMessageDTO;
}
} }

3
sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/dto/SlackMessageDTO.java
View File

@ -51,6 +51,9 @@ public class SlackMessageDTO implements MessageDTO {
slackSectionMessageBlock.applyImageaccessory(block.getImageUrl(), block.getAltText()); slackSectionMessageBlock.applyImageaccessory(block.getImageUrl(), block.getAltText());
addBlock(slackSectionMessageBlock); addBlock(slackSectionMessageBlock);
} }
public void addSectionBlock(String rawBlockMessage) {
addBlock(SlackSectionMessageBlock.ofMarkDown(rawBlockMessage));
}
public void addBlock(SlackMessageBlock block) { public void addBlock(SlackMessageBlock block) {
blocks.add(block); blocks.add(block);

7
support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/dto/BlockMessageDTO.java
View File

@ -4,6 +4,8 @@ import java.io.Serializable;
import java.time.Instant; import java.time.Instant;
import java.util.List; import java.util.List;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import lombok.Builder; import lombok.Builder;
import lombok.Getter; import lombok.Getter;
import lombok.NoArgsConstructor; import lombok.NoArgsConstructor;
@ -20,6 +22,7 @@ public class BlockMessageDTO implements Serializable {
private static final long serialVersionUID = -6992039884035135523L; private static final long serialVersionUID = -6992039884035135523L;
private PpomppuBoardName boardName;
private String title; private String title;
private List<Block> blocks; private List<Block> blocks;
private String url; private String url;
@ -27,8 +30,10 @@ public class BlockMessageDTO implements Serializable {
private Instant requestedAt; private Instant requestedAt;
@Builder @Builder
public BlockMessageDTO(String title, List<Block> blocks, String url, Instant publishedAt, public BlockMessageDTO(PpomppuBoardName boardName, String title,
List<Block> blocks, String url, Instant publishedAt,
Instant requestedAt) { Instant requestedAt) {
this.boardName = boardName;
this.title = title; this.title = title;
this.blocks = blocks; this.blocks = blocks;
this.url = url; this.url = url;

48
support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/dto/code/PpomppuBoardName.java
View File

@ -12,30 +12,30 @@ import lombok.Getter;
@Getter @Getter
@AllArgsConstructor @AllArgsConstructor
public enum PpomppuBoardName { public enum PpomppuBoardName {
PPOMPPU_DOMESTIC_ALL("/zboard/zboard.php?id=ppomppu", "국내-전체", true), PPOMPPU_DOMESTIC_ALL("/zboard/zboard.php?id=ppomppu", "국내-전체", false),
PPOMPPU_DOMESTIC_ETC("/zboard/zboard.php?id=ppomppu&category=1", "국내-기타", false), PPOMPPU_DOMESTIC_ETC("/zboard/zboard.php?id=ppomppu&category=1", "국내-기타", true),
PPOMPPU_DOMESTIC_COMPUTER("/zboard/zboard.php?id=ppomppu&category=4", "국내-컴퓨터", false), PPOMPPU_DOMESTIC_COMPUTER("/zboard/zboard.php?id=ppomppu&category=4", "국내-컴퓨터", true),
PPOMPPU_DOMESTIC_DIGITAL("/zboard/zboard.php?id=ppomppu&category=5", "국내-디지털", false), PPOMPPU_DOMESTIC_DIGITAL("/zboard/zboard.php?id=ppomppu&category=5", "국내-디지털", true),
PPOMPPU_DOMESTIC_FOOD("/zboard/zboard.php?id=ppomppu&category=6", "국내-식품/건강", false), PPOMPPU_DOMESTIC_FOOD("/zboard/zboard.php?id=ppomppu&category=6", "국내-식품/건강", true),
PPOMPPU_DOMESTIC_BOOK("/zboard/zboard.php?id=ppomppu&category=8", "국내-서적", false), PPOMPPU_DOMESTIC_BOOK("/zboard/zboard.php?id=ppomppu&category=8", "국내-서적", true),
PPOMPPU_DOMESTIC_APPLIANCES("/zboard/zboard.php?id=ppomppu&category=9", "국내-가전/가구", false), PPOMPPU_DOMESTIC_APPLIANCES("/zboard/zboard.php?id=ppomppu&category=9", "국내-가전/가구", true),
PPOMPPU_DOMESTIC_PARENTING("/zboard/zboard.php?id=ppomppu&category=10", "국내-육아", false), PPOMPPU_DOMESTIC_PARENTING("/zboard/zboard.php?id=ppomppu&category=10", "국내-육아", true),
PPOMPPU_DOMESTIC_GIFTCARD("/zboard/zboard.php?id=ppomppu&category=11", "국내-상품권", false), PPOMPPU_DOMESTIC_GIFTCARD("/zboard/zboard.php?id=ppomppu&category=11", "국내-상품권", true),
PPOMPPU_DOMESTIC_CLOTHES("/zboard/zboard.php?id=ppomppu&category=12", "국내-의류/잡화", false), PPOMPPU_DOMESTIC_CLOTHES("/zboard/zboard.php?id=ppomppu&category=12", "국내-의류/잡화", true),
PPOMPPU_DOMESTIC_COSMETIC("/zboard/zboard.php?id=ppomppu&category=13", "국내-화장품", false), PPOMPPU_DOMESTIC_COSMETIC("/zboard/zboard.php?id=ppomppu&category=13", "국내-화장품", true),
PPOMPPU_DOMESTIC_OUTDOOR("/zboard/zboard.php?id=ppomppu&category=15", "국내-등산/캠핑", false), PPOMPPU_DOMESTIC_OUTDOOR("/zboard/zboard.php?id=ppomppu&category=15", "국내-등산/캠핑", true),
PPOMPPU_OVERSEA_ALL("/zboard/zboard.php?id=ppomppu4", "해외-전체", true), PPOMPPU_OVERSEA_ALL("/zboard/zboard.php?id=ppomppu4", "해외-전체", false),
PPOMPPU_OVERSEA_ETC("/zboard/zboard.php?id=ppomppu4&category=1", "해외-기타", false), PPOMPPU_OVERSEA_ETC("/zboard/zboard.php?id=ppomppu4&category=1", "해외-기타", true),
PPOMPPU_OVERSEA_APPLIANCES("/zboard/zboard.php?id=ppomppu4&category=7", "해외-가전", false), PPOMPPU_OVERSEA_APPLIANCES("/zboard/zboard.php?id=ppomppu4&category=7", "해외-가전", true),
PPOMPPU_OVERSEA_TVAV("/zboard/zboard.php?id=ppomppu4&category=8", "해외-TV/영상", false), PPOMPPU_OVERSEA_TVAV("/zboard/zboard.php?id=ppomppu4&category=8", "해외-TV/영상", true),
PPOMPPU_OVERSEA_COMPUTER("/zboard/zboard.php?id=ppomppu4&category=3", "해외-컴퓨터", false), PPOMPPU_OVERSEA_COMPUTER("/zboard/zboard.php?id=ppomppu4&category=3", "해외-컴퓨터", true),
PPOMPPU_OVERSEA_DIGITAL("/zboard/zboard.php?id=ppomppu4&category=4", "해외-디지털", false), PPOMPPU_OVERSEA_DIGITAL("/zboard/zboard.php?id=ppomppu4&category=4", "해외-디지털", true),
PPOMPPU_OVERSEA_MOBILEACCESSORY("/zboard/zboard.php?id=ppomppu4&category=9", "해외-액세서리", false), PPOMPPU_OVERSEA_MOBILEACCESSORY("/zboard/zboard.php?id=ppomppu4&category=9", "해외-액세서리", true),
PPOMPPU_OVERSEA_CLOTHES("/zboard/zboard.php?id=ppomppu4&category=5", "해외-의류/잡화", false), PPOMPPU_OVERSEA_CLOTHES("/zboard/zboard.php?id=ppomppu4&category=5", "해외-의류/잡화", true),
PPOMPPU_OVERSEA_WATCH("/zboard/zboard.php?id=ppomppu4&category=2", "해외-시계", false), PPOMPPU_OVERSEA_WATCH("/zboard/zboard.php?id=ppomppu4&category=2", "해외-시계", true),
PPOMPPU_OVERSEA_SHOES("/zboard/zboard.php?id=ppomppu4&category=11", "해외-신발", false), PPOMPPU_OVERSEA_SHOES("/zboard/zboard.php?id=ppomppu4&category=11", "해외-신발", true),
PPOMPPU_OVERSEA_FOOD("/zboard/zboard.php?id=ppomppu4&category=10", "해외-식품/건강", false), PPOMPPU_OVERSEA_FOOD("/zboard/zboard.php?id=ppomppu4&category=10", "해외-식품/건강", true),
PPOMPPU_OVERSEA_PARENTING("/zboard/zboard.php?id=ppomppu4&category=6", "해외-육아", false), PPOMPPU_OVERSEA_PARENTING("/zboard/zboard.php?id=ppomppu4&category=6", "해외-육아", true),
; ;
private String resourcePath; private String resourcePath;