MyoaEngineering/ShoppingCrawler
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Apply accessory image

Browse Source
This commit is contained in:
woo-jin.shin
2021-12-04 01:07:42 +09:00
parent 7b230fdb74
commit 520a651a70
12 changed files with 158 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/controller/CrawlAPIController.java
View File

@@ -1,10 +1,13 @@
package com.myoa.engineering.crawl.ppomppu.processor.controller;
import java.util.List;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
import com.myoa.engineering.crawl.ppomppu.processor.dto.FeedParsedResult;
import com.myoa.engineering.crawl.ppomppu.processor.service.MessageSenderService;
import com.myoa.engineering.crawl.ppomppu.processor.service.PpomppuArticleService;
@@ -53,4 +56,15 @@ public class CrawlAPIController {
return publishedMessages.then(Mono.just(APIResponse.success(result.done())));
}
@PostMapping("/exploit/boards/{boardName}")
public Mono<APIResponse<String>> crawlBoardDryRun(
@PathVariable("boardName") PpomppuBoardName boardName) {
log.info("got request... {}", boardName);
Mono<String> publishedMessages =
ppomppuRSSFeedService.getArticles(boardName)
.flatMap(messageSenderService::sendBlockMessageToSlack);
return publishedMessages.map(APIResponse::success);
}
}

7
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/PpomppuArticle.java
View File

@@ -34,6 +34,9 @@ public class PpomppuArticle extends Auditable {
@Column
private String articleUrl;
@Column
private String thumbnailUrl;
@Column
private String title;
@@ -48,11 +51,13 @@ public class PpomppuArticle extends Auditable {
@Builder
public PpomppuArticle(Long id, Long articleId, PpomppuBoardName boardName, String articleUrl,
String title, Integer recommended, Integer hit, Instant registeredAt) {
String thumbnailUrl, String title, Integer recommended, Integer hit,
Instant registeredAt) {
this.id = id;
this.articleId = articleId;
this.boardName = boardName;
this.articleUrl = articleUrl;
this.thumbnailUrl = thumbnailUrl;
this.title = title;
this.recommended = recommended;
this.hit = hit;

6
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/PpomppuArticleParser.java
View File

@@ -27,6 +27,7 @@ public final class PpomppuArticleParser {
final long articleId = PpomppuArticleParser.parseArticleId(articleElement.get(0));
final String title = PpomppuArticleParser.parseTitle(articleElement.get(2));
final String articleUrl = PpomppuArticleParser.parseArticleUrl(articleElement.get(2));
final String thumbnailUrl = PpomppuArticleParser.parseThumbnailUrl(articleElement.get(3));
final int recommended = PpomppuArticleParser.parseRecommended(articleElement.get(6));
final int hit = PpomppuArticleParser.parseHit(articleElement.get(7));
final Instant registeredAt = PpomppuArticleParser.parseRegisteredAt(articleElement.get(5));
@@ -35,6 +36,7 @@ public final class PpomppuArticleParser {
.articleId(articleId)
.title(title)
.articleUrl(articleUrl)
.thumbnailUrl(thumbnailUrl)
.recommended(recommended)
.hit(hit)
.registeredAt(registeredAt)
@@ -53,6 +55,10 @@ public final class PpomppuArticleParser {
return PpomppuBoardName.ofViewPageUrl(td.getElementsByTag("a").attr("href"));
}
public static String parseThumbnailUrl(Element td) {
return "https:" + td.getElementsByTag("img").get(0).attr("src");
}
public static Integer parseRecommended(Element td) {
final String voteString = td.text();
final int recommended;

18
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/PpomppuArticleTransformer.java
View File

@@ -31,7 +31,8 @@ public final class PpomppuArticleTransformer {
.requestedAt(Instant.now())
.publishedAt(article.getRegisteredAt())
.title(String.format(MESSAGE_FORMAT_V1,
article.getBoardName().getMenuName(), article.getArticleUrl(), article.getTitle()))
article.getBoardName().getMenuName(), article.getArticleUrl(),
article.getTitle()))
.body(article.getArticleUrl())
.build();
@@ -50,14 +51,15 @@ public final class PpomppuArticleTransformer {
public static BlockMessageDTO transformToBlockMessage(List<PpomppuArticle> articles) {
Instant requestedAt = Instant.now();
List<String> body = articles.stream()
.map(PpomppuArticleTransformer::convertToInlineMessage)
.collect(Collectors.toList());
List<BlockMessageDTO.Block> body = articles.stream()
.map(e -> BlockMessageDTO.createBlock(convertToInlineMessage(e),
e.getThumbnailUrl()))
.collect(Collectors.toList());
return BlockMessageDTO.builder()
.requestedAt(requestedAt)
.title(DATE_TIME_FORMATTER.format(requestedAt))
.blocks(body)
.build();
.requestedAt(requestedAt)
.title(DATE_TIME_FORMATTER.format(requestedAt))
.blocks(body)
.build();
}
public static String convertToInlineMessage(PpomppuArticle article) {