MyoaEngineering
/
ShoppingCrawler
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Apply accessory image

This commit is contained in:
woo-jin.shin 2021-12-04 01:07:42 +09:00
parent 7b230fdb74
commit 520a651a70
12 changed files with 158 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/controller/CrawlAPIController.java
View File

@ -1,10 +1,13 @@
package com.myoa.engineering.crawl.ppomppu.processor.controller;
import java.util.List;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
import com.myoa.engineering.crawl.ppomppu.processor.dto.FeedParsedResult;
import com.myoa.engineering.crawl.ppomppu.processor.service.MessageSenderService;
import com.myoa.engineering.crawl.ppomppu.processor.service.PpomppuArticleService;
@ -53,4 +56,15 @@ public class CrawlAPIController {
return publishedMessages.then(Mono.just(APIResponse.success(result.done())));
}
@PostMapping("/exploit/boards/{boardName}")
public Mono<APIResponse<String>> crawlBoardDryRun(
@PathVariable("boardName") PpomppuBoardName boardName) {
log.info("got request... {}", boardName);
Mono<String> publishedMessages =
ppomppuRSSFeedService.getArticles(boardName)
.flatMap(messageSenderService::sendBlockMessageToSlack);
return publishedMessages.map(APIResponse::success);
}
}

7
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/PpomppuArticle.java
View File

@ -34,6 +34,9 @@ public class PpomppuArticle extends Auditable {
@Column
private String articleUrl;
@Column
private String thumbnailUrl;
@Column
private String title;
@ -48,11 +51,13 @@ public class PpomppuArticle extends Auditable {
@Builder
public PpomppuArticle(Long id, Long articleId, PpomppuBoardName boardName, String articleUrl,
String title, Integer recommended, Integer hit, Instant registeredAt) {
String thumbnailUrl, String title, Integer recommended, Integer hit,
Instant registeredAt) {
this.id = id;
this.articleId = articleId;
this.boardName = boardName;
this.articleUrl = articleUrl;
this.thumbnailUrl = thumbnailUrl;
this.title = title;
this.recommended = recommended;
this.hit = hit;

6
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/PpomppuArticleParser.java
View File

@ -27,6 +27,7 @@ public final class PpomppuArticleParser {
final long articleId = PpomppuArticleParser.parseArticleId(articleElement.get(0));
final String title = PpomppuArticleParser.parseTitle(articleElement.get(2));
final String articleUrl = PpomppuArticleParser.parseArticleUrl(articleElement.get(2));
final String thumbnailUrl = PpomppuArticleParser.parseThumbnailUrl(articleElement.get(3));
final int recommended = PpomppuArticleParser.parseRecommended(articleElement.get(6));
final int hit = PpomppuArticleParser.parseHit(articleElement.get(7));
final Instant registeredAt = PpomppuArticleParser.parseRegisteredAt(articleElement.get(5));
@ -35,6 +36,7 @@ public final class PpomppuArticleParser {
.articleId(articleId)
.title(title)
.articleUrl(articleUrl)
.thumbnailUrl(thumbnailUrl)
.recommended(recommended)
.hit(hit)
.registeredAt(registeredAt)
@ -53,6 +55,10 @@ public final class PpomppuArticleParser {
return PpomppuBoardName.ofViewPageUrl(td.getElementsByTag("a").attr("href"));
}
public static String parseThumbnailUrl(Element td) {
return "https:" + td.getElementsByTag("img").get(0).attr("src");
}
public static Integer parseRecommended(Element td) {
final String voteString = td.text();
final int recommended;

8
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/PpomppuArticleTransformer.java
View File

@ -31,7 +31,8 @@ public final class PpomppuArticleTransformer {
.requestedAt(Instant.now())
.publishedAt(article.getRegisteredAt())
.title(String.format(MESSAGE_FORMAT_V1,
article.getBoardName().getMenuName(), article.getArticleUrl(), article.getTitle()))
article.getBoardName().getMenuName(), article.getArticleUrl(),
article.getTitle()))
.body(article.getArticleUrl())
.build();
@ -50,8 +51,9 @@ public final class PpomppuArticleTransformer {
public static BlockMessageDTO transformToBlockMessage(List<PpomppuArticle> articles) {
Instant requestedAt = Instant.now();
List<String> body = articles.stream()
.map(PpomppuArticleTransformer::convertToInlineMessage)
List<BlockMessageDTO.Block> body = articles.stream()
.map(e -> BlockMessageDTO.createBlock(convertToInlineMessage(e),
e.getThumbnailUrl()))
.collect(Collectors.toList());
return BlockMessageDTO.builder()
.requestedAt(requestedAt)

3
sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/controller/MessageSenderAPIController.java
View File

@ -7,7 +7,6 @@ import org.springframework.web.bind.annotation.RestController;
import com.myoa.engineering.crawl.ppomppu.sender.dto.SlackBaseMessageBlock;
import com.myoa.engineering.crawl.ppomppu.sender.dto.SlackMessageDTO;
import com.myoa.engineering.crawl.ppomppu.sender.dto.SlackSectionMessageBlock;
import com.myoa.engineering.crawl.ppomppu.sender.infrastructure.client.MongeShoppingBotSlackMessageSender;
import com.myoa.engineering.crawl.ppomppu.support.dto.APIResponse;
import com.myoa.engineering.crawl.ppomppu.support.dto.BlockMessageDTO;
@ -46,7 +45,7 @@ public class MessageSenderAPIController {
return Mono.just(APIResponse.fail(dto, "empty blocks"));
}
SlackMessageDTO slackMessageDTO = sender.ofBlockMessageBased();
dto.getBlocks().forEach(slackMessageDTO::addBlock);
dto.getBlocks().forEach(slackMessageDTO::addSectionBlock);
slackMessageDTO.addBlock(SlackBaseMessageBlock.ofDivider());
return sender.sendMessage(slackMessageDTO)

1
sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/dto/SlackBaseMessageBlock.java
View File

@ -40,6 +40,7 @@ public class SlackBaseMessageBlock implements SlackMessageBlock {
.build();
}
@Override
public String getType() {
return type.getType();
}

49
sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/dto/SlackImageMessageBlock.java Normal file
View File

@ -0,0 +1,49 @@
package com.myoa.engineering.crawl.ppomppu.sender.dto;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.Builder;
import lombok.Getter;
import lombok.NoArgsConstructor;
/**
* SlackImageMessageBlock
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-11-30
*
*/
@Getter
@NoArgsConstructor
@JsonInclude(JsonInclude.Include.NON_NULL)
public class SlackImageMessageBlock implements SlackMessageBlock {
private static final long serialVersionUID = 1597984001727808419L;
private SlackMessageBlockType type;
@JsonProperty(value = "image_url", required = true)
private String imageUrl;
@JsonProperty(value = "alt_text", required = true)
private String altText;
@Builder
private SlackImageMessageBlock(SlackMessageBlockType type, String imageUrl, String altText) {
this.type = type;
this.imageUrl = imageUrl;
this.altText = altText;
}
public static SlackImageMessageBlock of(String imageUrl, String altText) {
return SlackImageMessageBlock.builder()
.type(SlackMessageBlockType.IMAGE)
.imageUrl(imageUrl)
.altText(altText)
.build();
}
@Override
public String getType() {
return type.getType();
}
}

3
sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/dto/SlackMessageBlock.java
View File

@ -9,4 +9,7 @@ import java.io.Serializable;
*
*/
public interface SlackMessageBlock extends Serializable {
String getType();
}

1
sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/dto/SlackMessageBlockType.java
View File

@ -15,6 +15,7 @@ public enum SlackMessageBlockType {
SECTION("section"),
MARKDOWN("mrkdwn"),
DIVIDER("divider"),
IMAGE("image"),
;
private String type;

8
sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/dto/SlackMessageDTO.java
View File

@ -4,6 +4,8 @@ import java.util.List;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.myoa.engineering.crawl.ppomppu.support.dto.BlockMessageDTO;
import com.myoa.engineering.crawl.ppomppu.support.dto.BlockMessageDTO.Block;
import lombok.Builder;
import lombok.Getter;
@ -44,8 +46,10 @@ public class SlackMessageDTO implements MessageDTO {
this.text = text;
}
public void addBlock(String blockRawMessage) {
addBlock(SlackSectionMessageBlock.ofMarkDown(blockRawMessage));
public void addSectionBlock(BlockMessageDTO.Block block) {
SlackSectionMessageBlock slackSectionMessageBlock = SlackSectionMessageBlock.ofMarkDown(block.getText());
slackSectionMessageBlock.applyImageaccessory(block.getImageUrl(), block.getAltText());
addBlock(slackSectionMessageBlock);
}
public void addBlock(SlackMessageBlock block) {

11
sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/dto/SlackSectionMessageBlock.java
View File

@ -20,11 +20,14 @@ public class SlackSectionMessageBlock implements SlackMessageBlock {
private SlackMessageBlockType type;
private SlackBaseMessageBlock text;
private SlackImageMessageBlock accessory;
@Builder
private SlackSectionMessageBlock(SlackMessageBlockType type, SlackBaseMessageBlock text) {
private SlackSectionMessageBlock(SlackMessageBlockType type, SlackBaseMessageBlock text,
SlackImageMessageBlock accessory) {
this.type = type;
this.text = text;
this.accessory = accessory;
}
public static SlackSectionMessageBlock ofMarkDown(String message) {
@ -34,6 +37,12 @@ public class SlackSectionMessageBlock implements SlackMessageBlock {
.build();
}
public SlackSectionMessageBlock applyImageaccessory(String imageUrl, String altText) {
this.accessory = SlackImageMessageBlock.of(imageUrl, altText);
return this;
}
@Override
public String getType() {
return type.getType();
}

39
support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/dto/BlockMessageDTO.java
View File

@ -1,4 +1,4 @@
package com.myoa.engineering.crawl.ppomppu.support.dto;
package com.myoa.engineering.crawl.ppomppu.support.dto;
import java.io.Serializable;
import java.time.Instant;
@ -8,26 +8,26 @@ import lombok.Builder;
import lombok.Getter;
import lombok.NoArgsConstructor;
/**
/**
* SimpleMessageDTO
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-11-21
*
*/
@Getter
@NoArgsConstructor
public class BlockMessageDTO implements Serializable {
@Getter
@NoArgsConstructor
public class BlockMessageDTO implements Serializable {
private static final long serialVersionUID = -6992039884035135523L;
private String title;
private List<String> blocks;
private List<Block> blocks;
private String url;
private Instant publishedAt;
private Instant requestedAt;
@Builder
public BlockMessageDTO(String title, List<String> blocks, String url, Instant publishedAt,
public BlockMessageDTO(String title, List<Block> blocks, String url, Instant publishedAt,
Instant requestedAt) {
this.title = title;
this.blocks = blocks;
@ -35,4 +35,29 @@ import lombok.NoArgsConstructor;
this.publishedAt = publishedAt;
this.requestedAt = requestedAt;
}
@Getter
@NoArgsConstructor
public static class Block implements Serializable {
private static final long serialVersionUID = 3633781631892663709L;
private String text;
private String imageUrl;
private String altText;
public Block(String text, String imageUrl, String altText) {
this.text = text;
this.imageUrl = imageUrl;
this.altText = altText;
}
}
public static Block createBlock(String text, String imageUrl) {
return new Block(text, imageUrl, "");
}
public static Block createBlock(String text, String imageUrl, String altText) {
return new Block(text, imageUrl, altText);
}
}