MyoaEngineering
/
ShoppingCrawler
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

[PPN-211113] Fix bug

This commit is contained in:
woo-jin.shin 2021-11-21 22:36:30 +09:00
parent c97c8dc01f
commit a0c3962e0d
13 changed files with 85 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/controller/CrawlAPIController.java
View File

@ -1,13 +1,11 @@
package com.myoa.engineering.crawl.ppomppu.processor.controller;
import java.util.List;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
import com.myoa.engineering.crawl.ppomppu.processor.dto.FeedParsedResult;
import com.myoa.engineering.crawl.ppomppu.processor.service.MessageSenderService;
import com.myoa.engineering.crawl.ppomppu.processor.service.PpomppuArticleService;
@ -45,13 +43,14 @@ public class CrawlAPIController {
public Mono<APIResponse<FeedParsedResult>> crawlBoard(@PathVariable("boardName") PpomppuBoardName boardName) {
log.info("got request... {}", boardName);
FeedParsedResult result = FeedParsedResult.of(boardName);
Mono<List<PpomppuArticle>> articles =
ppomppuRSSFeedService.getArticles(boardName)
.doOnNext(e -> ppomppuArticleService.filterOnlyNewArticles(boardName, e))
.doOnNext(e -> messageSenderService.sendMessageToSlack(e))
.doOnNext(e -> ppomppuArticleService.save(boardName, e));
return articles.then(Mono.just(APIResponse.success(result.done())));
Mono<String> publishedMessages =
ppomppuRSSFeedService.getArticles(boardName)
.map(e -> ppomppuArticleService.filterOnlyNewArticles(boardName, e))
.map(e -> ppomppuArticleService.save(boardName, e))
.flatMap(messageSenderService::sendMessageToSlack);
return publishedMessages.then(Mono.just(APIResponse.success(result.done())));
}
}

2
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/PpomppuArticleParser.java
View File

@ -46,7 +46,7 @@ public final class PpomppuArticleParser {
}
public static String parseTitle(Element td) {
return td.text();
return td.getElementsByTag("a").text(); // TODO cdn image extracting
}
public static String parseArticleUrl(Element td) {

7
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/PpomppuArticleTransformer.java
View File

@ -17,8 +17,11 @@ import com.myoa.engineering.crawl.ppomppu.support.dto.SimpleMessageDTO;
*
*/
public final class PpomppuArticleTransformer {
private PpomppuArticleTransformer() {}
private static final String MESSAGE_FORMAT_V1 = "%s)) `%s` <%s:LINK>";
private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")
.withZone(ZoneId.of("Asia/Seoul"));
@ -26,7 +29,7 @@ public final class PpomppuArticleTransformer {
SimpleMessageDTO.builder()
.requestedAt(Instant.now())
.publishedAt(entity.getRegisteredAt())
.title(String.format("[%s] %s", entity.getBoardName().getMenuName(), entity.getTitle()))
.title(String.format(MESSAGE_FORMAT_V1, entity.getBoardName().getMenuName(), entity.getTitle()))
.body(entity.getArticleUrl())
.build();
@ -44,7 +47,7 @@ public final class PpomppuArticleTransformer {
}
public static String convertToInlineMessage(PpomppuArticle article) {
return String.format("[%s] %s\n%s",
return String.format(MESSAGE_FORMAT_V1,
article.getBoardName().getMenuName(), article.getTitle(), article.getArticleUrl());
}
}

21
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/client/MessageSenderAPIClient.java
View File

@ -1,18 +1,20 @@
package com.myoa.engineering.crawl.ppomppu.processor.infrastructure.client;
import org.springframework.core.ParameterizedTypeReference;
import org.springframework.http.HttpHeaders;
import org.springframework.http.MediaType;
import org.springframework.stereotype.Component;
import org.springframework.web.reactive.function.client.WebClient;
import org.springframework.web.reactive.function.client.WebClientRequestException;
import com.myoa.engineering.crawl.ppomppu.processor.dto.constant.WebClientPropertiesUnitName;
import com.myoa.engineering.crawl.ppomppu.support.dto.SimpleMessageDTO;
import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebClientFilterFactory;
import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebFluxExchangeStragiesFactory;
import com.myoa.engineering.crawl.ppomppu.support.webclient.properties.WebClientProperties;
import com.myoa.engineering.crawl.ppomppu.support.webclient.properties.WebClientProperties.WebClientPropertiesUnit;
import lombok.extern.slf4j.Slf4j;
import org.springframework.core.ParameterizedTypeReference;
import org.springframework.http.HttpHeaders;
import org.springframework.http.MediaType;
import org.springframework.stereotype.Component;
import org.springframework.web.reactive.function.client.WebClient;
import org.springframework.web.reactive.function.client.WebClientRequestException;
import reactor.core.publisher.Mono;
import reactor.core.scheduler.Schedulers;
@ -35,8 +37,8 @@ public class MessageSenderAPIClient {
.baseUrl(webClientPropertiesUnit.getBaseUrl())
.exchangeStrategies(WebFluxExchangeStragiesFactory.ofDefault())
.defaultHeader(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_VALUE)
.filter(WebClientFilterFactory.logRequest())
.filter(WebClientFilterFactory.logResponse())
// .filter(WebClientFilterFactory.logRequest())
// .filter(WebClientFilterFactory.logResponse())
.build();
}
@ -49,7 +51,6 @@ public class MessageSenderAPIClient {
.onErrorResume(WebClientRequestException.class, t -> {
log.info("Exception occured, ignoring. : {}", t.getClass().getSimpleName());
return Mono.empty();
})
.doOnNext(e -> log.info("response: {} ", e));
});
}
}

17
processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/PpomppuArticleService.java
View File

@ -1,16 +1,19 @@
package com.myoa.engineering.crawl.ppomppu.processor.service;
import java.util.List;
import java.util.Optional;
import java.util.stream.Collectors;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuBoardFeedStatus;
import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository.PpomppuArticleRepository;
import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository.PpomppuBoardFeedStatusRepository;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import java.util.List;
import java.util.Optional;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
@Slf4j
@Service
@ -32,6 +35,7 @@ public class PpomppuArticleService {
Long latestArticleId = boardFeedStatus.map(PpomppuBoardFeedStatus::getLatestParsedArticleId)
.orElse(0L);
log.info("latestArticleId : {}", latestArticleId);
return articles.stream()
.filter(e -> e.getArticleId().compareTo(latestArticleId) > 0)
.collect(Collectors.toList());
@ -46,9 +50,12 @@ public class PpomppuArticleService {
// save PpomppuBoardFeedStatus
Optional<PpomppuBoardFeedStatus> boardFeedStatus = ppomppuBoardFeedStatusRepository.findByBoardName(boardName);
log.info("boardName: {}, isPresent?: {}", boardName, boardFeedStatus.isPresent());
boardFeedStatus.ifPresentOrElse(e -> {
if (latestArticleId.longValue() > 0L) {
e.updateArticleId(latestArticleId);
ppomppuBoardFeedStatusRepository.save(e);
}
},
() -> ppomppuBoardFeedStatusRepository.save(PpomppuBoardFeedStatus.of(boardName,
latestArticleId)));

14
receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/scheduler/ParseEventEmitter.java
View File

@ -26,15 +26,11 @@ public class ParseEventEmitter {
this.processorAPIService = processorAPIService;
}
@Scheduled(fixedRate = 60 * 1000L)
public void emitDomesticBoard() {
log.info("[emitDomesticBoard] trigger fired!");
processorAPIService.emitParseEvent(PpomppuBoardName.PPOMPPU_DOMESTIC_ETC).block();
}
@Scheduled(fixedRate = 300 * 1000L)
public void emitOverseaBoard() {
log.info("[emitOverseaBoard] trigger fired!");
processorAPIService.emitParseEvent(PpomppuBoardName.PPOMPPU_OVERSEA_ETC).block();
public void emitBoards() {
log.info("[emitDomesticBoard] trigger fired!");
for (PpomppuBoardName boardName : PpomppuBoardName.values()) {
processorAPIService.emitParseEvent(boardName).block();
}
}
}

5
sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/controller/MessageSenderAPIController.java
View File

@ -5,6 +5,7 @@ import com.myoa.engineering.crawl.ppomppu.support.dto.APIResponse;
import com.myoa.engineering.crawl.ppomppu.support.dto.SimpleMessageDTO;
import lombok.extern.slf4j.Slf4j;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import reactor.core.publisher.Mono;
@ -27,9 +28,7 @@ public class MessageSenderAPIController {
}
@PostMapping("/messages/sendMessage/messengers/slack")
public Mono<APIResponse<SimpleMessageDTO>> sendMessageToSlack(SimpleMessageDTO dto) {
log.info("received : {}, \nbody: {}", dto.getTitle(), dto.getBody());
public Mono<APIResponse<SimpleMessageDTO>> sendMessageToSlack(@RequestBody SimpleMessageDTO dto) {
return sender.sendMessage(sender.ofMessage(dto.getBody()))
.then(Mono.just(APIResponse.success(dto)));
}

4
sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/controller/TestAPIController.java
View File

@ -4,6 +4,8 @@ import com.myoa.engineering.crawl.ppomppu.sender.infrastructure.client.MongeShop
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import lombok.extern.slf4j.Slf4j;
import reactor.core.publisher.Mono;
/**
@ -12,6 +14,7 @@ import reactor.core.publisher.Mono;
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-11-15
*/
@Slf4j
@RestController
@RequestMapping("/api/v1")
public class TestAPIController {
@ -24,6 +27,7 @@ public class TestAPIController {
@GetMapping("/test")
public Mono<String> test() {
log.info("received!!!");
return sender.sendMessage(sender.ofMessage("testtesttest!!!"));
}
}

1
sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/infrastructure/client/MongeShoppingBotSlackMessageSender.java
View File

@ -13,7 +13,6 @@ public class MongeShoppingBotSlackMessageSender extends SlackMessageSender {
private static final String SLACK_SECRET_UNIT_NAME = "monge_shopping_bot";
private final SlackSecretPropertiesUnit slackProperties;
// private final SlackMessageSender slackMessageSender;
public MongeShoppingBotSlackMessageSender(SlackSecretProperties slackSecretProperties) {
super(slackSecretProperties.find(SLACK_SECRET_UNIT_NAME).getToken());

4
sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/infrastructure/client/SlackMessageSender.java
View File

@ -33,7 +33,6 @@ public class SlackMessageSender implements MessageSender<SlackMessageDTO> {
.exchangeStrategies(WebFluxExchangeStragiesFactory.ofDefault())
.defaultHeader(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_UTF8_VALUE)
.defaultHeader(HttpHeaders.ACCEPT_CHARSET, "UTF-8")
.filter(WebClientFilterFactory.logRequest())
.filter(WebClientFilterFactory.logResponse())
.build();
}
@ -48,8 +47,7 @@ public class SlackMessageSender implements MessageSender<SlackMessageDTO> {
.onErrorResume(WebClientRequestException.class, t -> {
log.info("Exception occured, ignoring. : {}", t.getClass().getSimpleName());
return Mono.empty();
})
.doOnNext(e -> log.info("response: {}", e));
});
}
}

5
sender/src/main/resources/application.yml
View File

@ -12,11 +12,6 @@ spring:
freemarker:
enabled: false
jpa:
hibernate:
ddl-auto: create-drop
generate-ddl: true
server:
port: 20080
error:

31
support/next_message_tempalte(block_kit_builder).json Normal file
View File

@ -0,0 +1,31 @@
{
"blocks": [
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": "기타)) `[쿠팡] 타푸코 이중 진공 스텐 세라믹 코팅 텀블러 (화이트만 11,130원/로켓와우무료) 4 [기타]`<https://www.ppomppu.co.kr/zboardview.php?id=ppomppu&page=1&divpage=69&category=1&&no=404126|LINK>"
},
"accessory": {
"type": "image",
"image_url": "cdn.ppomppu.co.kr/zboard/data3/2021/1121/m_20211121184835_zfyahnow.png",
"alt_text": "alt text for image"
}
},
{
"type": "divider"
},
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": "기타)) `[롯데온] 1매 149원 올국산 KF94새부리마스크 대형 200매 (29,920원/무료배송) 58 [기타]`<https://www.ppomppu.co.kr/zboardview.php?id=ppomppu&page=1&divpage=69&category=1&&no=404113|LINK>"
},
"accessory": {
"type": "image",
"image_url": "https://s3-media3.fl.yelpcdn.com/bphoto/c7ed05m9lC2EmA3Aruue7A/o.jpg",
"alt_text": "alt text for image"
}
}
]
}

6
support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/dto/code/PpomppuBoardName.java
View File

@ -12,7 +12,7 @@ import lombok.Getter;
@Getter
@AllArgsConstructor
public enum PpomppuBoardName {
PPOMPPU_DOMESTIC_ALL("/zboard/zboard.php?id=ppomppu", "전체", true),
PPOMPPU_DOMESTIC_ALL("/zboard/zboard.php?id=ppomppu", "전체", false),
PPOMPPU_DOMESTIC_ETC("/zboard/zboard.php?id=ppomppu&category=1", "기타", true),
PPOMPPU_DOMESTIC_COMPUTER("/zboard/zboard.php?id=ppomppu&category=4", "컴퓨터", true),
PPOMPPU_DOMESTIC_DIGITAL("/zboard/zboard.php?id=ppomppu&category=5", "디지털", true),
@ -24,7 +24,7 @@ public enum PpomppuBoardName {
PPOMPPU_DOMESTIC_CLOTHES("/zboard/zboard.php?id=ppomppu&category=12", "의류/잡화", true),
PPOMPPU_DOMESTIC_COSMETIC("/zboard/zboard.php?id=ppomppu&category=13", "화장품", true),
PPOMPPU_DOMESTIC_OUTDOOR("/zboard/zboard.php?id=ppomppu&category=15", "등산/캠핑", true),
PPOMPPU_OVERSEA_ALL("/zboard/zboard.php?id=ppomppu4", "전체", true),
PPOMPPU_OVERSEA_ALL("/zboard/zboard.php?id=ppomppu4", "전체", false),
PPOMPPU_OVERSEA_ETC("/zboard/zboard.php?id=ppomppu4&category=1", "기타", true),
PPOMPPU_OVERSEA_APPLIANCES("/zboard/zboard.php?id=ppomppu4&category=7", "가전", true),
PPOMPPU_OVERSEA_TVAV("/zboard/zboard.php?id=ppomppu4&category=8", "TV/영상", true),
@ -45,7 +45,7 @@ public enum PpomppuBoardName {
public static final String PPOMPPU_URL = "https://www.ppomppu.co.kr";
public static String ofViewPageUrl(String articleUrl) {
return PPOMPPU_URL + "/zboard" + articleUrl;
return PPOMPPU_URL + "/zboard/" + articleUrl;
}
}