[PPN-211113] Fix bug

This commit is contained in:
woo-jin.shin 2021-11-21 22:36:30 +09:00
parent c97c8dc01f
commit a0c3962e0d
13 changed files with 85 additions and 53 deletions

View File

@ -1,13 +1,11 @@
package com.myoa.engineering.crawl.ppomppu.processor.controller; package com.myoa.engineering.crawl.ppomppu.processor.controller;
import java.util.List; import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController; import org.springframework.web.bind.annotation.RestController;
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
import com.myoa.engineering.crawl.ppomppu.processor.dto.FeedParsedResult; import com.myoa.engineering.crawl.ppomppu.processor.dto.FeedParsedResult;
import com.myoa.engineering.crawl.ppomppu.processor.service.MessageSenderService; import com.myoa.engineering.crawl.ppomppu.processor.service.MessageSenderService;
import com.myoa.engineering.crawl.ppomppu.processor.service.PpomppuArticleService; import com.myoa.engineering.crawl.ppomppu.processor.service.PpomppuArticleService;
@ -45,13 +43,14 @@ public class CrawlAPIController {
public Mono<APIResponse<FeedParsedResult>> crawlBoard(@PathVariable("boardName") PpomppuBoardName boardName) { public Mono<APIResponse<FeedParsedResult>> crawlBoard(@PathVariable("boardName") PpomppuBoardName boardName) {
log.info("got request... {}", boardName); log.info("got request... {}", boardName);
FeedParsedResult result = FeedParsedResult.of(boardName); FeedParsedResult result = FeedParsedResult.of(boardName);
Mono<List<PpomppuArticle>> articles =
ppomppuRSSFeedService.getArticles(boardName)
.doOnNext(e -> ppomppuArticleService.filterOnlyNewArticles(boardName, e))
.doOnNext(e -> messageSenderService.sendMessageToSlack(e))
.doOnNext(e -> ppomppuArticleService.save(boardName, e));
return articles.then(Mono.just(APIResponse.success(result.done()))); Mono<String> publishedMessages =
ppomppuRSSFeedService.getArticles(boardName)
.map(e -> ppomppuArticleService.filterOnlyNewArticles(boardName, e))
.map(e -> ppomppuArticleService.save(boardName, e))
.flatMap(messageSenderService::sendMessageToSlack);
return publishedMessages.then(Mono.just(APIResponse.success(result.done())));
} }
} }

View File

@ -46,7 +46,7 @@ public final class PpomppuArticleParser {
} }
public static String parseTitle(Element td) { public static String parseTitle(Element td) {
return td.text(); return td.getElementsByTag("a").text(); // TODO cdn image extracting
} }
public static String parseArticleUrl(Element td) { public static String parseArticleUrl(Element td) {

View File

@ -17,8 +17,11 @@ import com.myoa.engineering.crawl.ppomppu.support.dto.SimpleMessageDTO;
* *
*/ */
public final class PpomppuArticleTransformer { public final class PpomppuArticleTransformer {
private PpomppuArticleTransformer() {} private PpomppuArticleTransformer() {}
private static final String MESSAGE_FORMAT_V1 = "%s)) `%s` <%s:LINK>";
private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss") private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")
.withZone(ZoneId.of("Asia/Seoul")); .withZone(ZoneId.of("Asia/Seoul"));
@ -26,7 +29,7 @@ public final class PpomppuArticleTransformer {
SimpleMessageDTO.builder() SimpleMessageDTO.builder()
.requestedAt(Instant.now()) .requestedAt(Instant.now())
.publishedAt(entity.getRegisteredAt()) .publishedAt(entity.getRegisteredAt())
.title(String.format("[%s] %s", entity.getBoardName().getMenuName(), entity.getTitle())) .title(String.format(MESSAGE_FORMAT_V1, entity.getBoardName().getMenuName(), entity.getTitle()))
.body(entity.getArticleUrl()) .body(entity.getArticleUrl())
.build(); .build();
@ -44,7 +47,7 @@ public final class PpomppuArticleTransformer {
} }
public static String convertToInlineMessage(PpomppuArticle article) { public static String convertToInlineMessage(PpomppuArticle article) {
return String.format("[%s] %s\n%s", return String.format(MESSAGE_FORMAT_V1,
article.getBoardName().getMenuName(), article.getTitle(), article.getArticleUrl()); article.getBoardName().getMenuName(), article.getTitle(), article.getArticleUrl());
} }
} }

View File

@ -1,18 +1,20 @@
package com.myoa.engineering.crawl.ppomppu.processor.infrastructure.client; package com.myoa.engineering.crawl.ppomppu.processor.infrastructure.client;
import org.springframework.core.ParameterizedTypeReference;
import org.springframework.http.HttpHeaders;
import org.springframework.http.MediaType;
import org.springframework.stereotype.Component;
import org.springframework.web.reactive.function.client.WebClient;
import org.springframework.web.reactive.function.client.WebClientRequestException;
import com.myoa.engineering.crawl.ppomppu.processor.dto.constant.WebClientPropertiesUnitName; import com.myoa.engineering.crawl.ppomppu.processor.dto.constant.WebClientPropertiesUnitName;
import com.myoa.engineering.crawl.ppomppu.support.dto.SimpleMessageDTO; import com.myoa.engineering.crawl.ppomppu.support.dto.SimpleMessageDTO;
import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebClientFilterFactory; import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebClientFilterFactory;
import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebFluxExchangeStragiesFactory; import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebFluxExchangeStragiesFactory;
import com.myoa.engineering.crawl.ppomppu.support.webclient.properties.WebClientProperties; import com.myoa.engineering.crawl.ppomppu.support.webclient.properties.WebClientProperties;
import com.myoa.engineering.crawl.ppomppu.support.webclient.properties.WebClientProperties.WebClientPropertiesUnit; import com.myoa.engineering.crawl.ppomppu.support.webclient.properties.WebClientProperties.WebClientPropertiesUnit;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.core.ParameterizedTypeReference;
import org.springframework.http.HttpHeaders;
import org.springframework.http.MediaType;
import org.springframework.stereotype.Component;
import org.springframework.web.reactive.function.client.WebClient;
import org.springframework.web.reactive.function.client.WebClientRequestException;
import reactor.core.publisher.Mono; import reactor.core.publisher.Mono;
import reactor.core.scheduler.Schedulers; import reactor.core.scheduler.Schedulers;
@ -35,8 +37,8 @@ public class MessageSenderAPIClient {
.baseUrl(webClientPropertiesUnit.getBaseUrl()) .baseUrl(webClientPropertiesUnit.getBaseUrl())
.exchangeStrategies(WebFluxExchangeStragiesFactory.ofDefault()) .exchangeStrategies(WebFluxExchangeStragiesFactory.ofDefault())
.defaultHeader(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_VALUE) .defaultHeader(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_VALUE)
.filter(WebClientFilterFactory.logRequest()) // .filter(WebClientFilterFactory.logRequest())
.filter(WebClientFilterFactory.logResponse()) // .filter(WebClientFilterFactory.logResponse())
.build(); .build();
} }
@ -49,7 +51,6 @@ public class MessageSenderAPIClient {
.onErrorResume(WebClientRequestException.class, t -> { .onErrorResume(WebClientRequestException.class, t -> {
log.info("Exception occured, ignoring. : {}", t.getClass().getSimpleName()); log.info("Exception occured, ignoring. : {}", t.getClass().getSimpleName());
return Mono.empty(); return Mono.empty();
}) });
.doOnNext(e -> log.info("response: {} ", e));
} }
} }

View File

@ -1,16 +1,19 @@
package com.myoa.engineering.crawl.ppomppu.processor.service; package com.myoa.engineering.crawl.ppomppu.processor.service;
import java.util.List;
import java.util.Optional;
import java.util.stream.Collectors;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle; import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuBoardFeedStatus; import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuBoardFeedStatus;
import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository.PpomppuArticleRepository; import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository.PpomppuArticleRepository;
import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository.PpomppuBoardFeedStatusRepository; import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository.PpomppuBoardFeedStatusRepository;
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName; import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
import java.util.List;
import java.util.Optional;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
@Slf4j @Slf4j
@Service @Service
@ -32,6 +35,7 @@ public class PpomppuArticleService {
Long latestArticleId = boardFeedStatus.map(PpomppuBoardFeedStatus::getLatestParsedArticleId) Long latestArticleId = boardFeedStatus.map(PpomppuBoardFeedStatus::getLatestParsedArticleId)
.orElse(0L); .orElse(0L);
log.info("latestArticleId : {}", latestArticleId);
return articles.stream() return articles.stream()
.filter(e -> e.getArticleId().compareTo(latestArticleId) > 0) .filter(e -> e.getArticleId().compareTo(latestArticleId) > 0)
.collect(Collectors.toList()); .collect(Collectors.toList());
@ -46,9 +50,12 @@ public class PpomppuArticleService {
// save PpomppuBoardFeedStatus // save PpomppuBoardFeedStatus
Optional<PpomppuBoardFeedStatus> boardFeedStatus = ppomppuBoardFeedStatusRepository.findByBoardName(boardName); Optional<PpomppuBoardFeedStatus> boardFeedStatus = ppomppuBoardFeedStatusRepository.findByBoardName(boardName);
log.info("boardName: {}, isPresent?: {}", boardName, boardFeedStatus.isPresent());
boardFeedStatus.ifPresentOrElse(e -> { boardFeedStatus.ifPresentOrElse(e -> {
e.updateArticleId(latestArticleId); if (latestArticleId.longValue() > 0L) {
ppomppuBoardFeedStatusRepository.save(e); e.updateArticleId(latestArticleId);
ppomppuBoardFeedStatusRepository.save(e);
}
}, },
() -> ppomppuBoardFeedStatusRepository.save(PpomppuBoardFeedStatus.of(boardName, () -> ppomppuBoardFeedStatusRepository.save(PpomppuBoardFeedStatus.of(boardName,
latestArticleId))); latestArticleId)));

View File

@ -26,15 +26,11 @@ public class ParseEventEmitter {
this.processorAPIService = processorAPIService; this.processorAPIService = processorAPIService;
} }
@Scheduled(fixedRate = 60 * 1000L)
public void emitDomesticBoard() {
log.info("[emitDomesticBoard] trigger fired!");
processorAPIService.emitParseEvent(PpomppuBoardName.PPOMPPU_DOMESTIC_ETC).block();
}
@Scheduled(fixedRate = 300 * 1000L) @Scheduled(fixedRate = 300 * 1000L)
public void emitOverseaBoard() { public void emitBoards() {
log.info("[emitOverseaBoard] trigger fired!"); log.info("[emitDomesticBoard] trigger fired!");
processorAPIService.emitParseEvent(PpomppuBoardName.PPOMPPU_OVERSEA_ETC).block(); for (PpomppuBoardName boardName : PpomppuBoardName.values()) {
processorAPIService.emitParseEvent(boardName).block();
}
} }
} }

View File

@ -5,6 +5,7 @@ import com.myoa.engineering.crawl.ppomppu.support.dto.APIResponse;
import com.myoa.engineering.crawl.ppomppu.support.dto.SimpleMessageDTO; import com.myoa.engineering.crawl.ppomppu.support.dto.SimpleMessageDTO;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController; import org.springframework.web.bind.annotation.RestController;
import reactor.core.publisher.Mono; import reactor.core.publisher.Mono;
@ -27,9 +28,7 @@ public class MessageSenderAPIController {
} }
@PostMapping("/messages/sendMessage/messengers/slack") @PostMapping("/messages/sendMessage/messengers/slack")
public Mono<APIResponse<SimpleMessageDTO>> sendMessageToSlack(SimpleMessageDTO dto) { public Mono<APIResponse<SimpleMessageDTO>> sendMessageToSlack(@RequestBody SimpleMessageDTO dto) {
log.info("received : {}, \nbody: {}", dto.getTitle(), dto.getBody());
return sender.sendMessage(sender.ofMessage(dto.getBody())) return sender.sendMessage(sender.ofMessage(dto.getBody()))
.then(Mono.just(APIResponse.success(dto))); .then(Mono.just(APIResponse.success(dto)));
} }

View File

@ -4,6 +4,8 @@ import com.myoa.engineering.crawl.ppomppu.sender.infrastructure.client.MongeShop
import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController; import org.springframework.web.bind.annotation.RestController;
import lombok.extern.slf4j.Slf4j;
import reactor.core.publisher.Mono; import reactor.core.publisher.Mono;
/** /**
@ -12,6 +14,7 @@ import reactor.core.publisher.Mono;
* @author Shin Woo-jin (woo-jin.shin@linecorp.com) * @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-11-15 * @since 2021-11-15
*/ */
@Slf4j
@RestController @RestController
@RequestMapping("/api/v1") @RequestMapping("/api/v1")
public class TestAPIController { public class TestAPIController {
@ -24,6 +27,7 @@ public class TestAPIController {
@GetMapping("/test") @GetMapping("/test")
public Mono<String> test() { public Mono<String> test() {
log.info("received!!!");
return sender.sendMessage(sender.ofMessage("testtesttest!!!")); return sender.sendMessage(sender.ofMessage("testtesttest!!!"));
} }
} }

View File

@ -13,7 +13,6 @@ public class MongeShoppingBotSlackMessageSender extends SlackMessageSender {
private static final String SLACK_SECRET_UNIT_NAME = "monge_shopping_bot"; private static final String SLACK_SECRET_UNIT_NAME = "monge_shopping_bot";
private final SlackSecretPropertiesUnit slackProperties; private final SlackSecretPropertiesUnit slackProperties;
// private final SlackMessageSender slackMessageSender;
public MongeShoppingBotSlackMessageSender(SlackSecretProperties slackSecretProperties) { public MongeShoppingBotSlackMessageSender(SlackSecretProperties slackSecretProperties) {
super(slackSecretProperties.find(SLACK_SECRET_UNIT_NAME).getToken()); super(slackSecretProperties.find(SLACK_SECRET_UNIT_NAME).getToken());

View File

@ -33,7 +33,6 @@ public class SlackMessageSender implements MessageSender<SlackMessageDTO> {
.exchangeStrategies(WebFluxExchangeStragiesFactory.ofDefault()) .exchangeStrategies(WebFluxExchangeStragiesFactory.ofDefault())
.defaultHeader(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_UTF8_VALUE) .defaultHeader(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_UTF8_VALUE)
.defaultHeader(HttpHeaders.ACCEPT_CHARSET, "UTF-8") .defaultHeader(HttpHeaders.ACCEPT_CHARSET, "UTF-8")
.filter(WebClientFilterFactory.logRequest())
.filter(WebClientFilterFactory.logResponse()) .filter(WebClientFilterFactory.logResponse())
.build(); .build();
} }
@ -48,8 +47,7 @@ public class SlackMessageSender implements MessageSender<SlackMessageDTO> {
.onErrorResume(WebClientRequestException.class, t -> { .onErrorResume(WebClientRequestException.class, t -> {
log.info("Exception occured, ignoring. : {}", t.getClass().getSimpleName()); log.info("Exception occured, ignoring. : {}", t.getClass().getSimpleName());
return Mono.empty(); return Mono.empty();
}) });
.doOnNext(e -> log.info("response: {}", e));
} }
} }

View File

@ -12,11 +12,6 @@ spring:
freemarker: freemarker:
enabled: false enabled: false
jpa:
hibernate:
ddl-auto: create-drop
generate-ddl: true
server: server:
port: 20080 port: 20080
error: error:

View File

@ -0,0 +1,31 @@
{
"blocks": [
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": "기타)) `[쿠팡] 타푸코 이중 진공 스텐 세라믹 코팅 텀블러 (화이트만 11,130원/로켓와우무료) 4 [기타]`<https://www.ppomppu.co.kr/zboardview.php?id=ppomppu&page=1&divpage=69&category=1&&no=404126|LINK>"
},
"accessory": {
"type": "image",
"image_url": "cdn.ppomppu.co.kr/zboard/data3/2021/1121/m_20211121184835_zfyahnow.png",
"alt_text": "alt text for image"
}
},
{
"type": "divider"
},
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": "기타)) `[롯데온] 1매 149원 올국산 KF94새부리마스크 대형 200매 (29,920원/무료배송) 58 [기타]`<https://www.ppomppu.co.kr/zboardview.php?id=ppomppu&page=1&divpage=69&category=1&&no=404113|LINK>"
},
"accessory": {
"type": "image",
"image_url": "https://s3-media3.fl.yelpcdn.com/bphoto/c7ed05m9lC2EmA3Aruue7A/o.jpg",
"alt_text": "alt text for image"
}
}
]
}

View File

@ -12,7 +12,7 @@ import lombok.Getter;
@Getter @Getter
@AllArgsConstructor @AllArgsConstructor
public enum PpomppuBoardName { public enum PpomppuBoardName {
PPOMPPU_DOMESTIC_ALL("/zboard/zboard.php?id=ppomppu", "전체", true), PPOMPPU_DOMESTIC_ALL("/zboard/zboard.php?id=ppomppu", "전체", false),
PPOMPPU_DOMESTIC_ETC("/zboard/zboard.php?id=ppomppu&category=1", "기타", true), PPOMPPU_DOMESTIC_ETC("/zboard/zboard.php?id=ppomppu&category=1", "기타", true),
PPOMPPU_DOMESTIC_COMPUTER("/zboard/zboard.php?id=ppomppu&category=4", "컴퓨터", true), PPOMPPU_DOMESTIC_COMPUTER("/zboard/zboard.php?id=ppomppu&category=4", "컴퓨터", true),
PPOMPPU_DOMESTIC_DIGITAL("/zboard/zboard.php?id=ppomppu&category=5", "디지털", true), PPOMPPU_DOMESTIC_DIGITAL("/zboard/zboard.php?id=ppomppu&category=5", "디지털", true),
@ -24,7 +24,7 @@ public enum PpomppuBoardName {
PPOMPPU_DOMESTIC_CLOTHES("/zboard/zboard.php?id=ppomppu&category=12", "의류/잡화", true), PPOMPPU_DOMESTIC_CLOTHES("/zboard/zboard.php?id=ppomppu&category=12", "의류/잡화", true),
PPOMPPU_DOMESTIC_COSMETIC("/zboard/zboard.php?id=ppomppu&category=13", "화장품", true), PPOMPPU_DOMESTIC_COSMETIC("/zboard/zboard.php?id=ppomppu&category=13", "화장품", true),
PPOMPPU_DOMESTIC_OUTDOOR("/zboard/zboard.php?id=ppomppu&category=15", "등산/캠핑", true), PPOMPPU_DOMESTIC_OUTDOOR("/zboard/zboard.php?id=ppomppu&category=15", "등산/캠핑", true),
PPOMPPU_OVERSEA_ALL("/zboard/zboard.php?id=ppomppu4", "전체", true), PPOMPPU_OVERSEA_ALL("/zboard/zboard.php?id=ppomppu4", "전체", false),
PPOMPPU_OVERSEA_ETC("/zboard/zboard.php?id=ppomppu4&category=1", "기타", true), PPOMPPU_OVERSEA_ETC("/zboard/zboard.php?id=ppomppu4&category=1", "기타", true),
PPOMPPU_OVERSEA_APPLIANCES("/zboard/zboard.php?id=ppomppu4&category=7", "가전", true), PPOMPPU_OVERSEA_APPLIANCES("/zboard/zboard.php?id=ppomppu4&category=7", "가전", true),
PPOMPPU_OVERSEA_TVAV("/zboard/zboard.php?id=ppomppu4&category=8", "TV/영상", true), PPOMPPU_OVERSEA_TVAV("/zboard/zboard.php?id=ppomppu4&category=8", "TV/영상", true),
@ -45,7 +45,7 @@ public enum PpomppuBoardName {
public static final String PPOMPPU_URL = "https://www.ppomppu.co.kr"; public static final String PPOMPPU_URL = "https://www.ppomppu.co.kr";
public static String ofViewPageUrl(String articleUrl) { public static String ofViewPageUrl(String articleUrl) {
return PPOMPPU_URL + "/zboard" + articleUrl; return PPOMPPU_URL + "/zboard/" + articleUrl;
} }
} }