[PPN-211113] Fix bug
This commit is contained in:
parent
c97c8dc01f
commit
a0c3962e0d
|
@ -1,13 +1,11 @@
|
|||
package com.myoa.engineering.crawl.ppomppu.processor.controller;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
|
||||
import com.myoa.engineering.crawl.ppomppu.processor.dto.FeedParsedResult;
|
||||
import com.myoa.engineering.crawl.ppomppu.processor.service.MessageSenderService;
|
||||
import com.myoa.engineering.crawl.ppomppu.processor.service.PpomppuArticleService;
|
||||
|
@ -45,13 +43,14 @@ public class CrawlAPIController {
|
|||
public Mono<APIResponse<FeedParsedResult>> crawlBoard(@PathVariable("boardName") PpomppuBoardName boardName) {
|
||||
log.info("got request... {}", boardName);
|
||||
FeedParsedResult result = FeedParsedResult.of(boardName);
|
||||
Mono<List<PpomppuArticle>> articles =
|
||||
ppomppuRSSFeedService.getArticles(boardName)
|
||||
.doOnNext(e -> ppomppuArticleService.filterOnlyNewArticles(boardName, e))
|
||||
.doOnNext(e -> messageSenderService.sendMessageToSlack(e))
|
||||
.doOnNext(e -> ppomppuArticleService.save(boardName, e));
|
||||
|
||||
return articles.then(Mono.just(APIResponse.success(result.done())));
|
||||
Mono<String> publishedMessages =
|
||||
ppomppuRSSFeedService.getArticles(boardName)
|
||||
.map(e -> ppomppuArticleService.filterOnlyNewArticles(boardName, e))
|
||||
.map(e -> ppomppuArticleService.save(boardName, e))
|
||||
.flatMap(messageSenderService::sendMessageToSlack);
|
||||
|
||||
return publishedMessages.then(Mono.just(APIResponse.success(result.done())));
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -46,7 +46,7 @@ public final class PpomppuArticleParser {
|
|||
}
|
||||
|
||||
public static String parseTitle(Element td) {
|
||||
return td.text();
|
||||
return td.getElementsByTag("a").text(); // TODO cdn image extracting
|
||||
}
|
||||
|
||||
public static String parseArticleUrl(Element td) {
|
||||
|
|
|
@ -17,8 +17,11 @@ import com.myoa.engineering.crawl.ppomppu.support.dto.SimpleMessageDTO;
|
|||
*
|
||||
*/
|
||||
public final class PpomppuArticleTransformer {
|
||||
|
||||
private PpomppuArticleTransformer() {}
|
||||
|
||||
private static final String MESSAGE_FORMAT_V1 = "%s)) `%s` <%s:LINK>";
|
||||
|
||||
private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")
|
||||
.withZone(ZoneId.of("Asia/Seoul"));
|
||||
|
||||
|
@ -26,7 +29,7 @@ public final class PpomppuArticleTransformer {
|
|||
SimpleMessageDTO.builder()
|
||||
.requestedAt(Instant.now())
|
||||
.publishedAt(entity.getRegisteredAt())
|
||||
.title(String.format("[%s] %s", entity.getBoardName().getMenuName(), entity.getTitle()))
|
||||
.title(String.format(MESSAGE_FORMAT_V1, entity.getBoardName().getMenuName(), entity.getTitle()))
|
||||
.body(entity.getArticleUrl())
|
||||
.build();
|
||||
|
||||
|
@ -44,7 +47,7 @@ public final class PpomppuArticleTransformer {
|
|||
}
|
||||
|
||||
public static String convertToInlineMessage(PpomppuArticle article) {
|
||||
return String.format("[%s] %s\n%s",
|
||||
return String.format(MESSAGE_FORMAT_V1,
|
||||
article.getBoardName().getMenuName(), article.getTitle(), article.getArticleUrl());
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1,18 +1,20 @@
|
|||
package com.myoa.engineering.crawl.ppomppu.processor.infrastructure.client;
|
||||
|
||||
import org.springframework.core.ParameterizedTypeReference;
|
||||
import org.springframework.http.HttpHeaders;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.web.reactive.function.client.WebClient;
|
||||
import org.springframework.web.reactive.function.client.WebClientRequestException;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.processor.dto.constant.WebClientPropertiesUnitName;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.dto.SimpleMessageDTO;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebClientFilterFactory;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebFluxExchangeStragiesFactory;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.webclient.properties.WebClientProperties;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.webclient.properties.WebClientProperties.WebClientPropertiesUnit;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.core.ParameterizedTypeReference;
|
||||
import org.springframework.http.HttpHeaders;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.web.reactive.function.client.WebClient;
|
||||
import org.springframework.web.reactive.function.client.WebClientRequestException;
|
||||
import reactor.core.publisher.Mono;
|
||||
import reactor.core.scheduler.Schedulers;
|
||||
|
||||
|
@ -35,8 +37,8 @@ public class MessageSenderAPIClient {
|
|||
.baseUrl(webClientPropertiesUnit.getBaseUrl())
|
||||
.exchangeStrategies(WebFluxExchangeStragiesFactory.ofDefault())
|
||||
.defaultHeader(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_VALUE)
|
||||
.filter(WebClientFilterFactory.logRequest())
|
||||
.filter(WebClientFilterFactory.logResponse())
|
||||
// .filter(WebClientFilterFactory.logRequest())
|
||||
// .filter(WebClientFilterFactory.logResponse())
|
||||
.build();
|
||||
}
|
||||
|
||||
|
@ -49,7 +51,6 @@ public class MessageSenderAPIClient {
|
|||
.onErrorResume(WebClientRequestException.class, t -> {
|
||||
log.info("Exception occured, ignoring. : {}", t.getClass().getSimpleName());
|
||||
return Mono.empty();
|
||||
})
|
||||
.doOnNext(e -> log.info("response: {} ", e));
|
||||
});
|
||||
}
|
||||
}
|
|
@ -1,16 +1,19 @@
|
|||
package com.myoa.engineering.crawl.ppomppu.processor.service;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
|
||||
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle;
|
||||
import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuBoardFeedStatus;
|
||||
import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository.PpomppuArticleRepository;
|
||||
import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository.PpomppuBoardFeedStatusRepository;
|
||||
import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
|
@ -32,6 +35,7 @@ public class PpomppuArticleService {
|
|||
Long latestArticleId = boardFeedStatus.map(PpomppuBoardFeedStatus::getLatestParsedArticleId)
|
||||
.orElse(0L);
|
||||
|
||||
log.info("latestArticleId : {}", latestArticleId);
|
||||
return articles.stream()
|
||||
.filter(e -> e.getArticleId().compareTo(latestArticleId) > 0)
|
||||
.collect(Collectors.toList());
|
||||
|
@ -46,9 +50,12 @@ public class PpomppuArticleService {
|
|||
|
||||
// save PpomppuBoardFeedStatus
|
||||
Optional<PpomppuBoardFeedStatus> boardFeedStatus = ppomppuBoardFeedStatusRepository.findByBoardName(boardName);
|
||||
log.info("boardName: {}, isPresent?: {}", boardName, boardFeedStatus.isPresent());
|
||||
boardFeedStatus.ifPresentOrElse(e -> {
|
||||
e.updateArticleId(latestArticleId);
|
||||
ppomppuBoardFeedStatusRepository.save(e);
|
||||
if (latestArticleId.longValue() > 0L) {
|
||||
e.updateArticleId(latestArticleId);
|
||||
ppomppuBoardFeedStatusRepository.save(e);
|
||||
}
|
||||
},
|
||||
() -> ppomppuBoardFeedStatusRepository.save(PpomppuBoardFeedStatus.of(boardName,
|
||||
latestArticleId)));
|
||||
|
|
|
@ -26,15 +26,11 @@ public class ParseEventEmitter {
|
|||
this.processorAPIService = processorAPIService;
|
||||
}
|
||||
|
||||
@Scheduled(fixedRate = 60 * 1000L)
|
||||
public void emitDomesticBoard() {
|
||||
log.info("[emitDomesticBoard] trigger fired!");
|
||||
processorAPIService.emitParseEvent(PpomppuBoardName.PPOMPPU_DOMESTIC_ETC).block();
|
||||
}
|
||||
|
||||
@Scheduled(fixedRate = 300 * 1000L)
|
||||
public void emitOverseaBoard() {
|
||||
log.info("[emitOverseaBoard] trigger fired!");
|
||||
processorAPIService.emitParseEvent(PpomppuBoardName.PPOMPPU_OVERSEA_ETC).block();
|
||||
public void emitBoards() {
|
||||
log.info("[emitDomesticBoard] trigger fired!");
|
||||
for (PpomppuBoardName boardName : PpomppuBoardName.values()) {
|
||||
processorAPIService.emitParseEvent(boardName).block();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -5,6 +5,7 @@ import com.myoa.engineering.crawl.ppomppu.support.dto.APIResponse;
|
|||
import com.myoa.engineering.crawl.ppomppu.support.dto.SimpleMessageDTO;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
import reactor.core.publisher.Mono;
|
||||
|
@ -27,9 +28,7 @@ public class MessageSenderAPIController {
|
|||
}
|
||||
|
||||
@PostMapping("/messages/sendMessage/messengers/slack")
|
||||
public Mono<APIResponse<SimpleMessageDTO>> sendMessageToSlack(SimpleMessageDTO dto) {
|
||||
|
||||
log.info("received : {}, \nbody: {}", dto.getTitle(), dto.getBody());
|
||||
public Mono<APIResponse<SimpleMessageDTO>> sendMessageToSlack(@RequestBody SimpleMessageDTO dto) {
|
||||
return sender.sendMessage(sender.ofMessage(dto.getBody()))
|
||||
.then(Mono.just(APIResponse.success(dto)));
|
||||
}
|
||||
|
|
|
@ -4,6 +4,8 @@ import com.myoa.engineering.crawl.ppomppu.sender.infrastructure.client.MongeShop
|
|||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import reactor.core.publisher.Mono;
|
||||
|
||||
/**
|
||||
|
@ -12,6 +14,7 @@ import reactor.core.publisher.Mono;
|
|||
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
|
||||
* @since 2021-11-15
|
||||
*/
|
||||
@Slf4j
|
||||
@RestController
|
||||
@RequestMapping("/api/v1")
|
||||
public class TestAPIController {
|
||||
|
@ -24,6 +27,7 @@ public class TestAPIController {
|
|||
|
||||
@GetMapping("/test")
|
||||
public Mono<String> test() {
|
||||
log.info("received!!!");
|
||||
return sender.sendMessage(sender.ofMessage("testtesttest!!!"));
|
||||
}
|
||||
}
|
||||
|
|
|
@ -13,7 +13,6 @@ public class MongeShoppingBotSlackMessageSender extends SlackMessageSender {
|
|||
private static final String SLACK_SECRET_UNIT_NAME = "monge_shopping_bot";
|
||||
|
||||
private final SlackSecretPropertiesUnit slackProperties;
|
||||
// private final SlackMessageSender slackMessageSender;
|
||||
|
||||
public MongeShoppingBotSlackMessageSender(SlackSecretProperties slackSecretProperties) {
|
||||
super(slackSecretProperties.find(SLACK_SECRET_UNIT_NAME).getToken());
|
||||
|
|
|
@ -33,7 +33,6 @@ public class SlackMessageSender implements MessageSender<SlackMessageDTO> {
|
|||
.exchangeStrategies(WebFluxExchangeStragiesFactory.ofDefault())
|
||||
.defaultHeader(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_UTF8_VALUE)
|
||||
.defaultHeader(HttpHeaders.ACCEPT_CHARSET, "UTF-8")
|
||||
.filter(WebClientFilterFactory.logRequest())
|
||||
.filter(WebClientFilterFactory.logResponse())
|
||||
.build();
|
||||
}
|
||||
|
@ -48,8 +47,7 @@ public class SlackMessageSender implements MessageSender<SlackMessageDTO> {
|
|||
.onErrorResume(WebClientRequestException.class, t -> {
|
||||
log.info("Exception occured, ignoring. : {}", t.getClass().getSimpleName());
|
||||
return Mono.empty();
|
||||
})
|
||||
.doOnNext(e -> log.info("response: {}", e));
|
||||
});
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -12,11 +12,6 @@ spring:
|
|||
freemarker:
|
||||
enabled: false
|
||||
|
||||
jpa:
|
||||
hibernate:
|
||||
ddl-auto: create-drop
|
||||
generate-ddl: true
|
||||
|
||||
server:
|
||||
port: 20080
|
||||
error:
|
||||
|
|
|
@ -0,0 +1,31 @@
|
|||
{
|
||||
"blocks": [
|
||||
{
|
||||
"type": "section",
|
||||
"text": {
|
||||
"type": "mrkdwn",
|
||||
"text": "기타)) `[쿠팡] 타푸코 이중 진공 스텐 세라믹 코팅 텀블러 (화이트만 11,130원/로켓와우무료) 4 [기타]`<https://www.ppomppu.co.kr/zboardview.php?id=ppomppu&page=1&divpage=69&category=1&&no=404126|LINK>"
|
||||
},
|
||||
"accessory": {
|
||||
"type": "image",
|
||||
"image_url": "cdn.ppomppu.co.kr/zboard/data3/2021/1121/m_20211121184835_zfyahnow.png",
|
||||
"alt_text": "alt text for image"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "divider"
|
||||
},
|
||||
{
|
||||
"type": "section",
|
||||
"text": {
|
||||
"type": "mrkdwn",
|
||||
"text": "기타)) `[롯데온] 1매 149원 올국산 KF94새부리마스크 대형 200매 (29,920원/무료배송) 58 [기타]`<https://www.ppomppu.co.kr/zboardview.php?id=ppomppu&page=1&divpage=69&category=1&&no=404113|LINK>"
|
||||
},
|
||||
"accessory": {
|
||||
"type": "image",
|
||||
"image_url": "https://s3-media3.fl.yelpcdn.com/bphoto/c7ed05m9lC2EmA3Aruue7A/o.jpg",
|
||||
"alt_text": "alt text for image"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
|
@ -12,7 +12,7 @@ import lombok.Getter;
|
|||
@Getter
|
||||
@AllArgsConstructor
|
||||
public enum PpomppuBoardName {
|
||||
PPOMPPU_DOMESTIC_ALL("/zboard/zboard.php?id=ppomppu", "전체", true),
|
||||
PPOMPPU_DOMESTIC_ALL("/zboard/zboard.php?id=ppomppu", "전체", false),
|
||||
PPOMPPU_DOMESTIC_ETC("/zboard/zboard.php?id=ppomppu&category=1", "기타", true),
|
||||
PPOMPPU_DOMESTIC_COMPUTER("/zboard/zboard.php?id=ppomppu&category=4", "컴퓨터", true),
|
||||
PPOMPPU_DOMESTIC_DIGITAL("/zboard/zboard.php?id=ppomppu&category=5", "디지털", true),
|
||||
|
@ -24,7 +24,7 @@ public enum PpomppuBoardName {
|
|||
PPOMPPU_DOMESTIC_CLOTHES("/zboard/zboard.php?id=ppomppu&category=12", "의류/잡화", true),
|
||||
PPOMPPU_DOMESTIC_COSMETIC("/zboard/zboard.php?id=ppomppu&category=13", "화장품", true),
|
||||
PPOMPPU_DOMESTIC_OUTDOOR("/zboard/zboard.php?id=ppomppu&category=15", "등산/캠핑", true),
|
||||
PPOMPPU_OVERSEA_ALL("/zboard/zboard.php?id=ppomppu4", "전체", true),
|
||||
PPOMPPU_OVERSEA_ALL("/zboard/zboard.php?id=ppomppu4", "전체", false),
|
||||
PPOMPPU_OVERSEA_ETC("/zboard/zboard.php?id=ppomppu4&category=1", "기타", true),
|
||||
PPOMPPU_OVERSEA_APPLIANCES("/zboard/zboard.php?id=ppomppu4&category=7", "가전", true),
|
||||
PPOMPPU_OVERSEA_TVAV("/zboard/zboard.php?id=ppomppu4&category=8", "TV/영상", true),
|
||||
|
@ -45,7 +45,7 @@ public enum PpomppuBoardName {
|
|||
public static final String PPOMPPU_URL = "https://www.ppomppu.co.kr";
|
||||
|
||||
public static String ofViewPageUrl(String articleUrl) {
|
||||
return PPOMPPU_URL + "/zboard" + articleUrl;
|
||||
return PPOMPPU_URL + "/zboard/" + articleUrl;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue