From b4accbc2c08b6d39c62a4f46cd04be0571afc321 Mon Sep 17 00:00:00 2001 From: woozu-shin Date: Thu, 9 May 2024 08:48:36 +0900 Subject: [PATCH] [NO-ISSUE] Initialize v2 --- .gitignore | 5 +- .../configuration/FeignDefaultConfig.java | 30 ++++++++ ...hoppingCrawlerDatasourceConfiguration.java | 4 +- .../shopping/domain/entity/v2/AppUser.java | 3 + .../domain/entity/v2/SubscribedKeyword.java | 3 + .../domain/model/UserNotifyModel.java | 24 ++++++ .../domain/model/v2/AppUserModel.java | 26 +++++++ .../domain/model/v2/ArticleModel.java | 6 +- .../v2/SubscribedKeywordAggregatedModel.java | 27 +++++++ .../shopping/event/ArticleUpsertEvent.java | 7 +- ...nd.java => ArticleUpsertEventPayload.java} | 10 +-- .../handler/ArticleUpsertEventListener.java | 75 +++++++++++++++++-- .../infra/client/slack/SlackAPIClient.java | 12 ++- .../v2/SubscribedKeywordRepository.java | 9 +++ .../shopping/service/AppUserQueryService.java | 25 +++++++ .../service/ArticleCommandService.java | 4 +- .../SubscribedKeywordCacheService.java | 41 ++++++++++ .../SubscribedKeywordQueryService.java | 8 +- .../shopping/service/UserNotifyService.java | 34 +++++++++ .../crawl/shopping/util/AhoCorasickUtils.java | 18 +++++ .../src/main/resources/application-local.yml | 8 +- .../src/main/resources/datasource/local.yml | 42 ----------- .../main/resources/development/database.yml | 40 ---------- .../resources/logback/logback-development.xml | 2 - .../resources/logback/logback-production.xml | 2 - support/src/main/resources/slack/local.yml | 8 -- 26 files changed, 348 insertions(+), 125 deletions(-) create mode 100644 shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/FeignDefaultConfig.java create mode 100644 shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/model/UserNotifyModel.java create mode 100644 shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/model/v2/AppUserModel.java create mode 100644 shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/model/v2/SubscribedKeywordAggregatedModel.java rename shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/event/{ArticleUpsertEventCommand.java => ArticleUpsertEventPayload.java} (60%) create mode 100644 shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/AppUserQueryService.java create mode 100644 shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/SubscribedKeywordCacheService.java create mode 100644 shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/UserNotifyService.java create mode 100644 shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/util/AhoCorasickUtils.java delete mode 100644 support/src/main/resources/datasource/local.yml delete mode 100644 support/src/main/resources/development/database.yml delete mode 100644 support/src/main/resources/slack/local.yml diff --git a/.gitignore b/.gitignore index 670f49a..3f1f94d 100644 --- a/.gitignore +++ b/.gitignore @@ -36,4 +36,7 @@ out/ ### VS Code ### .vscode/ -temppassword.yml \ No newline at end of file +temppassword.yml +data.sql +**/src/main/resources/slack +**/src/main/resources/datasource \ No newline at end of file diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/FeignDefaultConfig.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/FeignDefaultConfig.java new file mode 100644 index 0000000..f01a4d4 --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/FeignDefaultConfig.java @@ -0,0 +1,30 @@ +package com.myoa.engineering.crawl.shopping.configuration; + +import feign.Logger; +import feign.RequestInterceptor; +import feign.codec.ErrorDecoder; +import lombok.extern.slf4j.Slf4j; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; + +@Slf4j +@Configuration +public class FeignDefaultConfig { + + @Bean + Logger.Level feignLoggerLevel() { + return Logger.Level.FULL; + } + + + public static final String MIME_TYPE = + MediaType.APPLICATION_JSON_VALUE + ";charset=utf-8"; + + @Bean + public RequestInterceptor requestInterceptor() { + return requestTemplate -> requestTemplate.header(HttpHeaders.CONTENT_TYPE, MIME_TYPE); + } + +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/datasource/ShoppingCrawlerDatasourceConfiguration.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/datasource/ShoppingCrawlerDatasourceConfiguration.java index 14df143..bd83f05 100644 --- a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/datasource/ShoppingCrawlerDatasourceConfiguration.java +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/datasource/ShoppingCrawlerDatasourceConfiguration.java @@ -99,8 +99,8 @@ public class ShoppingCrawlerDatasourceConfiguration { properties.put(AvailableSettings.IMPLICIT_NAMING_STRATEGY, ImplicitNamingStrategyJpaCompliantImpl.class.getName()); properties.put(AvailableSettings.PHYSICAL_NAMING_STRATEGY, CamelCaseToUnderscoresNamingStrategy.class.getName()); properties.put(AvailableSettings.GENERATE_STATISTICS, "false"); - properties.put(AvailableSettings.GLOBALLY_QUOTED_IDENTIFIERS, "true"); - properties.put(AvailableSettings.GLOBALLY_QUOTED_IDENTIFIERS_SKIP_COLUMN_DEFINITIONS, "true"); +// properties.put(AvailableSettings.GLOBALLY_QUOTED_IDENTIFIERS, "true"); +// properties.put(AvailableSettings.GLOBALLY_QUOTED_IDENTIFIERS_SKIP_COLUMN_DEFINITIONS, "true"); properties.put(AvailableSettings.STATEMENT_BATCH_SIZE, "20"); properties.put(AvailableSettings.ORDER_INSERTS, "true"); properties.put(AvailableSettings.ORDER_UPDATES, "true"); diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v2/AppUser.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v2/AppUser.java index 47e31ca..fc31469 100644 --- a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v2/AppUser.java +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v2/AppUser.java @@ -21,6 +21,9 @@ public class AppUser extends Auditable { @Column private String name; + @Column + private String slackId; + @Column private Boolean enabled; } diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v2/SubscribedKeyword.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v2/SubscribedKeyword.java index e859e9f..1aa294c 100644 --- a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v2/SubscribedKeyword.java +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v2/SubscribedKeyword.java @@ -27,4 +27,7 @@ public class SubscribedKeyword extends Auditable { @Enumerated(EnumType.STRING) private CrawlTarget crawlTarget; + @Column + private String userId; + } diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/model/UserNotifyModel.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/model/UserNotifyModel.java new file mode 100644 index 0000000..36a261a --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/model/UserNotifyModel.java @@ -0,0 +1,24 @@ +package com.myoa.engineering.crawl.shopping.domain.model; + +import com.myoa.engineering.crawl.shopping.domain.model.v2.ArticleModel; +import lombok.*; + +import java.util.List; + + +@ToString +@Getter +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class UserNotifyModel { + private String userId; + private List articles; + + public static UserNotifyModel of(String userId, List articles) { + return UserNotifyModel.builder() + .userId(userId) + .articles(articles) + .build(); + } +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/model/v2/AppUserModel.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/model/v2/AppUserModel.java new file mode 100644 index 0000000..8c22c7d --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/model/v2/AppUserModel.java @@ -0,0 +1,26 @@ +package com.myoa.engineering.crawl.shopping.domain.model.v2; + +import com.myoa.engineering.crawl.shopping.domain.entity.v2.AppUser; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; + +@Getter +@Builder +@AllArgsConstructor +public class AppUserModel { + + private Long id; + private String name; + private String slackId; + private Boolean enabled; + + public static AppUserModel from(AppUser entity) { + return AppUserModel.builder() + .id(entity.getId()) + .name(entity.getName()) + .slackId(entity.getSlackId()) + .enabled(entity.getEnabled()) + .build(); + } +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/model/v2/ArticleModel.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/model/v2/ArticleModel.java index b1435ff..ec7626b 100644 --- a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/model/v2/ArticleModel.java +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/model/v2/ArticleModel.java @@ -1,13 +1,11 @@ package com.myoa.engineering.crawl.shopping.domain.model.v2; import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget; -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Getter; -import lombok.NoArgsConstructor; +import lombok.*; import java.time.ZonedDateTime; +@ToString @Getter @Builder @NoArgsConstructor diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/model/v2/SubscribedKeywordAggregatedModel.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/model/v2/SubscribedKeywordAggregatedModel.java new file mode 100644 index 0000000..a6fa5dd --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/model/v2/SubscribedKeywordAggregatedModel.java @@ -0,0 +1,27 @@ +package com.myoa.engineering.crawl.shopping.domain.model.v2; + +import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget; +import com.myoa.engineering.crawl.shopping.util.AhoCorasickUtils; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; +import org.ahocorasick.trie.Trie; + +import java.util.List; + +@Getter +@Builder +@AllArgsConstructor +public class SubscribedKeywordAggregatedModel { + private final Trie ahoCorasickTrie; + private final String userId; + private final CrawlTarget crawlTarget; + + public static SubscribedKeywordAggregatedModel of(String userId, CrawlTarget crawlTarget, List keywords) { + return SubscribedKeywordAggregatedModel.builder() + .userId(userId) + .crawlTarget(crawlTarget) + .ahoCorasickTrie(AhoCorasickUtils.generateTrie(keywords)) + .build(); + } +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/event/ArticleUpsertEvent.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/event/ArticleUpsertEvent.java index e58b59d..f7cbe79 100644 --- a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/event/ArticleUpsertEvent.java +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/event/ArticleUpsertEvent.java @@ -1,12 +1,13 @@ package com.myoa.engineering.crawl.shopping.event; +import com.myoa.engineering.crawl.shopping.domain.model.v2.ArticleModel; import org.springframework.context.ApplicationEvent; +import java.util.List; + public class ArticleUpsertEvent extends ApplicationEvent { - public ArticleUpsertEvent(Object source) { + public ArticleUpsertEvent(List source) { super(source); } } - - diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/event/ArticleUpsertEventCommand.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/event/ArticleUpsertEventPayload.java similarity index 60% rename from shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/event/ArticleUpsertEventCommand.java rename to shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/event/ArticleUpsertEventPayload.java index fb2d709..2fab09b 100644 --- a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/event/ArticleUpsertEventCommand.java +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/event/ArticleUpsertEventPayload.java @@ -2,18 +2,14 @@ package com.myoa.engineering.crawl.shopping.event; import com.myoa.engineering.crawl.shopping.domain.model.v2.ArticleModel; import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget; +import lombok.AllArgsConstructor; import lombok.Getter; import java.util.List; @Getter -@Deprecated -public class ArticleUpsertEventCommand { +@AllArgsConstructor +public class ArticleUpsertEventPayload { private final List articles; private final CrawlTarget crawlTarget; - - public ArticleUpsertEventCommand(List articles, CrawlTarget crawlTarget) { - this.articles = articles; - this.crawlTarget = crawlTarget; - } } diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/event/handler/ArticleUpsertEventListener.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/event/handler/ArticleUpsertEventListener.java index 846a641..f0ac31c 100644 --- a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/event/handler/ArticleUpsertEventListener.java +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/event/handler/ArticleUpsertEventListener.java @@ -1,23 +1,86 @@ package com.myoa.engineering.crawl.shopping.event.handler; +import com.myoa.engineering.crawl.shopping.domain.model.UserNotifyModel; +import com.myoa.engineering.crawl.shopping.domain.model.v2.AppUserModel; +import com.myoa.engineering.crawl.shopping.domain.model.v2.ArticleModel; +import com.myoa.engineering.crawl.shopping.domain.model.v2.SubscribedKeywordAggregatedModel; import com.myoa.engineering.crawl.shopping.event.ArticleUpsertEvent; -import com.myoa.engineering.crawl.shopping.service.SubscribedKeywordQueryService; -import org.ahocorasick.trie.Trie; +import com.myoa.engineering.crawl.shopping.service.AppUserQueryService; +import com.myoa.engineering.crawl.shopping.service.SubscribedKeywordCacheService; +import com.myoa.engineering.crawl.shopping.service.UserNotifyService; +import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget; import org.springframework.context.event.EventListener; import org.springframework.stereotype.Component; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; + @Component public class ArticleUpsertEventListener { - private final SubscribedKeywordQueryService subscribedKeywordQueryService; + private final SubscribedKeywordCacheService subscribedKeywordCacheService; + private final AppUserQueryService appUserQueryService; + private final UserNotifyService userNotifyService; - public ArticleUpsertEventListener(SubscribedKeywordQueryService subscribedKeywordQueryService) { - this.subscribedKeywordQueryService = subscribedKeywordQueryService; + + public ArticleUpsertEventListener(SubscribedKeywordCacheService subscribedKeywordCacheService, + AppUserQueryService appUserQueryService, UserNotifyService userNotifyService) { + this.subscribedKeywordCacheService = subscribedKeywordCacheService; + this.appUserQueryService = appUserQueryService; + this.userNotifyService = userNotifyService; } + @EventListener public void handleArticleUpsertEvent(ArticleUpsertEvent event) { + Map> articleMap = + ((List) event.getSource()).stream() + .collect(Collectors.groupingBy(ArticleModel::getCrawlTarget)); + List appUsers = appUserQueryService.findAll(); + + appUsers.stream() + .filter(AppUserModel::getEnabled) + .map(user -> { + List filteredArticles = handleAhoCorasick(articleMap) + .apply(subscribedKeywordCacheService.getSubscribedKeywordsCached(user.getName())); + return UserNotifyModel.of(user.getName(), filteredArticles); + }) + .forEach(this::notifyMessage); - System.out.println("event = " + event); } + + private Function, List> handleAhoCorasick( + Map> articleMap) { + return userTrieModel -> { + return userTrieModel + .entrySet() + .stream().filter(e -> articleMap.containsKey(e.getKey())) + .map((entry) -> filterAhocorasick(articleMap.get(entry.getKey()), entry.getValue())) + .flatMap(List::stream) + .toList(); +// return UserNotifyModel.of(userTrieModel.values().stream().findFirst().get().getUserId(), +// filteredArticle); + }; + } + + private List filterAhocorasick(List articles, + SubscribedKeywordAggregatedModel trieModel) { + return articles.stream() + .filter(article -> !trieModel.getAhoCorasickTrie() + .parseText(article.getTitle()) + .isEmpty()) + .toList(); + //ArticleUpsertEventListener::printArticle + } + + private void notifyMessage(UserNotifyModel article) { + System.out.println("article = " + article); + if (article.getArticles().isEmpty()){ + return; + } + userNotifyService.notify("안녕 " + article.getUserId() + "\n" + article.getArticles()); + } + } diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/slack/SlackAPIClient.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/slack/SlackAPIClient.java index 0144288..1d1ed89 100644 --- a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/slack/SlackAPIClient.java +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/slack/SlackAPIClient.java @@ -1,11 +1,17 @@ package com.myoa.engineering.crawl.shopping.infra.client.slack; +import com.myoa.engineering.crawl.shopping.configuration.FeignDefaultConfig; +import com.myoa.engineering.crawl.shopping.dto.slack.SlackMessageDTO; import org.springframework.cloud.openfeign.FeignClient; import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestHeader; -@FeignClient(value = "slack-api-client", url = "https://slack.com") +@FeignClient(value = "slack-api-client", url = "https://slack.com/api", + configuration = FeignDefaultConfig.class) public interface SlackAPIClient { - @PostMapping("/api/v1/messages/sendMessage/messengers/slack") - String sendMessage(); + @PostMapping("/chat.postMessage") + String sendMessage(@RequestBody SlackMessageDTO message, + @RequestHeader("Authorization") String token); } diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/repository/v2/SubscribedKeywordRepository.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/repository/v2/SubscribedKeywordRepository.java index 6870073..90028c1 100644 --- a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/repository/v2/SubscribedKeywordRepository.java +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/repository/v2/SubscribedKeywordRepository.java @@ -11,4 +11,13 @@ import java.util.List; public interface SubscribedKeywordRepository extends JpaRepository { List findByCrawlTarget(CrawlTarget crawlTarget); + + + /* @Query("SELECT new com.myoa.engineering.crawl.shopping.domain.model.v2.SubscribedKeywordUserAggregatedModel(" + + " s.userId, s.keyword, s.crawlTarget) " + + " FROM SubscribedKeyword s GROUP BY s.userId ") + List findGroupByUserId(String userId);*/ + List findByUserIdAndCrawlTarget(String userId, CrawlTarget crawlTarget); + + List findByUserId(String userId); } diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/AppUserQueryService.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/AppUserQueryService.java new file mode 100644 index 0000000..c2f1d01 --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/AppUserQueryService.java @@ -0,0 +1,25 @@ +package com.myoa.engineering.crawl.shopping.service; + +import com.myoa.engineering.crawl.shopping.domain.model.v2.AppUserModel; +import com.myoa.engineering.crawl.shopping.infra.repository.v2.AppUserRepository; +import org.springframework.stereotype.Service; + +import java.util.List; + +@Service +public class AppUserQueryService { + + private final AppUserRepository appUserRepository; + + public AppUserQueryService(AppUserRepository appUserRepository) { + this.appUserRepository = appUserRepository; + } + + public List findAll() { + return appUserRepository.findAll() + .stream() + .map(AppUserModel::from) + .toList(); + } + +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/ArticleCommandService.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/ArticleCommandService.java index fe8f22d..20aa894 100644 --- a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/ArticleCommandService.java +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/ArticleCommandService.java @@ -48,10 +48,10 @@ public class ArticleCommandService { articleRepository.saveAll(updated); articleRepository.saveAll(newArticles); - publish(newArticles); + publishEvent(newArticles); } - private void publish(List
articles) { + private void publishEvent(List
articles) { List articleModels = articles.stream() .map(transformer) diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/SubscribedKeywordCacheService.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/SubscribedKeywordCacheService.java new file mode 100644 index 0000000..a0fd644 --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/SubscribedKeywordCacheService.java @@ -0,0 +1,41 @@ +package com.myoa.engineering.crawl.shopping.service; + +import com.myoa.engineering.crawl.shopping.domain.entity.v2.SubscribedKeyword; +import com.myoa.engineering.crawl.shopping.domain.model.v2.SubscribedKeywordAggregatedModel; +import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget; +import org.springframework.cache.annotation.Cacheable; +import org.springframework.stereotype.Component; + +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +@Component +public class SubscribedKeywordCacheService { + + private final SubscribedKeywordQueryService subscribedKeywordQueryService; + + public SubscribedKeywordCacheService(SubscribedKeywordQueryService subscribedKeywordQueryService) { + this.subscribedKeywordQueryService = subscribedKeywordQueryService; + } + + @Cacheable(cacheNames = "subscribe.keywords", key = "#userId + '_' + #crawlTarget.name()") + public SubscribedKeywordAggregatedModel getSubscribedKeywordsCached(String userId, CrawlTarget crawlTarget) { + System.out.println("getSubscribedKeywordsCached"); + List keywords = subscribedKeywordQueryService.findByUserWithTarget(userId, crawlTarget) + .stream().map(SubscribedKeyword::getKeyword).toList(); + return SubscribedKeywordAggregatedModel.of(userId, crawlTarget, keywords); + } + + @Cacheable(cacheNames = "subscribe.keywords", key = "#userId") + public Map getSubscribedKeywordsCached(String userId) { + System.out.println("getSubscribedKeywordsCached"); + return subscribedKeywordQueryService.findByUser(userId) + .stream() + .collect(Collectors.groupingBy(SubscribedKeyword::getCrawlTarget, + Collectors.mapping(SubscribedKeyword::getKeyword, Collectors.toList()))) + .entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getKey, e -> SubscribedKeywordAggregatedModel.of(userId, e.getKey(), e.getValue()))); + } + +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/SubscribedKeywordQueryService.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/SubscribedKeywordQueryService.java index 9aa461e..dc06a58 100644 --- a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/SubscribedKeywordQueryService.java +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/SubscribedKeywordQueryService.java @@ -20,7 +20,11 @@ public class SubscribedKeywordQueryService { return subscribedKeywordRepository.findAll(); } - public List findByCrawlTarget(CrawlTarget crawlTarget) { - return subscribedKeywordRepository.findByCrawlTarget(crawlTarget); + public List findByUserWithTarget(String userId, CrawlTarget crawlTarget) { + return subscribedKeywordRepository.findByUserIdAndCrawlTarget(userId, crawlTarget); + } + + public List findByUser(String userId) { + return subscribedKeywordRepository.findByUserId(userId); } } diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/UserNotifyService.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/UserNotifyService.java new file mode 100644 index 0000000..13486ee --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/UserNotifyService.java @@ -0,0 +1,34 @@ +package com.myoa.engineering.crawl.shopping.service; + +import com.myoa.engineering.crawl.shopping.configuration.slack.properties.SlackSecretProperties; +import com.myoa.engineering.crawl.shopping.dto.slack.SlackMessageDTO; +import com.myoa.engineering.crawl.shopping.infra.client.slack.SlackAPIClient; +import org.springframework.stereotype.Service; + +@Service +public class UserNotifyService { + + private static final String SLACK_PROPERTIES_UNIT_NAME = "shopping-crawler"; +// private static final String NOTIFY_CHANNEL_ID = "notify_shopping"; +// private static final String NOTIFY_ICON_EMOJI = ":monge_big:"; +// private static final String NOTIFY_BOT_NAME = "몽이 탈호구봇"; + + private final SlackAPIClient slackAPIClient; + private final SlackSecretProperties.SlackSecretPropertiesUnit slackSecretProperties; + + public UserNotifyService(SlackAPIClient slackAPIClient, + SlackSecretProperties slackSecretProperties) { + this.slackAPIClient = slackAPIClient; + this.slackSecretProperties = slackSecretProperties.find(SLACK_PROPERTIES_UNIT_NAME); + } + + public void notify(String message) { + SlackMessageDTO slackMessageDTO = SlackMessageDTO.builder() + .channel(slackSecretProperties.getChannel()) + .text(message) + .iconEmoji(slackSecretProperties.getIconEmoji()) + .username(slackSecretProperties.getUsername()) + .build(); + slackAPIClient.sendMessage(slackMessageDTO, slackSecretProperties.getToken()); + } +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/util/AhoCorasickUtils.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/util/AhoCorasickUtils.java new file mode 100644 index 0000000..1066198 --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/util/AhoCorasickUtils.java @@ -0,0 +1,18 @@ +package com.myoa.engineering.crawl.shopping.util; + +import org.ahocorasick.trie.Trie; + +import java.util.List; + +public final class AhoCorasickUtils { + + private AhoCorasickUtils() { + } + + public static Trie generateTrie(List keywords) { + return Trie.builder() + .addKeywords(keywords) + .ignoreCase() + .build(); + } +} diff --git a/shopping-crawler/src/main/resources/application-local.yml b/shopping-crawler/src/main/resources/application-local.yml index 7737491..67afb03 100644 --- a/shopping-crawler/src/main/resources/application-local.yml +++ b/shopping-crawler/src/main/resources/application-local.yml @@ -9,4 +9,10 @@ spring: server: port: 20080 - # import: optional:configserver:http://localhost:11080 # can be start up even config server was not found. \ No newline at end of file + # import: optional:configserver:http://localhost:11080 # can be start up even config server was not found. + +feign: + client: + config: + default: + loggerLevel: FULL \ No newline at end of file diff --git a/support/src/main/resources/datasource/local.yml b/support/src/main/resources/datasource/local.yml deleted file mode 100644 index 2814984..0000000 --- a/support/src/main/resources/datasource/local.yml +++ /dev/null @@ -1,42 +0,0 @@ -spring: - jpa: - open-in-view: false - hibernate: - ddl-auto: create - datasource: - # driver-class-name: com.mysql.cj.jdbc.Driver - driver-class-name: org.h2.Driver - url: jdbc:h2:mem:crawler-shopping;DB_CLOSE_DELAY=-1 - hikari: - minimum:idle: 5 - maximum-pool-size: 10 - idle-timeout: 600000 - validation-timeout: 5000 - connection-timeout: 5000 - max-lifetime: 1800000 - auto-commit: false - h2: - console: - enabled: true - path: /h2 - port: 20082 - -datasource: - init: true - units: - - unit-name: crawler-shopping - schema-name: crawler-shopping - db-connection-url: jdbc:h2:mem:crawler-shopping - is-simple-connection-url: true - driver-class-name: org.h2.Driver - username: sa - password: sa - -hibernate: - units: - - unit-name: crawler-shopping - dialect: org.hibernate.dialect.H2Dialect - format-sql: true - show-sql: true - hbm2ddl-auto: create - disable-auto-commit: true diff --git a/support/src/main/resources/development/database.yml b/support/src/main/resources/development/database.yml deleted file mode 100644 index 3f64bae..0000000 --- a/support/src/main/resources/development/database.yml +++ /dev/null @@ -1,40 +0,0 @@ -spring: - jpa: - open-in-view: false - hibernate: - ddl-auto: create - datasource: - # driver-class-name: com.mysql.cj.jdbc.Driver - driver-class-name: org.h2.Driver - url: jdbc:h2:mem:crawler-shopping;DB_CLOSE_DELAY=-1 - hikari: - minimum:idle: 5 - maximum-pool-size: 10 - idle-timeout: 600000 - validation-timeout: 5000 - connection-timeout: 5000 - max-lifetime: 1800000 - auto-commit: false - h2: - console: - enabled: true - path: /h2 - port: 20082 - -datasource: - init: true - units: - - unit-name: crawler-shopping - schema-name: crawler-shopping - db-connection-url: jdbc:h2:mem:crawler-shopping - simple-connection-url: true - - -hibernate: - units: - - unit-name: crawler-shopping - dialect: org.hibernate.dialect.H2Dialect - format-sql: true - show-sql: true - hbm2ddl-auto: create - disable-auto-commit: true diff --git a/support/src/main/resources/logback/logback-development.xml b/support/src/main/resources/logback/logback-development.xml index 458e3d8..ed84966 100644 --- a/support/src/main/resources/logback/logback-development.xml +++ b/support/src/main/resources/logback/logback-development.xml @@ -10,8 +10,6 @@ - - diff --git a/support/src/main/resources/logback/logback-production.xml b/support/src/main/resources/logback/logback-production.xml index f824e41..df845f5 100644 --- a/support/src/main/resources/logback/logback-production.xml +++ b/support/src/main/resources/logback/logback-production.xml @@ -10,8 +10,6 @@ - - diff --git a/support/src/main/resources/slack/local.yml b/support/src/main/resources/slack/local.yml deleted file mode 100644 index 8bc78cd..0000000 --- a/support/src/main/resources/slack/local.yml +++ /dev/null @@ -1,8 +0,0 @@ -slack: - bot: - units: - - bot-unit-name: shopping-crawler - username: "몽이 탈호구봇" - icon-emoji: ":monge_big:" - channel: "notify_shopping" - token: "xoxb-2688454277126-2695026012277-YKlaeoSBY42NtF6Teh4z7dLK"