[NO-ISSUE] Initialize v2

This commit is contained in:
woozu-shin 2024-05-09 08:48:36 +09:00
parent 0c4be3cc05
commit b4accbc2c0
26 changed files with 348 additions and 125 deletions

5
.gitignore vendored
View File

@ -36,4 +36,7 @@ out/
### VS Code ###
.vscode/
temppassword.yml
temppassword.yml
data.sql
**/src/main/resources/slack
**/src/main/resources/datasource

View File

@ -0,0 +1,30 @@
package com.myoa.engineering.crawl.shopping.configuration;
import feign.Logger;
import feign.RequestInterceptor;
import feign.codec.ErrorDecoder;
import lombok.extern.slf4j.Slf4j;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.http.HttpHeaders;
import org.springframework.http.MediaType;
@Slf4j
@Configuration
public class FeignDefaultConfig {
@Bean
Logger.Level feignLoggerLevel() {
return Logger.Level.FULL;
}
public static final String MIME_TYPE =
MediaType.APPLICATION_JSON_VALUE + ";charset=utf-8";
@Bean
public RequestInterceptor requestInterceptor() {
return requestTemplate -> requestTemplate.header(HttpHeaders.CONTENT_TYPE, MIME_TYPE);
}
}

View File

@ -99,8 +99,8 @@ public class ShoppingCrawlerDatasourceConfiguration {
properties.put(AvailableSettings.IMPLICIT_NAMING_STRATEGY, ImplicitNamingStrategyJpaCompliantImpl.class.getName());
properties.put(AvailableSettings.PHYSICAL_NAMING_STRATEGY, CamelCaseToUnderscoresNamingStrategy.class.getName());
properties.put(AvailableSettings.GENERATE_STATISTICS, "false");
properties.put(AvailableSettings.GLOBALLY_QUOTED_IDENTIFIERS, "true");
properties.put(AvailableSettings.GLOBALLY_QUOTED_IDENTIFIERS_SKIP_COLUMN_DEFINITIONS, "true");
// properties.put(AvailableSettings.GLOBALLY_QUOTED_IDENTIFIERS, "true");
// properties.put(AvailableSettings.GLOBALLY_QUOTED_IDENTIFIERS_SKIP_COLUMN_DEFINITIONS, "true");
properties.put(AvailableSettings.STATEMENT_BATCH_SIZE, "20");
properties.put(AvailableSettings.ORDER_INSERTS, "true");
properties.put(AvailableSettings.ORDER_UPDATES, "true");

View File

@ -21,6 +21,9 @@ public class AppUser extends Auditable {
@Column
private String name;
@Column
private String slackId;
@Column
private Boolean enabled;
}

View File

@ -27,4 +27,7 @@ public class SubscribedKeyword extends Auditable {
@Enumerated(EnumType.STRING)
private CrawlTarget crawlTarget;
@Column
private String userId;
}

View File

@ -0,0 +1,24 @@
package com.myoa.engineering.crawl.shopping.domain.model;
import com.myoa.engineering.crawl.shopping.domain.model.v2.ArticleModel;
import lombok.*;
import java.util.List;
@ToString
@Getter
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class UserNotifyModel {
private String userId;
private List<ArticleModel> articles;
public static UserNotifyModel of(String userId, List<ArticleModel> articles) {
return UserNotifyModel.builder()
.userId(userId)
.articles(articles)
.build();
}
}

View File

@ -0,0 +1,26 @@
package com.myoa.engineering.crawl.shopping.domain.model.v2;
import com.myoa.engineering.crawl.shopping.domain.entity.v2.AppUser;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Getter;
@Getter
@Builder
@AllArgsConstructor
public class AppUserModel {
private Long id;
private String name;
private String slackId;
private Boolean enabled;
public static AppUserModel from(AppUser entity) {
return AppUserModel.builder()
.id(entity.getId())
.name(entity.getName())
.slackId(entity.getSlackId())
.enabled(entity.getEnabled())
.build();
}
}

View File

@ -1,13 +1,11 @@
package com.myoa.engineering.crawl.shopping.domain.model.v2;
import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.*;
import java.time.ZonedDateTime;
@ToString
@Getter
@Builder
@NoArgsConstructor

View File

@ -0,0 +1,27 @@
package com.myoa.engineering.crawl.shopping.domain.model.v2;
import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget;
import com.myoa.engineering.crawl.shopping.util.AhoCorasickUtils;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Getter;
import org.ahocorasick.trie.Trie;
import java.util.List;
@Getter
@Builder
@AllArgsConstructor
public class SubscribedKeywordAggregatedModel {
private final Trie ahoCorasickTrie;
private final String userId;
private final CrawlTarget crawlTarget;
public static SubscribedKeywordAggregatedModel of(String userId, CrawlTarget crawlTarget, List<String> keywords) {
return SubscribedKeywordAggregatedModel.builder()
.userId(userId)
.crawlTarget(crawlTarget)
.ahoCorasickTrie(AhoCorasickUtils.generateTrie(keywords))
.build();
}
}

View File

@ -1,12 +1,13 @@
package com.myoa.engineering.crawl.shopping.event;
import com.myoa.engineering.crawl.shopping.domain.model.v2.ArticleModel;
import org.springframework.context.ApplicationEvent;
import java.util.List;
public class ArticleUpsertEvent extends ApplicationEvent {
public ArticleUpsertEvent(Object source) {
public ArticleUpsertEvent(List<ArticleModel> source) {
super(source);
}
}

View File

@ -2,18 +2,14 @@ package com.myoa.engineering.crawl.shopping.event;
import com.myoa.engineering.crawl.shopping.domain.model.v2.ArticleModel;
import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget;
import lombok.AllArgsConstructor;
import lombok.Getter;
import java.util.List;
@Getter
@Deprecated
public class ArticleUpsertEventCommand {
@AllArgsConstructor
public class ArticleUpsertEventPayload {
private final List<ArticleModel> articles;
private final CrawlTarget crawlTarget;
public ArticleUpsertEventCommand(List<ArticleModel> articles, CrawlTarget crawlTarget) {
this.articles = articles;
this.crawlTarget = crawlTarget;
}
}

View File

@ -1,23 +1,86 @@
package com.myoa.engineering.crawl.shopping.event.handler;
import com.myoa.engineering.crawl.shopping.domain.model.UserNotifyModel;
import com.myoa.engineering.crawl.shopping.domain.model.v2.AppUserModel;
import com.myoa.engineering.crawl.shopping.domain.model.v2.ArticleModel;
import com.myoa.engineering.crawl.shopping.domain.model.v2.SubscribedKeywordAggregatedModel;
import com.myoa.engineering.crawl.shopping.event.ArticleUpsertEvent;
import com.myoa.engineering.crawl.shopping.service.SubscribedKeywordQueryService;
import org.ahocorasick.trie.Trie;
import com.myoa.engineering.crawl.shopping.service.AppUserQueryService;
import com.myoa.engineering.crawl.shopping.service.SubscribedKeywordCacheService;
import com.myoa.engineering.crawl.shopping.service.UserNotifyService;
import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget;
import org.springframework.context.event.EventListener;
import org.springframework.stereotype.Component;
import java.util.List;
import java.util.Map;
import java.util.function.Function;
import java.util.stream.Collectors;
@Component
public class ArticleUpsertEventListener {
private final SubscribedKeywordQueryService subscribedKeywordQueryService;
private final SubscribedKeywordCacheService subscribedKeywordCacheService;
private final AppUserQueryService appUserQueryService;
private final UserNotifyService userNotifyService;
public ArticleUpsertEventListener(SubscribedKeywordQueryService subscribedKeywordQueryService) {
this.subscribedKeywordQueryService = subscribedKeywordQueryService;
public ArticleUpsertEventListener(SubscribedKeywordCacheService subscribedKeywordCacheService,
AppUserQueryService appUserQueryService, UserNotifyService userNotifyService) {
this.subscribedKeywordCacheService = subscribedKeywordCacheService;
this.appUserQueryService = appUserQueryService;
this.userNotifyService = userNotifyService;
}
@EventListener
public void handleArticleUpsertEvent(ArticleUpsertEvent event) {
Map<CrawlTarget, List<ArticleModel>> articleMap =
((List<ArticleModel>) event.getSource()).stream()
.collect(Collectors.groupingBy(ArticleModel::getCrawlTarget));
List<AppUserModel> appUsers = appUserQueryService.findAll();
appUsers.stream()
.filter(AppUserModel::getEnabled)
.map(user -> {
List<ArticleModel> filteredArticles = handleAhoCorasick(articleMap)
.apply(subscribedKeywordCacheService.getSubscribedKeywordsCached(user.getName()));
return UserNotifyModel.of(user.getName(), filteredArticles);
})
.forEach(this::notifyMessage);
System.out.println("event = " + event);
}
private Function<Map<CrawlTarget, SubscribedKeywordAggregatedModel>, List<ArticleModel>> handleAhoCorasick(
Map<CrawlTarget, List<ArticleModel>> articleMap) {
return userTrieModel -> {
return userTrieModel
.entrySet()
.stream().filter(e -> articleMap.containsKey(e.getKey()))
.map((entry) -> filterAhocorasick(articleMap.get(entry.getKey()), entry.getValue()))
.flatMap(List::stream)
.toList();
// return UserNotifyModel.of(userTrieModel.values().stream().findFirst().get().getUserId(),
// filteredArticle);
};
}
private List<ArticleModel> filterAhocorasick(List<ArticleModel> articles,
SubscribedKeywordAggregatedModel trieModel) {
return articles.stream()
.filter(article -> !trieModel.getAhoCorasickTrie()
.parseText(article.getTitle())
.isEmpty())
.toList();
//ArticleUpsertEventListener::printArticle
}
private void notifyMessage(UserNotifyModel article) {
System.out.println("article = " + article);
if (article.getArticles().isEmpty()){
return;
}
userNotifyService.notify("안녕 " + article.getUserId() + "\n" + article.getArticles());
}
}

View File

@ -1,11 +1,17 @@
package com.myoa.engineering.crawl.shopping.infra.client.slack;
import com.myoa.engineering.crawl.shopping.configuration.FeignDefaultConfig;
import com.myoa.engineering.crawl.shopping.dto.slack.SlackMessageDTO;
import org.springframework.cloud.openfeign.FeignClient;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestHeader;
@FeignClient(value = "slack-api-client", url = "https://slack.com")
@FeignClient(value = "slack-api-client", url = "https://slack.com/api",
configuration = FeignDefaultConfig.class)
public interface SlackAPIClient {
@PostMapping("/api/v1/messages/sendMessage/messengers/slack")
String sendMessage();
@PostMapping("/chat.postMessage")
String sendMessage(@RequestBody SlackMessageDTO message,
@RequestHeader("Authorization") String token);
}

View File

@ -11,4 +11,13 @@ import java.util.List;
public interface SubscribedKeywordRepository extends JpaRepository<SubscribedKeyword, Long> {
List<SubscribedKeyword> findByCrawlTarget(CrawlTarget crawlTarget);
/* @Query("SELECT new com.myoa.engineering.crawl.shopping.domain.model.v2.SubscribedKeywordUserAggregatedModel(" +
" s.userId, s.keyword, s.crawlTarget) " +
" FROM SubscribedKeyword s GROUP BY s.userId ")
List<SubscribedKeywordUserAggregatedModel> findGroupByUserId(String userId);*/
List<SubscribedKeyword> findByUserIdAndCrawlTarget(String userId, CrawlTarget crawlTarget);
List<SubscribedKeyword> findByUserId(String userId);
}

View File

@ -0,0 +1,25 @@
package com.myoa.engineering.crawl.shopping.service;
import com.myoa.engineering.crawl.shopping.domain.model.v2.AppUserModel;
import com.myoa.engineering.crawl.shopping.infra.repository.v2.AppUserRepository;
import org.springframework.stereotype.Service;
import java.util.List;
@Service
public class AppUserQueryService {
private final AppUserRepository appUserRepository;
public AppUserQueryService(AppUserRepository appUserRepository) {
this.appUserRepository = appUserRepository;
}
public List<AppUserModel> findAll() {
return appUserRepository.findAll()
.stream()
.map(AppUserModel::from)
.toList();
}
}

View File

@ -48,10 +48,10 @@ public class ArticleCommandService {
articleRepository.saveAll(updated);
articleRepository.saveAll(newArticles);
publish(newArticles);
publishEvent(newArticles);
}
private void publish(List<Article> articles) {
private void publishEvent(List<Article> articles) {
List<ArticleModel> articleModels =
articles.stream()
.map(transformer)

View File

@ -0,0 +1,41 @@
package com.myoa.engineering.crawl.shopping.service;
import com.myoa.engineering.crawl.shopping.domain.entity.v2.SubscribedKeyword;
import com.myoa.engineering.crawl.shopping.domain.model.v2.SubscribedKeywordAggregatedModel;
import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget;
import org.springframework.cache.annotation.Cacheable;
import org.springframework.stereotype.Component;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
@Component
public class SubscribedKeywordCacheService {
private final SubscribedKeywordQueryService subscribedKeywordQueryService;
public SubscribedKeywordCacheService(SubscribedKeywordQueryService subscribedKeywordQueryService) {
this.subscribedKeywordQueryService = subscribedKeywordQueryService;
}
@Cacheable(cacheNames = "subscribe.keywords", key = "#userId + '_' + #crawlTarget.name()")
public SubscribedKeywordAggregatedModel getSubscribedKeywordsCached(String userId, CrawlTarget crawlTarget) {
System.out.println("getSubscribedKeywordsCached");
List<String> keywords = subscribedKeywordQueryService.findByUserWithTarget(userId, crawlTarget)
.stream().map(SubscribedKeyword::getKeyword).toList();
return SubscribedKeywordAggregatedModel.of(userId, crawlTarget, keywords);
}
@Cacheable(cacheNames = "subscribe.keywords", key = "#userId")
public Map<CrawlTarget, SubscribedKeywordAggregatedModel> getSubscribedKeywordsCached(String userId) {
System.out.println("getSubscribedKeywordsCached");
return subscribedKeywordQueryService.findByUser(userId)
.stream()
.collect(Collectors.groupingBy(SubscribedKeyword::getCrawlTarget,
Collectors.mapping(SubscribedKeyword::getKeyword, Collectors.toList())))
.entrySet().stream()
.collect(Collectors.toMap(Map.Entry::getKey, e -> SubscribedKeywordAggregatedModel.of(userId, e.getKey(), e.getValue())));
}
}

View File

@ -20,7 +20,11 @@ public class SubscribedKeywordQueryService {
return subscribedKeywordRepository.findAll();
}
public List<SubscribedKeyword> findByCrawlTarget(CrawlTarget crawlTarget) {
return subscribedKeywordRepository.findByCrawlTarget(crawlTarget);
public List<SubscribedKeyword> findByUserWithTarget(String userId, CrawlTarget crawlTarget) {
return subscribedKeywordRepository.findByUserIdAndCrawlTarget(userId, crawlTarget);
}
public List<SubscribedKeyword> findByUser(String userId) {
return subscribedKeywordRepository.findByUserId(userId);
}
}

View File

@ -0,0 +1,34 @@
package com.myoa.engineering.crawl.shopping.service;
import com.myoa.engineering.crawl.shopping.configuration.slack.properties.SlackSecretProperties;
import com.myoa.engineering.crawl.shopping.dto.slack.SlackMessageDTO;
import com.myoa.engineering.crawl.shopping.infra.client.slack.SlackAPIClient;
import org.springframework.stereotype.Service;
@Service
public class UserNotifyService {
private static final String SLACK_PROPERTIES_UNIT_NAME = "shopping-crawler";
// private static final String NOTIFY_CHANNEL_ID = "notify_shopping";
// private static final String NOTIFY_ICON_EMOJI = ":monge_big:";
// private static final String NOTIFY_BOT_NAME = "몽이 탈호구봇";
private final SlackAPIClient slackAPIClient;
private final SlackSecretProperties.SlackSecretPropertiesUnit slackSecretProperties;
public UserNotifyService(SlackAPIClient slackAPIClient,
SlackSecretProperties slackSecretProperties) {
this.slackAPIClient = slackAPIClient;
this.slackSecretProperties = slackSecretProperties.find(SLACK_PROPERTIES_UNIT_NAME);
}
public void notify(String message) {
SlackMessageDTO slackMessageDTO = SlackMessageDTO.builder()
.channel(slackSecretProperties.getChannel())
.text(message)
.iconEmoji(slackSecretProperties.getIconEmoji())
.username(slackSecretProperties.getUsername())
.build();
slackAPIClient.sendMessage(slackMessageDTO, slackSecretProperties.getToken());
}
}

View File

@ -0,0 +1,18 @@
package com.myoa.engineering.crawl.shopping.util;
import org.ahocorasick.trie.Trie;
import java.util.List;
public final class AhoCorasickUtils {
private AhoCorasickUtils() {
}
public static Trie generateTrie(List<String> keywords) {
return Trie.builder()
.addKeywords(keywords)
.ignoreCase()
.build();
}
}

View File

@ -9,4 +9,10 @@ spring:
server:
port: 20080
# import: optional:configserver:http://localhost:11080 # can be start up even config server was not found.
# import: optional:configserver:http://localhost:11080 # can be start up even config server was not found.
feign:
client:
config:
default:
loggerLevel: FULL

View File

@ -1,42 +0,0 @@
spring:
jpa:
open-in-view: false
hibernate:
ddl-auto: create
datasource:
# driver-class-name: com.mysql.cj.jdbc.Driver
driver-class-name: org.h2.Driver
url: jdbc:h2:mem:crawler-shopping;DB_CLOSE_DELAY=-1
hikari:
minimum:idle: 5
maximum-pool-size: 10
idle-timeout: 600000
validation-timeout: 5000
connection-timeout: 5000
max-lifetime: 1800000
auto-commit: false
h2:
console:
enabled: true
path: /h2
port: 20082
datasource:
init: true
units:
- unit-name: crawler-shopping
schema-name: crawler-shopping
db-connection-url: jdbc:h2:mem:crawler-shopping
is-simple-connection-url: true
driver-class-name: org.h2.Driver
username: sa
password: sa
hibernate:
units:
- unit-name: crawler-shopping
dialect: org.hibernate.dialect.H2Dialect
format-sql: true
show-sql: true
hbm2ddl-auto: create
disable-auto-commit: true

View File

@ -1,40 +0,0 @@
spring:
jpa:
open-in-view: false
hibernate:
ddl-auto: create
datasource:
# driver-class-name: com.mysql.cj.jdbc.Driver
driver-class-name: org.h2.Driver
url: jdbc:h2:mem:crawler-shopping;DB_CLOSE_DELAY=-1
hikari:
minimum:idle: 5
maximum-pool-size: 10
idle-timeout: 600000
validation-timeout: 5000
connection-timeout: 5000
max-lifetime: 1800000
auto-commit: false
h2:
console:
enabled: true
path: /h2
port: 20082
datasource:
init: true
units:
- unit-name: crawler-shopping
schema-name: crawler-shopping
db-connection-url: jdbc:h2:mem:crawler-shopping
simple-connection-url: true
hibernate:
units:
- unit-name: crawler-shopping
dialect: org.hibernate.dialect.H2Dialect
format-sql: true
show-sql: true
hbm2ddl-auto: create
disable-auto-commit: true

View File

@ -10,8 +10,6 @@
<!-- =========== include appender =========== -->
<include resource="org/springframework/boot/logging/logback/defaults.xml"/>
<include resource="org/springframework/boot/logging/logback/console-appender.xml"/>
<include resource="logback/component/logback-nelo2.xml"/>
<include resource="logback/component/logback-datachain.xml"/>
<!-- =========== root logger ============== -->
<root level="${DEFAULT_LEVEL}">
<appender-ref ref="CONSOLE"/>

View File

@ -10,8 +10,6 @@
<!-- =========== include appender =========== -->
<include resource="org/springframework/boot/logging/logback/defaults.xml"/>
<include resource="org/springframework/boot/logging/logback/console-appender.xml"/>
<include resource="logback/component/logback-nelo2.xml"/>
<include resource="logback/component/logback-datachain.xml"/>
<!-- =========== root logger ============== -->
<root level="${DEFAULT_LEVEL}">
<appender-ref ref="CONSOLE"/>

View File

@ -1,8 +0,0 @@
slack:
bot:
units:
- bot-unit-name: shopping-crawler
username: "몽이 탈호구봇"
icon-emoji: ":monge_big:"
channel: "notify_shopping"
token: "xoxb-2688454277126-2695026012277-YKlaeoSBY42NtF6Teh4z7dLK"