diff --git a/build.gradle b/build.gradle index c5a593a..7adae98 100644 --- a/build.gradle +++ b/build.gradle @@ -1,13 +1,14 @@ plugins { - id 'org.springframework.boot' version '2.5.4' - id 'io.spring.dependency-management' version '1.0.11.RELEASE' id 'java' id 'idea' + id 'org.springframework.boot' version '3.2.5' + id 'io.spring.dependency-management' version '1.1.4' } group = 'com.myoa.engineering.crawl.ppomppu' version = '1.0.1' -sourceCompatibility = '11' +sourceCompatibility = JavaVersion.VERSION_21 +targetCompatibility = JavaVersion.VERSION_21 configurations { compileOnly { @@ -20,8 +21,8 @@ repositories { } allprojects { - group = 'com.myoa.engineering.crawl.ppomppu' - version = '1.0.1' + group = 'com.myoa.engineering.crawl.shopping' + version = '2.0.0' apply plugin: 'java' apply plugin: 'idea' @@ -36,7 +37,7 @@ allprojects { } ext { - set('springCloudVersion', "2020.0.4") + set('springCloudVersion', "2023.0.1") } dependencyManagement { diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 05679dc..6677206 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,6 @@ +#Sun Apr 28 23:47:38 KST 2024 distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.1.1-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.5-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/gradlew b/gradlew old mode 100644 new mode 100755 diff --git a/processor-1.0.1.jar b/processor-1.0.1.jar deleted file mode 100644 index 448df79..0000000 Binary files a/processor-1.0.1.jar and /dev/null differ diff --git a/processor/build.gradle b/processor/build.gradle index eb68c4c..dee2708 100644 --- a/processor/build.gradle +++ b/processor/build.gradle @@ -12,14 +12,15 @@ dependencies { implementation 'org.springframework.boot:spring-boot-starter-jdbc' implementation 'org.springframework.cloud:spring-cloud-starter-config' implementation 'org.springframework.boot:spring-boot-starter-actuator' - implementation 'com.rometools:rome:1.16.0' - implementation 'org.jsoup:jsoup:1.14.2' - implementation 'com.h2database:h2:1.4.200' + implementation 'com.rometools:rome:2.1.0' + implementation 'org.jsoup:jsoup:1.15.3' + implementation 'com.h2database:h2:2.2.220' annotationProcessor 'org.springframework.boot:spring-boot-configuration-processor' annotationProcessor 'org.projectlombok:lombok' testImplementation 'org.springframework.boot:spring-boot-starter-test' + testImplementation("org.assertj:assertj-core:3.24.2") } test { diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/ProcessorApplication.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/ProcessorApplication.java index eb9f40c..c47a0d2 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/ProcessorApplication.java +++ b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/ProcessorApplication.java @@ -4,7 +4,7 @@ import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.context.annotation.Import; -import com.myoa.engineering.crawl.ppomppu.support.webclient.PpomppuNotifierWebClientConfiguration; +import com.myoa.engineering.crawl.shopping.support.webclient.PpomppuNotifierWebClientConfiguration; /** * ProcessorApplication diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/controller/CrawlAPIController.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/controller/CrawlAPIController.java index 2e92503..e7428a1 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/controller/CrawlAPIController.java +++ b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/controller/CrawlAPIController.java @@ -1,6 +1,5 @@ package com.myoa.engineering.crawl.ppomppu.processor.controller; -import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestMapping; @@ -10,8 +9,8 @@ import com.myoa.engineering.crawl.ppomppu.processor.dto.FeedParsedResult; import com.myoa.engineering.crawl.ppomppu.processor.service.MessageSenderService; import com.myoa.engineering.crawl.ppomppu.processor.service.PpomppuArticleService; import com.myoa.engineering.crawl.ppomppu.processor.service.PpomppuFeedService; -import com.myoa.engineering.crawl.ppomppu.support.dto.APIResponse; -import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName; +import com.myoa.engineering.crawl.shopping.support.dto.APIResponse; +import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName; import lombok.extern.slf4j.Slf4j; import reactor.core.publisher.Mono; diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/SubscribedBoard.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/SubscribedBoard.java deleted file mode 100644 index 2eb7565..0000000 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/SubscribedBoard.java +++ /dev/null @@ -1,32 +0,0 @@ -package com.myoa.engineering.crawl.ppomppu.processor.domain; - -import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName; -import javax.persistence.Column; -import javax.persistence.Entity; -import javax.persistence.EnumType; -import javax.persistence.Enumerated; -import javax.persistence.GeneratedValue; -import javax.persistence.GenerationType; -import javax.persistence.Id; -import javax.persistence.Table; -import lombok.Getter; -import lombok.NoArgsConstructor; - -@Getter -@NoArgsConstructor -@Entity -@Table(name = "subscribed_board") -public class SubscribedBoard extends Auditable { - - @Id - @GeneratedValue(strategy = GenerationType.IDENTITY) - private Long id; - - @Column - private Long userId; - - @Column - @Enumerated(EnumType.STRING) - private PpomppuBoardName boardName; - -} diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/repository/BaseScanRepository.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/repository/BaseScanRepository.java deleted file mode 100644 index ebbfedc..0000000 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/repository/BaseScanRepository.java +++ /dev/null @@ -1,4 +0,0 @@ -package com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository; - -public interface BaseScanRepository { -} diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/PpomppuArticleParser.java b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/parser/PpomppuArticleParser.java similarity index 95% rename from processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/PpomppuArticleParser.java rename to processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/parser/PpomppuArticleParser.java index 4387642..58e7b25 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/PpomppuArticleParser.java +++ b/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/parser/PpomppuArticleParser.java @@ -1,4 +1,4 @@ -package com.myoa.engineering.crawl.ppomppu.processor.dto; +package com.myoa.engineering.crawl.ppomppu.processor.parser; import java.time.Instant; import java.time.ZoneId; @@ -8,7 +8,7 @@ import org.jsoup.nodes.Element; import org.jsoup.select.Elements; import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle; -import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName; +import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName; /** * PpomppuArticleTransformer diff --git a/processor/src/test/resources/testdata/zboard/file1.html b/processor/src/test/resources/testdata/zboard/file1.html new file mode 100644 index 0000000..b9268d3 --- /dev/null +++ b/processor/src/test/resources/testdata/zboard/file1.html @@ -0,0 +1,1100 @@ + + + + + + + + + + + + + + + + 뽐뿌 - 뽐뿌게시판 + + + + + + + + + + + + + + + + + + + + +
+
+
+
+ +
+
+
+ +
+
+
+
 
+
+
+
+ 뽐뿌 + 뽐뿌 +
+ +
+
+
+
+ + +
+ +
+ + +
+
+ +
+ +
+ + +
+ + + + +
+ 로그인 + 회원가입|아이디비번찾기 +
+
+
+ +
+ +
+

뽐뿌게시판 입니다.

+ +
+
+ 가성비 좋은 상품과 쇼핑정보를 공유하는 공간입니다. [뽐뿌게시판 이용규칙] +
+ + + + + + + + + + + + + + + + + + + +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
   번호     글쓴이  제목 등록일 추천    조회   
공지관리자3[필독] 제목에는 특수 문자 사용을 절대 금지합니다.- 2010년 2월 24일0 4324677
알림관리자아이콘알리익스프레스 상품은 [알리뽐뿌]에 공유해 주세요!
알림관리자아이콘[키워드 알림] 관심 키워드 등록하고, 앱push 알림 받자!
537951매력적인뽐뿌 + +
+
+ [지마켓]천연 향수 비누 사봉파퓨메 135g 3개+사은품 (9,950원/무료)1 +
+ [기타] +
1069
537950123aaa + +
+
+ [지마켓]순면 국내생산 무지티3장 (유클12,410원/무배) +
+ [의류/잡화] +
1508
537949유유자적루 + +
+
+ [네이버-미루몰] 벨루아체 올인원 포맨 4종 (29,900원/무료)5 +
+ [화장품] +
1201
537947illilliiill + +
+
+ [티몬]역대가 핫딜! 슈퍼대디 세이프 77평량 아기물티슈 (17,991원/무배)74  +
+ [기타] +
9 - 0 14917
5379461회제공량 + +
+
+ [CGV씨네샵]타카라토미 버즈라이트이어 49,000 / 3,0006 +
+ [기타] +
9522
537945가을말차라떼 + +
+
+ [위메프]샤오미 스마트 선풍기 7세대 1X 업그레이드/Type-C 충전 포트 (43,160원/무배)16 +
+ [기타] +
1 - 0 17666
537944거북반 + +
+
+ [무신사]FCMM 풋볼 프로 스트레치 윈드브레이커 (32,364원/무료)3 +
+ [의류/잡화] +
13039
537942jimin1120 + +
+
+ 북유럽 스타일 양손 스프볼 도자기 스프그릇 소스접시 짱구그릇 라면그릇 만두접시 특가전(8,400원~/무료)23 +
+ [기타] +
2 - 0 5767
537941너구리사랑 + +
+
+ [롯데온]제스프리 골드키위, 토레타, 가자미밥상, 쿡반 새우볶음밥(다양/무료)20 +
+ [식품/건강] +
16 - 0 10050
537940깡패건달 + +
+
+ [자사몰]푸마 메쉬 폼스트라이프 에디션 5종 세트 (18,220원/무료)6 +
+ [의류/잡화] +
4 - 0 7273
537939심의켄 + +
+
+ [위메프]오리온 비쵸비 초코비스킷5pX8개 외 치즈톡/감자톡/오징어톡,도도한 나쵸(13,220원/무료)10 +
+ [식품/건강] +
8 - 0 7724
537938벌꿀오소리는강해 + +
+
+ [네이버]추억의 오란다 250g x2봉(6,200원/무배)9 +
+ [식품/건강] +
5892
537937매카aa + +
+
+ [G마켓]휴대폰 젤리케이스1+1 갤럭시,아이폰 (3,610원/무배)5 +
+ [기타] +
5186
537936나무007 + +
+
+ [CJ온스타일]동원 지중해 황도 410g 12캔(16,110/무료)44 +
+ [식품/건강] +
15 - 0 13237
537935IilliillIi + +
+
+ [네이버]맥주효모 비오틴 아누카사과 프리미엄 1박스 (1개월분) (34,900원/무료)2  +
+ [식품/건강] +
3597
537934최군어뭉 + +
+
+ [위메프]델리만쥬 붕어빵/핫도그 600g 2봉 (8,700/무료)15 +
+ [식품/건강] +
6 - 0 6264
537933무념무상해 + +
+
+ [인터파크]산리오 미니 말랑스티커/케이크하우스 외 다수 (2,970원부터/3개 이상 주문시 무료) +
+ [육아] +
1 - 0 6198
537931p1ano + +
+
+ [위메프]비오틴 울트라 4개월분 (4,900원/무배)8  +
+ [식품/건강] +
3 - 0 6065
537930킹갓제네럴MZ + +
+
+ [위메프]아스트로글라이드젤 148ml 1+1 (27,470원/무료배송)44 +
+ [기타] +
1 - 0 16634
537929꿉지 + +
+
+ [옥션]다농바이오 가무치소주 43도 375ml(32,900원/무료)3 +
+ [식품/건강] +
1 - 0 5952
쇼핑뽐뿌 아이콘아스라슬 + +
+
+ [카카오톡 스토어] 캘빈클라인 코튼 반팔티 15,900원 외 폴로반팔티, 반바지, PK카라티, 원피스, (무배,무교,무반) + 256 +
+
+ 		56 - 05325
쇼핑뽐뿌 아이콘LG전자쿠팡공식대행... + +
+
+ [쿠팡] LG 그램 주말 특가 찬스! 와우 즉시할인+농협카드 추가 할인+내일 배송 도착 + 246 +
+
+ 		57 - 03128
+
+

+ 쇼핑포럼뽐뿌상담실 아이콘더 많은 쇼핑 정보와 구매 전 자유로운 쇼핑 이야기는 쇼핑포럼에서! +

+
+
핫딜 아이콘 + 쿠팡 + + +
+
+ 나이키 에브리데이 쿠션크루 트레이닝삭스 장목양말 1세트 3켤레입 (9,490원) +
+
+ 		7
쇼핑포럼 아이콘 + ll427ll + + +
+
+ [알리]펩시제로 라임향 355ml 24캔(13,582원/무료) +
+ [식품/건강]
+ 		907
쇼핑포럼 아이콘 + HBAF + + +
+
+  국산 갈치 1마리 39800원 +
+ [식품/건강]
+ 		1 - 010894
쇼핑포럼 아이콘 + 시뻘건맛 + + +
+
+  남자 집에서 입을 반바지 통풍잘되는거 있을까요? +
+ [의류/잡화]
+ 		10935
+
+
+ +
+
+
+
+
+
+ + + + +
+ +
+
+

+ 실시간 질렀어요 +
    +
  1. 1
  2. 2
+

+ +
    +
  • +
    • +
  • +
    • +
+
+ +
+

+ 최근 내가 본 상품 +
    +
+

+
    +
  • 최근 내가 본 상품이 없습니다.
+
+
+ + + + + + +
+ +  -새로고침  -다음페이지   +
+ + + + + + + +
+ + 1 2 3 4 5 6 7 8 9 10 다음 +
+ + + + + + + + + + + + +
+ + + + + +
+ + + + + + + + + +
+
+ + + + +
+ +
+ + +
+ + + + + + + +
+
+ + + +
+ +
+ + 사이트소개 + 운영참여 + + + 게시중단요청 + + 이용약관 + 개인정보처리방침 + 청소년보호정책 + 불법촬영물등 신고 + 광고/제휴 + 메일문의 + + + 뽐뿌채용 + + 다크모드 +
+ + + + + +
+
+
+

뽐뿌는 쇼핑 정보 커뮤니티입니다. 상품 주문, 배송 및 환불의 의무와 책임은 각 판매 업체에 있습니다.

+

(주)뽐뿌커뮤니케이션 / 대표이사 : 고재성 / 사업자등록번호 : 117-81-66008 / 통신판매업신고번호 : 제2015-서울영등포-1441호

+

주소 : 서울특별시 영등포구 당산로 41길 당산 SK V1센터 W동 1411호 + / 팩스 : 02-6280-7029 / 사업자회원 FAQ

+ +
+ + Copyright 2005-2024 ppomppu. All rights reserved +
+ +
+
+ + + + + + + + + + + + + + + + + diff --git a/receiver/build.gradle b/reader/build.gradle similarity index 100% rename from receiver/build.gradle rename to reader/build.gradle diff --git a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/ReceiverApplication.java b/reader/src/main/java/com/myoa/engineering/crawl/ppomppu/reader/ReaderApplication.java similarity index 52% rename from receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/ReceiverApplication.java rename to reader/src/main/java/com/myoa/engineering/crawl/ppomppu/reader/ReaderApplication.java index 339a01d..b50e587 100644 --- a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/ReceiverApplication.java +++ b/reader/src/main/java/com/myoa/engineering/crawl/ppomppu/reader/ReaderApplication.java @@ -1,12 +1,10 @@ -package com.myoa.engineering.crawl.ppomppu.receiver; +package com.myoa.engineering.crawl.ppomppu.reader; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Import; -import com.myoa.engineering.crawl.ppomppu.receiver.configuration.properties.TelegramBotProperties; -import com.myoa.engineering.crawl.ppomppu.support.webclient.PpomppuNotifierWebClientConfiguration; +import com.myoa.engineering.crawl.shopping.support.webclient.PpomppuNotifierWebClientConfiguration; /** * ReceiverApplication @@ -17,9 +15,9 @@ import com.myoa.engineering.crawl.ppomppu.support.webclient.PpomppuNotifierWebCl @Import({ PpomppuNotifierWebClientConfiguration.class}) @SpringBootApplication // @EnableConfigurationProperties({ TelegramBotProperties.class }) -public class ReceiverApplication { +public class ReaderApplication { public static void main(String[] args) { - SpringApplication.run(ReceiverApplication.class, args); + SpringApplication.run(ReaderApplication.class, args); } } diff --git a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/configuration/TelegramBotConfiguration.java b/reader/src/main/java/com/myoa/engineering/crawl/ppomppu/reader/configuration/TelegramBotConfiguration.java similarity index 73% rename from receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/configuration/TelegramBotConfiguration.java rename to reader/src/main/java/com/myoa/engineering/crawl/ppomppu/reader/configuration/TelegramBotConfiguration.java index dd120dc..2370c11 100644 --- a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/configuration/TelegramBotConfiguration.java +++ b/reader/src/main/java/com/myoa/engineering/crawl/ppomppu/reader/configuration/TelegramBotConfiguration.java @@ -1,4 +1,4 @@ -package com.myoa.engineering.crawl.ppomppu.receiver.configuration; +package com.myoa.engineering.crawl.ppomppu.reader.configuration; import java.util.List; @@ -8,10 +8,10 @@ import org.telegram.telegrambots.meta.TelegramBotsApi; import org.telegram.telegrambots.meta.exceptions.TelegramApiException; import org.telegram.telegrambots.updatesreceivers.DefaultBotSession; -import com.myoa.engineering.crawl.ppomppu.receiver.configuration.properties.TelegramBotProperties; -import com.myoa.engineering.crawl.ppomppu.receiver.configuration.properties.TelegramBotProperties.TelegramBotPropertiesUnit; -import com.myoa.engineering.crawl.ppomppu.receiver.dispatch.MessageDispatcher; -import com.myoa.engineering.crawl.ppomppu.receiver.handler.message.MessageHandler; +import com.myoa.engineering.crawl.ppomppu.reader.configuration.properties.TelegramBotProperties; +import com.myoa.engineering.crawl.ppomppu.reader.configuration.properties.TelegramBotProperties.TelegramBotPropertiesUnit; +import com.myoa.engineering.crawl.ppomppu.reader.dispatch.MessageDispatcher; +import com.myoa.engineering.crawl.ppomppu.reader.handler.message.MessageHandler; /** * TelegramBotConfiguration diff --git a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/configuration/properties/TelegramBotProperties.java b/reader/src/main/java/com/myoa/engineering/crawl/ppomppu/reader/configuration/properties/TelegramBotProperties.java similarity index 83% rename from receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/configuration/properties/TelegramBotProperties.java rename to reader/src/main/java/com/myoa/engineering/crawl/ppomppu/reader/configuration/properties/TelegramBotProperties.java index e438889..04113aa 100644 --- a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/configuration/properties/TelegramBotProperties.java +++ b/reader/src/main/java/com/myoa/engineering/crawl/ppomppu/reader/configuration/properties/TelegramBotProperties.java @@ -1,16 +1,13 @@ -package com.myoa.engineering.crawl.ppomppu.receiver.configuration.properties; +package com.myoa.engineering.crawl.ppomppu.reader.configuration.properties; import java.util.ArrayList; import java.util.List; import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.boot.context.properties.ConstructorBinding; import org.springframework.stereotype.Component; -import lombok.AllArgsConstructor; import lombok.Data; import lombok.Getter; -import lombok.NoArgsConstructor; import lombok.Setter; /** diff --git a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/controller/v1/EventAPIController.java b/reader/src/main/java/com/myoa/engineering/crawl/ppomppu/reader/controller/v1/EventAPIController.java similarity index 79% rename from receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/controller/v1/EventAPIController.java rename to reader/src/main/java/com/myoa/engineering/crawl/ppomppu/reader/controller/v1/EventAPIController.java index d67089c..c071d8c 100644 --- a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/controller/v1/EventAPIController.java +++ b/reader/src/main/java/com/myoa/engineering/crawl/ppomppu/reader/controller/v1/EventAPIController.java @@ -1,12 +1,12 @@ -package com.myoa.engineering.crawl.ppomppu.receiver.controller.v1; +package com.myoa.engineering.crawl.ppomppu.reader.controller.v1; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; -import com.myoa.engineering.crawl.ppomppu.receiver.service.ProcessorAPIService; -import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName; +import com.myoa.engineering.crawl.ppomppu.reader.service.ProcessorAPIService; +import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName; import reactor.core.publisher.Mono; diff --git a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/dispatch/MessageDispatcher.java b/reader/src/main/java/com/myoa/engineering/crawl/ppomppu/reader/dispatch/MessageDispatcher.java similarity index 90% rename from receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/dispatch/MessageDispatcher.java rename to reader/src/main/java/com/myoa/engineering/crawl/ppomppu/reader/dispatch/MessageDispatcher.java index 22ea270..3c20384 100644 --- a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/dispatch/MessageDispatcher.java +++ b/reader/src/main/java/com/myoa/engineering/crawl/ppomppu/reader/dispatch/MessageDispatcher.java @@ -1,4 +1,4 @@ -package com.myoa.engineering.crawl.ppomppu.receiver.dispatch; +package com.myoa.engineering.crawl.ppomppu.reader.dispatch; import java.util.List; @@ -6,7 +6,7 @@ import org.telegram.telegrambots.bots.TelegramLongPollingBot; import org.telegram.telegrambots.meta.api.objects.Message; import org.telegram.telegrambots.meta.api.objects.Update; -import com.myoa.engineering.crawl.ppomppu.receiver.handler.message.MessageHandler; +import com.myoa.engineering.crawl.ppomppu.reader.handler.message.MessageHandler; import lombok.extern.slf4j.Slf4j; diff --git a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/HelloWorldMessageHandler.java b/reader/src/main/java/com/myoa/engineering/crawl/ppomppu/reader/handler/message/HelloWorldMessageHandler.java similarity index 76% rename from receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/HelloWorldMessageHandler.java rename to reader/src/main/java/com/myoa/engineering/crawl/ppomppu/reader/handler/message/HelloWorldMessageHandler.java index ee5f83e..5b3f4df 100644 --- a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/HelloWorldMessageHandler.java +++ b/reader/src/main/java/com/myoa/engineering/crawl/ppomppu/reader/handler/message/HelloWorldMessageHandler.java @@ -1,6 +1,6 @@ -package com.myoa.engineering.crawl.ppomppu.receiver.handler.message; +package com.myoa.engineering.crawl.ppomppu.reader.handler.message; -import com.myoa.engineering.crawl.ppomppu.support.util.ObjectUtil; +import com.myoa.engineering.crawl.shopping.support.util.ObjectUtil; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; import org.telegram.telegrambots.meta.api.objects.Message; diff --git a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/ImageMessageHandler.java b/reader/src/main/java/com/myoa/engineering/crawl/ppomppu/reader/handler/message/ImageMessageHandler.java similarity index 70% rename from receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/ImageMessageHandler.java rename to reader/src/main/java/com/myoa/engineering/crawl/ppomppu/reader/handler/message/ImageMessageHandler.java index b8cc1bc..6f3ed72 100644 --- a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/ImageMessageHandler.java +++ b/reader/src/main/java/com/myoa/engineering/crawl/ppomppu/reader/handler/message/ImageMessageHandler.java @@ -1,4 +1,4 @@ -package com.myoa.engineering.crawl.ppomppu.receiver.handler.message; +package com.myoa.engineering.crawl.ppomppu.reader.handler.message; /** * ImageHandler diff --git a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/MessageHandler.java b/reader/src/main/java/com/myoa/engineering/crawl/ppomppu/reader/handler/message/MessageHandler.java similarity index 80% rename from receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/MessageHandler.java rename to reader/src/main/java/com/myoa/engineering/crawl/ppomppu/reader/handler/message/MessageHandler.java index 01e44f0..a948fd5 100644 --- a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/MessageHandler.java +++ b/reader/src/main/java/com/myoa/engineering/crawl/ppomppu/reader/handler/message/MessageHandler.java @@ -1,4 +1,4 @@ -package com.myoa.engineering.crawl.ppomppu.receiver.handler.message; +package com.myoa.engineering.crawl.ppomppu.reader.handler.message; import org.telegram.telegrambots.meta.api.objects.Message; diff --git a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/TextMessageHandler.java b/reader/src/main/java/com/myoa/engineering/crawl/ppomppu/reader/handler/message/TextMessageHandler.java similarity index 74% rename from receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/TextMessageHandler.java rename to reader/src/main/java/com/myoa/engineering/crawl/ppomppu/reader/handler/message/TextMessageHandler.java index 07c185e..5e0c194 100644 --- a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/TextMessageHandler.java +++ b/reader/src/main/java/com/myoa/engineering/crawl/ppomppu/reader/handler/message/TextMessageHandler.java @@ -1,6 +1,6 @@ -package com.myoa.engineering.crawl.ppomppu.receiver.handler.message; +package com.myoa.engineering.crawl.ppomppu.reader.handler.message; -import com.myoa.engineering.crawl.ppomppu.support.util.ObjectUtil; +import com.myoa.engineering.crawl.shopping.support.util.ObjectUtil; import org.telegram.telegrambots.meta.api.objects.Message; /** diff --git a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/text/CommandHandler.java b/reader/src/main/java/com/myoa/engineering/crawl/ppomppu/reader/handler/message/text/CommandHandler.java similarity index 89% rename from receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/text/CommandHandler.java rename to reader/src/main/java/com/myoa/engineering/crawl/ppomppu/reader/handler/message/text/CommandHandler.java index 3c22204..f249a86 100644 --- a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/text/CommandHandler.java +++ b/reader/src/main/java/com/myoa/engineering/crawl/ppomppu/reader/handler/message/text/CommandHandler.java @@ -1,11 +1,11 @@ -package com.myoa.engineering.crawl.ppomppu.receiver.handler.message.text; +package com.myoa.engineering.crawl.ppomppu.reader.handler.message.text; import java.util.List; import org.springframework.stereotype.Component; import org.telegram.telegrambots.meta.api.objects.Message; -import com.myoa.engineering.crawl.ppomppu.receiver.handler.message.TextMessageHandler; +import com.myoa.engineering.crawl.ppomppu.reader.handler.message.TextMessageHandler; import lombok.extern.slf4j.Slf4j; diff --git a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/text/CommonTextHandler.java b/reader/src/main/java/com/myoa/engineering/crawl/ppomppu/reader/handler/message/text/CommonTextHandler.java similarity index 79% rename from receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/text/CommonTextHandler.java rename to reader/src/main/java/com/myoa/engineering/crawl/ppomppu/reader/handler/message/text/CommonTextHandler.java index 761e14f..46f833e 100644 --- a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/text/CommonTextHandler.java +++ b/reader/src/main/java/com/myoa/engineering/crawl/ppomppu/reader/handler/message/text/CommonTextHandler.java @@ -1,9 +1,9 @@ -package com.myoa.engineering.crawl.ppomppu.receiver.handler.message.text; +package com.myoa.engineering.crawl.ppomppu.reader.handler.message.text; import org.springframework.stereotype.Component; import org.telegram.telegrambots.meta.api.objects.Message; -import com.myoa.engineering.crawl.ppomppu.receiver.handler.message.TextMessageHandler; +import com.myoa.engineering.crawl.ppomppu.reader.handler.message.TextMessageHandler; import lombok.extern.slf4j.Slf4j; diff --git a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/text/EmptyTextCommandProcessor.java b/reader/src/main/java/com/myoa/engineering/crawl/ppomppu/reader/handler/message/text/EmptyTextCommandProcessor.java similarity index 87% rename from receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/text/EmptyTextCommandProcessor.java rename to reader/src/main/java/com/myoa/engineering/crawl/ppomppu/reader/handler/message/text/EmptyTextCommandProcessor.java index e0c4449..bb79c61 100644 --- a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/text/EmptyTextCommandProcessor.java +++ b/reader/src/main/java/com/myoa/engineering/crawl/ppomppu/reader/handler/message/text/EmptyTextCommandProcessor.java @@ -1,4 +1,4 @@ -package com.myoa.engineering.crawl.ppomppu.receiver.handler.message.text; +package com.myoa.engineering.crawl.ppomppu.reader.handler.message.text; import org.springframework.stereotype.Component; import org.telegram.telegrambots.meta.api.objects.Message; diff --git a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/text/StartTextCommandProcessor.java b/reader/src/main/java/com/myoa/engineering/crawl/ppomppu/reader/handler/message/text/StartTextCommandProcessor.java similarity index 89% rename from receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/text/StartTextCommandProcessor.java rename to reader/src/main/java/com/myoa/engineering/crawl/ppomppu/reader/handler/message/text/StartTextCommandProcessor.java index 08c8315..157ee0b 100644 --- a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/text/StartTextCommandProcessor.java +++ b/reader/src/main/java/com/myoa/engineering/crawl/ppomppu/reader/handler/message/text/StartTextCommandProcessor.java @@ -1,4 +1,4 @@ -package com.myoa.engineering.crawl.ppomppu.receiver.handler.message.text; +package com.myoa.engineering.crawl.ppomppu.reader.handler.message.text; import org.springframework.stereotype.Component; import org.telegram.telegrambots.meta.api.objects.Message; diff --git a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/text/TextCommandCode.java b/reader/src/main/java/com/myoa/engineering/crawl/ppomppu/reader/handler/message/text/TextCommandCode.java similarity index 90% rename from receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/text/TextCommandCode.java rename to reader/src/main/java/com/myoa/engineering/crawl/ppomppu/reader/handler/message/text/TextCommandCode.java index 364d655..1729af0 100644 --- a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/text/TextCommandCode.java +++ b/reader/src/main/java/com/myoa/engineering/crawl/ppomppu/reader/handler/message/text/TextCommandCode.java @@ -1,4 +1,4 @@ -package com.myoa.engineering.crawl.ppomppu.receiver.handler.message.text; +package com.myoa.engineering.crawl.ppomppu.reader.handler.message.text; import java.util.Arrays; diff --git a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/text/TextCommandProcessor.java b/reader/src/main/java/com/myoa/engineering/crawl/ppomppu/reader/handler/message/text/TextCommandProcessor.java similarity index 80% rename from receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/text/TextCommandProcessor.java rename to reader/src/main/java/com/myoa/engineering/crawl/ppomppu/reader/handler/message/text/TextCommandProcessor.java index 3f8ff49..2598d3c 100644 --- a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/text/TextCommandProcessor.java +++ b/reader/src/main/java/com/myoa/engineering/crawl/ppomppu/reader/handler/message/text/TextCommandProcessor.java @@ -1,4 +1,4 @@ -package com.myoa.engineering.crawl.ppomppu.receiver.handler.message.text; +package com.myoa.engineering.crawl.ppomppu.reader.handler.message.text; import org.telegram.telegrambots.meta.api.objects.Message; diff --git a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/infrastructure/client/ProcessorAPIWebClient.java b/reader/src/main/java/com/myoa/engineering/crawl/ppomppu/reader/infrastructure/client/ProcessorAPIWebClient.java similarity index 75% rename from receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/infrastructure/client/ProcessorAPIWebClient.java rename to reader/src/main/java/com/myoa/engineering/crawl/ppomppu/reader/infrastructure/client/ProcessorAPIWebClient.java index 2dbca2a..a55241b 100644 --- a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/infrastructure/client/ProcessorAPIWebClient.java +++ b/reader/src/main/java/com/myoa/engineering/crawl/ppomppu/reader/infrastructure/client/ProcessorAPIWebClient.java @@ -1,15 +1,15 @@ -package com.myoa.engineering.crawl.ppomppu.receiver.infrastructure.client; +package com.myoa.engineering.crawl.ppomppu.reader.infrastructure.client; import org.springframework.core.ParameterizedTypeReference; import org.springframework.stereotype.Component; import org.springframework.web.reactive.function.client.WebClient; import org.springframework.web.reactive.function.client.WebClientRequestException; -import com.myoa.engineering.crawl.ppomppu.receiver.dto.constant.WebClientPropertiesUnitName; -import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName; -import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebClientFilterFactory; -import com.myoa.engineering.crawl.ppomppu.support.webclient.properties.WebClientProperties; -import com.myoa.engineering.crawl.ppomppu.support.webclient.properties.WebClientProperties.WebClientPropertiesUnit; +import com.myoa.engineering.crawl.ppomppu.reader.dto.constant.WebClientPropertiesUnitName; +import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName; +import com.myoa.engineering.crawl.shopping.support.webclient.factory.WebClientFilterFactory; +import com.myoa.engineering.crawl.shopping.support.webclient.properties.WebClientProperties; +import com.myoa.engineering.crawl.shopping.support.webclient.properties.WebClientProperties.WebClientPropertiesUnit; import lombok.extern.slf4j.Slf4j; import reactor.core.publisher.Mono; diff --git a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/scheduler/ParseEventEmitter.java b/reader/src/main/java/com/myoa/engineering/crawl/ppomppu/reader/scheduler/ParseEventEmitter.java similarity index 79% rename from receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/scheduler/ParseEventEmitter.java rename to reader/src/main/java/com/myoa/engineering/crawl/ppomppu/reader/scheduler/ParseEventEmitter.java index bf94c88..ac47fbf 100644 --- a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/scheduler/ParseEventEmitter.java +++ b/reader/src/main/java/com/myoa/engineering/crawl/ppomppu/reader/scheduler/ParseEventEmitter.java @@ -1,11 +1,11 @@ -package com.myoa.engineering.crawl.ppomppu.receiver.scheduler; +package com.myoa.engineering.crawl.ppomppu.reader.scheduler; import org.springframework.scheduling.annotation.EnableScheduling; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; -import com.myoa.engineering.crawl.ppomppu.receiver.service.ProcessorAPIService; -import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName; +import com.myoa.engineering.crawl.ppomppu.reader.service.ProcessorAPIService; +import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName; import lombok.extern.slf4j.Slf4j; diff --git a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/service/ProcessorAPIService.java b/reader/src/main/java/com/myoa/engineering/crawl/ppomppu/reader/service/ProcessorAPIService.java similarity index 72% rename from receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/service/ProcessorAPIService.java rename to reader/src/main/java/com/myoa/engineering/crawl/ppomppu/reader/service/ProcessorAPIService.java index b793dbb..705cbeb 100644 --- a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/service/ProcessorAPIService.java +++ b/reader/src/main/java/com/myoa/engineering/crawl/ppomppu/reader/service/ProcessorAPIService.java @@ -1,9 +1,9 @@ -package com.myoa.engineering.crawl.ppomppu.receiver.service; +package com.myoa.engineering.crawl.ppomppu.reader.service; import org.springframework.stereotype.Service; -import com.myoa.engineering.crawl.ppomppu.receiver.infrastructure.client.ProcessorAPIWebClient; -import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName; +import com.myoa.engineering.crawl.ppomppu.reader.infrastructure.client.ProcessorAPIWebClient; +import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName; import lombok.extern.slf4j.Slf4j; import reactor.core.publisher.Mono; diff --git a/receiver/src/main/resources/application-development.yml b/reader/src/main/resources/application-development.yml similarity index 100% rename from receiver/src/main/resources/application-development.yml rename to reader/src/main/resources/application-development.yml diff --git a/receiver/src/main/resources/application-local.yml b/reader/src/main/resources/application-local.yml similarity index 100% rename from receiver/src/main/resources/application-local.yml rename to reader/src/main/resources/application-local.yml diff --git a/receiver/src/main/resources/application-production.yml b/reader/src/main/resources/application-production.yml similarity index 100% rename from receiver/src/main/resources/application-production.yml rename to reader/src/main/resources/application-production.yml diff --git a/receiver/src/main/resources/application.yml b/reader/src/main/resources/application.yml similarity index 100% rename from receiver/src/main/resources/application.yml rename to reader/src/main/resources/application.yml diff --git a/receiver/src/main/resources/logback-spring.xml b/reader/src/main/resources/logback-spring.xml similarity index 100% rename from receiver/src/main/resources/logback-spring.xml rename to reader/src/main/resources/logback-spring.xml diff --git a/receiver/src/main/resources/logback/component/logback-file.xml b/reader/src/main/resources/logback/component/logback-file.xml similarity index 100% rename from receiver/src/main/resources/logback/component/logback-file.xml rename to reader/src/main/resources/logback/component/logback-file.xml diff --git a/receiver/src/main/resources/logback/logback-development.xml b/reader/src/main/resources/logback/logback-development.xml similarity index 100% rename from receiver/src/main/resources/logback/logback-development.xml rename to reader/src/main/resources/logback/logback-development.xml diff --git a/receiver/src/main/resources/logback/logback-production.xml b/reader/src/main/resources/logback/logback-production.xml similarity index 100% rename from receiver/src/main/resources/logback/logback-production.xml rename to reader/src/main/resources/logback/logback-production.xml diff --git a/receiver-1.0.1.jar b/receiver-1.0.1.jar deleted file mode 100644 index cc03e1a..0000000 Binary files a/receiver-1.0.1.jar and /dev/null differ diff --git a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/dto/FeedParsedResult.java b/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/dto/FeedParsedResult.java deleted file mode 100644 index 20d0c5e..0000000 --- a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/dto/FeedParsedResult.java +++ /dev/null @@ -1,25 +0,0 @@ -package com.myoa.engineering.crawl.ppomppu.receiver.dto; - -import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName; -import java.io.Serializable; -import java.time.Instant; -import lombok.Getter; -import lombok.NoArgsConstructor; - -/** - * FeedParsedResult - * - * @author Shin Woo-jin (woozu.shin@kakaoent.com) - * @since 2021-09-08 - */ -@Getter -@NoArgsConstructor -public class FeedParsedResult implements Serializable { - - private static final long serialVersionUID = -3771310078623481348L; - - private PpomppuBoardName boardName; - private Instant requestedAt; - private Instant processedAt; - -} diff --git a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/dto/constant/WebClientPropertiesUnitName.java b/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/dto/constant/WebClientPropertiesUnitName.java deleted file mode 100644 index bdab14d..0000000 --- a/receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/dto/constant/WebClientPropertiesUnitName.java +++ /dev/null @@ -1,19 +0,0 @@ -package com.myoa.engineering.crawl.ppomppu.receiver.dto.constant; - -import lombok.AllArgsConstructor; -import lombok.Getter; - -/** - * WebClientPropertiesUnitName - * @author Shin Woo-jin (woo-jin.shin@linecorp.com) - * @since 2021-11-18 - * - */ -@Getter -@AllArgsConstructor -public enum WebClientPropertiesUnitName { - PPOMPPU_NOTIFIER_PROCESSOR_API("ppn-processor-api"), - ; - - private String unitName; -} diff --git a/sender-1.0.1.jar b/sender-1.0.1.jar deleted file mode 100644 index ff87444..0000000 Binary files a/sender-1.0.1.jar and /dev/null differ diff --git a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/SenderApplication.java b/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/SenderApplication.java deleted file mode 100644 index 20f8126..0000000 --- a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/SenderApplication.java +++ /dev/null @@ -1,22 +0,0 @@ -package com.myoa.engineering.crawl.ppomppu.sender; - -import org.springframework.boot.SpringApplication; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.context.annotation.Import; - -import com.myoa.engineering.crawl.ppomppu.support.webclient.PpomppuNotifierWebClientConfiguration; - -/** - * SenderApplication - * @author Shin Woo-jin (woo-jin.shin@linecorp.com) - * @since 2021-08-20 - * - */ -@Import({ PpomppuNotifierWebClientConfiguration.class }) -@SpringBootApplication -public class SenderApplication { - - public static void main(String[] args) { - SpringApplication.run(SenderApplication.class, args); - } -} diff --git a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/infrastructure/client/MessageSender.java b/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/infrastructure/client/MessageSender.java deleted file mode 100644 index 4153bb4..0000000 --- a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/infrastructure/client/MessageSender.java +++ /dev/null @@ -1,17 +0,0 @@ -package com.myoa.engineering.crawl.ppomppu.sender.infrastructure.client; - -import com.myoa.engineering.crawl.ppomppu.sender.dto.MessageDTO; - -import reactor.core.publisher.Mono; - -/** - * MessageSender - * @author Shin Woo-jin (woo-jin.shin@linecorp.com) - * @since 2021-11-14 - * - */ -public interface MessageSender { - - Mono sendMessage(T message); - -} diff --git a/settings.gradle b/settings.gradle index bc4de22..ab70af1 100644 --- a/settings.gradle +++ b/settings.gradle @@ -1,14 +1,14 @@ rootProject.name = 'PpomppuNotifier' -include 'receiver' -project(':receiver').projectDir = "$rootDir/receiver" as File - -include 'processor' -project(':processor').projectDir = "$rootDir/processor" as File - -include 'sender' -project(':sender').projectDir = "$rootDir/sender" as File +//include 'reader' +//project(':reader').projectDir = "$rootDir/reader" as File +//include 'processor' +//project(':processor').projectDir = "$rootDir/processor" as File +//include 'writer' +//project(':writer').projectDir = "$rootDir/writer" as File +include 'shopping-crawler' +project(':shopping-crawler').projectDir = "$rootDir/shopping-crawler" as File include 'support' project(':support').projectDir = "$rootDir/support" as File diff --git a/shopping-crawler/.jqwik-database b/shopping-crawler/.jqwik-database new file mode 100644 index 0000000..711006c Binary files /dev/null and b/shopping-crawler/.jqwik-database differ diff --git a/shopping-crawler/build.gradle b/shopping-crawler/build.gradle new file mode 100644 index 0000000..89c7625 --- /dev/null +++ b/shopping-crawler/build.gradle @@ -0,0 +1,42 @@ +dependencies { + developmentOnly 'org.springframework.boot:spring-boot-devtools' + runtimeOnly 'com.h2database:h2' + runtimeOnly 'com.mysql:mysql-connector-j' + compileOnly 'org.projectlombok:lombok' + + implementation project(':support') + // https://projectreactor.io/docs/core/release/reference/#debug-activate + implementation("org.springframework.boot:spring-boot-starter-web") { + exclude group: "org.springframework.boot", module: "spring-boot-starter-tomcat" + } + + implementation("org.springframework.boot:spring-boot-starter-undertow") { + exclude group: "io.undertow", module: "undertow-websockets-jsr" + } + implementation 'org.springframework.boot:spring-boot-starter-data-jpa' + implementation 'org.springframework.boot:spring-boot-configuration-processor' + implementation 'org.springframework.cloud:spring-cloud-starter-config' + implementation 'org.springframework.boot:spring-boot-starter-actuator' + implementation 'com.rometools:rome:2.1.0' + implementation 'org.jsoup:jsoup:1.17.2' + implementation 'com.h2database:h2:2.2.224' + implementation "org.springframework.cloud:spring-cloud-starter-openfeign" + implementation "io.github.openfeign:feign-hc5" + implementation 'org.ahocorasick:ahocorasick:0.6.3' + + annotationProcessor 'org.springframework.boot:spring-boot-configuration-processor' + annotationProcessor 'org.projectlombok:lombok' + + testImplementation 'org.springframework.boot:spring-boot-starter-test' + testImplementation('org.assertj:assertj-core:3.25.3') + testImplementation("org.jeasy:easy-random-core:5.0.0") + testCompileOnly 'org.projectlombok:lombok' + testAnnotationProcessor('org.projectlombok:lombok') +} + +test { + useJUnitPlatform() + testLogging { + events "passed", "skipped", "failed" + } +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/ShoppingCrawlerApplication.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/ShoppingCrawlerApplication.java new file mode 100644 index 0000000..71444d8 --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/ShoppingCrawlerApplication.java @@ -0,0 +1,19 @@ +package com.myoa.engineering.crawl.shopping; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.cloud.openfeign.EnableFeignClients; +import org.springframework.scheduling.annotation.EnableScheduling; + +@EnableScheduling +@EnableFeignClients +@EnableConfigurationProperties +@SpringBootApplication +public class ShoppingCrawlerApplication { + + public static void main(String[] args) { + SpringApplication.run(ShoppingCrawlerApplication.class, args); + } + +} diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/H2ConsoleConfiguration.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/datasource/H2ConsoleConfiguration.java similarity index 93% rename from processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/H2ConsoleConfiguration.java rename to shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/datasource/H2ConsoleConfiguration.java index 3fd4bd3..9f9ab8b 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/H2ConsoleConfiguration.java +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/datasource/H2ConsoleConfiguration.java @@ -1,6 +1,5 @@ -package com.myoa.engineering.crawl.ppomppu.processor.configuration; +package com.myoa.engineering.crawl.shopping.configuration.datasource; -import java.sql.SQLException; import lombok.extern.slf4j.Slf4j; import org.h2.tools.Server; import org.springframework.beans.factory.annotation.Value; @@ -10,6 +9,8 @@ import org.springframework.context.event.ContextClosedEvent; import org.springframework.context.event.ContextRefreshedEvent; import org.springframework.context.event.EventListener; +import java.sql.SQLException; + @Slf4j @Profile({"datasource-local", "datasource-development"}) @Configuration diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/PpomppuDatasourceConfiguration.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/datasource/ShoppingCrawlerDatasourceConfiguration.java similarity index 68% rename from processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/PpomppuDatasourceConfiguration.java rename to shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/datasource/ShoppingCrawlerDatasourceConfiguration.java index b78c475..14df143 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/PpomppuDatasourceConfiguration.java +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/datasource/ShoppingCrawlerDatasourceConfiguration.java @@ -1,54 +1,58 @@ -package com.myoa.engineering.crawl.ppomppu.processor.configuration; +package com.myoa.engineering.crawl.shopping.configuration.datasource; -import com.myoa.engineering.crawl.ppomppu.processor.configuration.properties.DatasourceProperties; -import com.myoa.engineering.crawl.ppomppu.processor.configuration.properties.DatasourceProperties.DataSourcePropertiesUnit; -import com.myoa.engineering.crawl.ppomppu.processor.configuration.properties.HibernateProperties; -import com.myoa.engineering.crawl.ppomppu.processor.configuration.properties.HikariProperties; -import com.myoa.engineering.crawl.ppomppu.processor.domain.BaseScanDomain; -import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository.BaseScanRepository; +import com.myoa.engineering.crawl.shopping.configuration.datasource.properties.DatasourceProperties; +import com.myoa.engineering.crawl.shopping.configuration.datasource.properties.HibernateProperties; +import com.myoa.engineering.crawl.shopping.configuration.datasource.properties.HikariProperties; +import com.myoa.engineering.crawl.shopping.domain.entity.BaseScanDomain; +import com.myoa.engineering.crawl.shopping.infra.repository.BaseScanRepository; import com.zaxxer.hikari.HikariConfig; import com.zaxxer.hikari.HikariDataSource; -import java.util.Enumeration; -import java.util.HashMap; -import java.util.Map; -import java.util.Properties; -import javax.persistence.EntityManagerFactory; -import javax.sql.DataSource; +import jakarta.persistence.EntityManagerFactory; import lombok.NonNull; +import org.hibernate.boot.model.naming.CamelCaseToUnderscoresNamingStrategy; +import org.hibernate.boot.model.naming.ImplicitNamingStrategyJpaCompliantImpl; import org.hibernate.cfg.AvailableSettings; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.orm.jpa.EntityManagerFactoryBuilder; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.data.jpa.repository.config.EnableJpaAuditing; import org.springframework.data.jpa.repository.config.EnableJpaRepositories; import org.springframework.orm.jpa.JpaTransactionManager; import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean; import org.springframework.transaction.PlatformTransactionManager; -@Configuration -@EnableJpaRepositories(basePackageClasses = BaseScanRepository.class, - entityManagerFactoryRef = "ppomppuNotifierProcessorEntityManagerFactory", - transactionManagerRef = "ppomppuNotifierProcessorTransactionManager" -) -public class PpomppuDatasourceConfiguration { +import javax.sql.DataSource; +import java.util.Enumeration; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; - private static final String DATA_SOURCE_UNIT_NAME = "ppn_mysql"; +@Configuration +@EnableJpaAuditing +@EnableJpaRepositories(basePackageClasses = BaseScanRepository.class, + entityManagerFactoryRef = "shoppingCrawlerEntityManagerFactory", + transactionManagerRef = "shoppingCrawlerTransactionManager" +) +public class ShoppingCrawlerDatasourceConfiguration { + + private static final String DATA_SOURCE_UNIT_NAME = "crawler-shopping"; private final DatasourceProperties dataSourceProeprties; private final HikariProperties hikariProperties; private final HibernateProperties hibernateProperties; - public PpomppuDatasourceConfiguration(DatasourceProperties dataSourceProeprties, - HikariProperties hikariProperties, - HibernateProperties hibernateProperties) { + public ShoppingCrawlerDatasourceConfiguration(DatasourceProperties dataSourceProeprties, + HikariProperties hikariProperties, + HibernateProperties hibernateProperties) { this.dataSourceProeprties = dataSourceProeprties; this.hikariProperties = hikariProperties; this.hibernateProperties = hibernateProperties; } - @Bean(name = "ppomppuNotifierProcessorDataSource") + @Bean(name = "shoppingCrawlerDataSource") public DataSource dataSource() { - DataSourcePropertiesUnit dataSourcePropertiesUnit = dataSourceProeprties.find(DATA_SOURCE_UNIT_NAME); + DatasourceProperties.DataSourcePropertiesUnit dataSourcePropertiesUnit = dataSourceProeprties.find(DATA_SOURCE_UNIT_NAME); final HikariConfig hikariConfig = new HikariConfig(); hikariConfig.setJdbcUrl(dataSourcePropertiesUnit.toCompletedJdbcUrl()); @@ -69,19 +73,19 @@ public class PpomppuDatasourceConfiguration { return dataSource; } - @Bean("ppomppuNotifierProcessorEntityManagerFactory") + @Bean("shoppingCrawlerEntityManagerFactory") public LocalContainerEntityManagerFactoryBean entityManagerFactory( - EntityManagerFactoryBuilder builder, - @Qualifier("ppomppuNotifierProcessorDataSource") DataSource dataSource) { + EntityManagerFactoryBuilder builder, + @Qualifier("shoppingCrawlerDataSource") DataSource dataSource) { return builder.dataSource(dataSource) .packages(BaseScanDomain.class) .properties(getPropsMap(DATA_SOURCE_UNIT_NAME)) .build(); } - @Bean("ppomppuNotifierProcessorTransactionManager") + @Bean("shoppingCrawlerTransactionManager") public PlatformTransactionManager transactionManager( - @Qualifier("ppomppuNotifierProcessorEntityManagerFactory") EntityManagerFactory entityManagerFactory) { + @Qualifier("shoppingCrawlerEntityManagerFactory") EntityManagerFactory entityManagerFactory) { return new JpaTransactionManager(entityManagerFactory); } @@ -91,12 +95,9 @@ public class PpomppuDatasourceConfiguration { properties.put(AvailableSettings.FORMAT_SQL, hibernateProperties.getFormatSql()); properties.put(AvailableSettings.SHOW_SQL, hibernateProperties.getShowSql()); properties.put(AvailableSettings.HBM2DDL_AUTO, hibernateProperties.getHbm2ddlAuto()); - properties.put(AvailableSettings.CONNECTION_PROVIDER_DISABLES_AUTOCOMMIT, - hibernateProperties.getDisableAutoCommit()); - properties.put(AvailableSettings.IMPLICIT_NAMING_STRATEGY, - "org.springframework.boot.orm.jpa.hibernate.SpringImplicitNamingStrategy"); - properties.put(AvailableSettings.PHYSICAL_NAMING_STRATEGY, - "org.springframework.boot.orm.jpa.hibernate.SpringPhysicalNamingStrategy"); + properties.put(AvailableSettings.CONNECTION_PROVIDER_DISABLES_AUTOCOMMIT, hibernateProperties.getDisableAutoCommit()); + properties.put(AvailableSettings.IMPLICIT_NAMING_STRATEGY, ImplicitNamingStrategyJpaCompliantImpl.class.getName()); + properties.put(AvailableSettings.PHYSICAL_NAMING_STRATEGY, CamelCaseToUnderscoresNamingStrategy.class.getName()); properties.put(AvailableSettings.GENERATE_STATISTICS, "false"); properties.put(AvailableSettings.GLOBALLY_QUOTED_IDENTIFIERS, "true"); properties.put(AvailableSettings.GLOBALLY_QUOTED_IDENTIFIERS_SKIP_COLUMN_DEFINITIONS, "true"); @@ -104,7 +105,7 @@ public class PpomppuDatasourceConfiguration { properties.put(AvailableSettings.ORDER_INSERTS, "true"); properties.put(AvailableSettings.ORDER_UPDATES, "true"); properties.put(AvailableSettings.BATCH_VERSIONED_DATA, "true"); - properties.put(AvailableSettings.USE_NEW_ID_GENERATOR_MAPPINGS, "false"); +// properties.put(AvailableSettings.JPA_ID_GENERATOR_GLOBAL_SCOPE_COMPLIANCE, "false"); return properties; } diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/properties/DatasourceProperties.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/datasource/properties/DatasourceProperties.java similarity index 62% rename from processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/properties/DatasourceProperties.java rename to shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/datasource/properties/DatasourceProperties.java index 351eb13..3a1e583 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/properties/DatasourceProperties.java +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/datasource/properties/DatasourceProperties.java @@ -1,29 +1,26 @@ -package com.myoa.engineering.crawl.ppomppu.processor.configuration.properties; +package com.myoa.engineering.crawl.shopping.configuration.datasource.properties; -import com.myoa.engineering.crawl.ppomppu.support.util.ObjectUtil; -import java.util.List; - -import lombok.Getter; -import lombok.Setter; +import com.myoa.engineering.crawl.shopping.support.util.ObjectUtil; +import lombok.Data; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.stereotype.Component; +import java.util.List; + @Component -@Setter -@Getter -@ConfigurationProperties(prefix = "infra.database") +@Data +@ConfigurationProperties(prefix = "datasource") public class DatasourceProperties { private List units; - @Getter - @Setter + @Data public static class DataSourcePropertiesUnit { private String unitName; private String schemaName; private String connectionParameters; - private String datasourceUrl; + private String dbConnectionUrl; private Boolean isSimpleConnectionUrl; private String username; private String password; @@ -31,9 +28,9 @@ public class DatasourceProperties { public String toCompletedJdbcUrl() { if (ObjectUtil.isEmpty(isSimpleConnectionUrl) || isSimpleConnectionUrl == false) { - return String.format("%s/%s?%s", datasourceUrl, schemaName, connectionParameters); + return String.format("%s/%s?%s", dbConnectionUrl, schemaName, connectionParameters); } - return datasourceUrl; + return dbConnectionUrl; } } @@ -42,7 +39,7 @@ public class DatasourceProperties { .filter(e -> e.getUnitName().equals(unitName)) .findFirst() .orElseThrow( - () -> new IllegalArgumentException(this.getClass().getName() + ": unitName Not found. " + unitName)); + () -> new IllegalArgumentException(this.getClass().getName() + ": unitName Not found. " + unitName)); } } diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/properties/HibernateProperties.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/datasource/properties/HibernateProperties.java similarity index 85% rename from processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/properties/HibernateProperties.java rename to shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/datasource/properties/HibernateProperties.java index 4543d8a..cbdfa49 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/properties/HibernateProperties.java +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/datasource/properties/HibernateProperties.java @@ -1,14 +1,12 @@ -package com.myoa.engineering.crawl.ppomppu.processor.configuration.properties; +package com.myoa.engineering.crawl.shopping.configuration.datasource.properties; -import java.util.List; import lombok.Getter; import lombok.Setter; - -import org.hibernate.dialect.MySQL8Dialect; -import org.hibernate.dialect.MySQLDialect; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.stereotype.Component; +import java.util.List; + @Component @Setter @Getter diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/properties/HikariProperties.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/datasource/properties/HikariProperties.java similarity index 86% rename from processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/properties/HikariProperties.java rename to shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/datasource/properties/HikariProperties.java index 91a52bf..47516db 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/configuration/properties/HikariProperties.java +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/datasource/properties/HikariProperties.java @@ -1,4 +1,4 @@ -package com.myoa.engineering.crawl.ppomppu.processor.configuration.properties; +package com.myoa.engineering.crawl.shopping.configuration.datasource.properties; import lombok.Getter; import lombok.Setter; diff --git a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/configuration/properties/SlackSecretProperties.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/slack/properties/SlackSecretProperties.java similarity index 77% rename from sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/configuration/properties/SlackSecretProperties.java rename to shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/slack/properties/SlackSecretProperties.java index d47983c..691f141 100644 --- a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/configuration/properties/SlackSecretProperties.java +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/slack/properties/SlackSecretProperties.java @@ -1,4 +1,4 @@ -package com.myoa.engineering.crawl.ppomppu.sender.configuration.properties; +package com.myoa.engineering.crawl.shopping.configuration.slack.properties; import java.util.List; import lombok.Data; @@ -7,18 +7,16 @@ import lombok.Setter; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.stereotype.Component; -@Getter -@Setter +@Data @Component -@ConfigurationProperties("infra.slack.bot") +@ConfigurationProperties(prefix = "slack.bot") public class SlackSecretProperties { private List units; @Data public static class SlackSecretPropertiesUnit { - - private String botName; + private String botUnitName; private String username; private String iconEmoji; private String channel; @@ -27,7 +25,7 @@ public class SlackSecretProperties { public SlackSecretPropertiesUnit find(String botUnitName) { return units.stream() - .filter(e -> e.getBotName().equals(botUnitName)) + .filter(e -> e.getBotUnitName().equals(botUnitName)) .findFirst() .orElseThrow(() -> new IllegalArgumentException("not found bot unit name : " + botUnitName)); } diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/controller/TestAPIController.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/controller/TestAPIController.java new file mode 100644 index 0000000..7992a38 --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/controller/TestAPIController.java @@ -0,0 +1,22 @@ +package com.myoa.engineering.crawl.shopping.controller; + +import com.myoa.engineering.crawl.shopping.crawlhandler.PpomppuCrawlDomesticHandler; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +@RestController +@RequestMapping("/api/v1/exploit") +public class TestAPIController { + + private final PpomppuCrawlDomesticHandler ppomppuCrawlDomesticHandler; + + public TestAPIController(PpomppuCrawlDomesticHandler ppomppuCrawlDomesticHandler) { + this.ppomppuCrawlDomesticHandler = ppomppuCrawlDomesticHandler; + } + + @GetMapping("/triggers") + public void triggerExploit() { + ppomppuCrawlDomesticHandler.handle(); + } +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/crawlhandler/CrawlHandler.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/crawlhandler/CrawlHandler.java new file mode 100644 index 0000000..8b60ff0 --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/crawlhandler/CrawlHandler.java @@ -0,0 +1,10 @@ +package com.myoa.engineering.crawl.shopping.crawlhandler; + +import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget; + +public interface CrawlHandler { + + CrawlTarget getCrawlTarget(); + + void handle(); +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/crawlhandler/FmkoreaCrawlHandler.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/crawlhandler/FmkoreaCrawlHandler.java new file mode 100644 index 0000000..bfb6938 --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/crawlhandler/FmkoreaCrawlHandler.java @@ -0,0 +1,18 @@ +package com.myoa.engineering.crawl.shopping.crawlhandler; + +import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; + +@Slf4j +@Component +public class FmkoreaCrawlHandler implements CrawlHandler { + @Override + public CrawlTarget getCrawlTarget() { + return CrawlTarget.FMKOREA; + } + + @Override + public void handle() { + } +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/crawlhandler/PpomppuCrawlDomesticHandler.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/crawlhandler/PpomppuCrawlDomesticHandler.java new file mode 100644 index 0000000..541c969 --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/crawlhandler/PpomppuCrawlDomesticHandler.java @@ -0,0 +1,58 @@ +package com.myoa.engineering.crawl.shopping.crawlhandler; + +import com.myoa.engineering.crawl.shopping.crawlhandler.parser.PpomppuArticleParserV2; +import com.myoa.engineering.crawl.shopping.domain.entity.v2.Article; +import com.myoa.engineering.crawl.shopping.infra.client.ppomppu.PpomppuBoardClientV2; +import com.myoa.engineering.crawl.shopping.service.ArticleCommandService; +import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget; +import org.springframework.stereotype.Component; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Stream; + +@Component +public class PpomppuCrawlDomesticHandler implements CrawlHandler { + + private final PpomppuBoardClientV2 ppomppuBoardClient; + private final PpomppuArticleParserV2 ppomppuArticleParserV2; + private final ArticleCommandService articleCommandService; + + public PpomppuCrawlDomesticHandler(PpomppuBoardClientV2 ppomppuBoardClient, + PpomppuArticleParserV2 ppomppuArticleParserV2, + ArticleCommandService articleCommandService) { + this.ppomppuBoardClient = ppomppuBoardClient; + this.ppomppuArticleParserV2 = ppomppuArticleParserV2; + this.articleCommandService = articleCommandService; + } + + @Override + public CrawlTarget getCrawlTarget() { + return CrawlTarget.PPOMPPU_DOMESTIC; + } + + @Override + public void handle() { + + String boardHtmlPage1 = ppomppuBoardClient.getBoardHtml("/zboard/zboard.php", generateRequestParams(1)); + List
parsedPage1 = ppomppuArticleParserV2.parse(boardHtmlPage1); + + String boardHtmlPage2 = ppomppuBoardClient.getBoardHtml("/zboard/zboard.php", generateRequestParams(2)); + List
parsedPage2 = ppomppuArticleParserV2.parse(boardHtmlPage2); + + List
merged = Stream.of(parsedPage1, parsedPage2) + .flatMap(List::stream) + .map(e -> e.updateCrawlTarget(getCrawlTarget())) + .toList(); + + articleCommandService.upsert(merged); + } + + private Map generateRequestParams(int pageId) { + Map params = new HashMap<>(); + params.put("id", "ppomppu"); + params.put("page", String.valueOf(pageId)); + return params; + } +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/crawlhandler/parser/PpomppuArticleParserV2.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/crawlhandler/parser/PpomppuArticleParserV2.java new file mode 100644 index 0000000..850f8a1 --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/crawlhandler/parser/PpomppuArticleParserV2.java @@ -0,0 +1,115 @@ +package com.myoa.engineering.crawl.shopping.crawlhandler.parser; + +import com.myoa.engineering.crawl.shopping.domain.entity.v2.Article; +import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName; +import com.myoa.engineering.crawl.shopping.util.DateTimeUtils; +import com.myoa.engineering.crawl.shopping.util.NumberUtils; +import io.micrometer.core.instrument.util.StringUtils; +import lombok.extern.slf4j.Slf4j; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; +import org.springframework.stereotype.Component; + +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +@Slf4j +@Component +public final class PpomppuArticleParserV2 { + + private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yy.MM.dd HH:mm:ss") + .withZone(ZoneId.of("Asia/Seoul")); + + private PpomppuArticleParserV2() { + } + + public List
parse(String html) { + Elements trElements = converHtmlToTrElements(html); + return trElements.stream() + .filter(this::isRealArticle) + .map(this::parse) + .toList(); + + } + + private Elements converHtmlToTrElements(String data) { + Document document = Jsoup.parse(data); + Elements trList = document.getElementById("revolution_main_table").getElementsByTag("tr"); + return trList; + } + + private boolean isRealArticle(Element tr) { + Elements tdList = tr.getElementsByTag("td"); + if (tdList.size() != 6) { + return false; + } + + if (!hasOnlyNumeric(tdList.get(0))) { + return false; + } + + return true; + } + + private Pattern pattern_numeric = Pattern.compile("\\d+"); + + private boolean hasOnlyNumeric(Element td) { + return pattern_numeric.matcher(td.text()).matches(); + } + + private Article parse(Element tr) { + Elements tdList = tr.getElementsByTag("td"); + Long articleId = Long.parseLong(tdList.get(0).text()); + + String title = tdList.get(2).text(); + String articleUrl = parseArticleUrl(tdList.get(2).getElementsByTag("a").attr("href")); + String boardName = parseBoardName(title); + Integer recommended = parseRecommended(tdList.get(4)); + Integer hit = NumberUtils.parseInt(tdList.get(5).text(), 0); + ZonedDateTime registeredAt = DateTimeUtils.parse(tdList.get(3).text()); + + return Article.builder() + .articleId(articleId) + .title(title) + .boardName(boardName) + .articleUrl(articleUrl) + .recommended(recommended) + .hit(hit) + .registeredAt(registeredAt) + .build(); + } + + public Integer parseRecommended(Element td) { + final String voteString = td.text(); + if (StringUtils.isEmpty(voteString)) { + return null; + } + + final int voteUp = Integer.parseInt(td.text().split(" - ")[0]); + final int voteDown = Integer.parseInt(td.text().split(" - ")[1]); + int recommended = voteUp - voteDown; + return recommended; + } + + public static String parseArticleUrl(String data) { + return PpomppuBoardName.ofViewPageUrl(data); + } + + Pattern patternBoardName = Pattern.compile("\\[(.+?)\\]"); + + public String parseBoardName(String fullTitle) { + Matcher matcher = patternBoardName.matcher(fullTitle); + String lastMatched = null; + while (matcher.find()) { + lastMatched = matcher.group(1); + } + return lastMatched; + } + +} diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/Auditable.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/Auditable.java similarity index 62% rename from processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/Auditable.java rename to shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/Auditable.java index e16f83b..ce5c073 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/Auditable.java +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/Auditable.java @@ -1,17 +1,18 @@ -package com.myoa.engineering.crawl.ppomppu.processor.domain; +package com.myoa.engineering.crawl.shopping.domain.entity; -import java.io.Serializable; -import java.time.Instant; -import javax.persistence.Column; +import lombok.Getter; import org.springframework.data.annotation.CreatedDate; import org.springframework.data.annotation.LastModifiedDate; -/** - * Auditable - * - * @author Shin Woo-jin (woozu.shin@kakaoent.com) - * @since 2021-09-08 - */ +import jakarta.persistence.*; +import org.springframework.data.jpa.domain.support.AuditingEntityListener; + +import java.io.Serializable; +import java.time.Instant; + +@Getter +@MappedSuperclass +@EntityListeners(AuditingEntityListener.class) public abstract class Auditable implements Serializable { private static final long serialVersionUID = -7105030870015828551L; diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/BaseScanDomain.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/BaseScanDomain.java similarity index 69% rename from processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/BaseScanDomain.java rename to shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/BaseScanDomain.java index 62bcf9f..bddd3dd 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/BaseScanDomain.java +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/BaseScanDomain.java @@ -1,4 +1,4 @@ -package com.myoa.engineering.crawl.ppomppu.processor.domain; +package com.myoa.engineering.crawl.shopping.domain.entity; /** * BaseScanDomain diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/PpomppuArticle.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v1/PpomppuArticle.java similarity index 61% rename from processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/PpomppuArticle.java rename to shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v1/PpomppuArticle.java index 9d88870..49fac26 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/PpomppuArticle.java +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v1/PpomppuArticle.java @@ -1,19 +1,16 @@ -package com.myoa.engineering.crawl.ppomppu.processor.domain; +package com.myoa.engineering.crawl.shopping.domain.entity.v1; -import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName; -import java.time.Instant; -import javax.persistence.Column; -import javax.persistence.Entity; -import javax.persistence.EnumType; -import javax.persistence.Enumerated; -import javax.persistence.GeneratedValue; -import javax.persistence.GenerationType; -import javax.persistence.Id; -import javax.persistence.Table; +import com.myoa.engineering.crawl.shopping.domain.entity.Auditable; +import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName; +import jakarta.persistence.*; import lombok.Builder; import lombok.Getter; import lombok.NoArgsConstructor; +import lombok.ToString; +import java.time.ZonedDateTime; + +@ToString @Getter @NoArgsConstructor @Entity @@ -44,11 +41,11 @@ public class PpomppuArticle extends Auditable { private Integer recommended; @Column - private Instant registeredAt; + private ZonedDateTime registeredAt; @Builder public PpomppuArticle(Long id, Long articleId, PpomppuBoardName boardName, String articleUrl, - String title, Integer recommended, Integer hit, Instant registeredAt) { + String title, Integer recommended, Integer hit, ZonedDateTime registeredAt) { this.id = id; this.articleId = articleId; this.boardName = boardName; @@ -59,8 +56,4 @@ public class PpomppuArticle extends Auditable { this.registeredAt = registeredAt; } - public PpomppuArticle updateBoardName(PpomppuBoardName boardName) { - this.boardName = boardName; - return this; - } } diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/PpomppuBoardFeedStatus.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v1/PpomppuBoardFeedStatus.java similarity index 76% rename from processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/PpomppuBoardFeedStatus.java rename to shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v1/PpomppuBoardFeedStatus.java index b751c13..1ff7d5e 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/PpomppuBoardFeedStatus.java +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v1/PpomppuBoardFeedStatus.java @@ -1,19 +1,14 @@ -package com.myoa.engineering.crawl.ppomppu.processor.domain; +package com.myoa.engineering.crawl.shopping.domain.entity.v1; -import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName; -import java.time.Instant; -import javax.persistence.Column; -import javax.persistence.Entity; -import javax.persistence.EnumType; -import javax.persistence.Enumerated; -import javax.persistence.GeneratedValue; -import javax.persistence.GenerationType; -import javax.persistence.Id; -import javax.persistence.Table; +import com.myoa.engineering.crawl.shopping.domain.entity.Auditable; +import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName; +import jakarta.persistence.*; import lombok.Builder; import lombok.Getter; import lombok.NoArgsConstructor; +import java.time.Instant; + @Getter @NoArgsConstructor @Entity diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/PublishedHistory.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v1/PublishedHistory.java similarity index 61% rename from processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/PublishedHistory.java rename to shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v1/PublishedHistory.java index e52c069..46d39dd 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/PublishedHistory.java +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v1/PublishedHistory.java @@ -1,15 +1,12 @@ -package com.myoa.engineering.crawl.ppomppu.processor.domain; +package com.myoa.engineering.crawl.shopping.domain.entity.v1; -import java.time.Instant; -import javax.persistence.Column; -import javax.persistence.Entity; -import javax.persistence.GeneratedValue; -import javax.persistence.GenerationType; -import javax.persistence.Id; -import javax.persistence.Table; +import com.myoa.engineering.crawl.shopping.domain.entity.Auditable; import lombok.Getter; import lombok.NoArgsConstructor; +import jakarta.persistence.*; +import java.time.Instant; + @Getter @NoArgsConstructor @Entity diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v1/SubscribedBoard.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v1/SubscribedBoard.java new file mode 100644 index 0000000..ebe0e10 --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v1/SubscribedBoard.java @@ -0,0 +1,27 @@ +package com.myoa.engineering.crawl.shopping.domain.entity.v1; + +import com.myoa.engineering.crawl.shopping.domain.entity.Auditable; +import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName; +import lombok.Getter; +import lombok.NoArgsConstructor; + +import jakarta.persistence.*; + +@Getter +@NoArgsConstructor +@Entity +@Table(name = "subscribed_board") +public class SubscribedBoard extends Auditable { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + @Column + private Long userId; + + @Column + @Enumerated(EnumType.STRING) + private PpomppuBoardName boardName; + +} diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/SubscribedUser.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v1/SubscribedUser.java similarity index 50% rename from processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/SubscribedUser.java rename to shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v1/SubscribedUser.java index fd89bde..a8dc32e 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/domain/SubscribedUser.java +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v1/SubscribedUser.java @@ -1,20 +1,17 @@ -package com.myoa.engineering.crawl.ppomppu.processor.domain; +package com.myoa.engineering.crawl.shopping.domain.entity.v1; -import java.time.Instant; -import javax.persistence.Column; -import javax.persistence.Entity; -import javax.persistence.GeneratedValue; -import javax.persistence.GenerationType; -import javax.persistence.Id; -import javax.persistence.Table; +import com.myoa.engineering.crawl.shopping.domain.entity.Auditable; import lombok.Getter; import lombok.NoArgsConstructor; +import jakarta.persistence.*; +import java.time.Instant; + @Getter @NoArgsConstructor @Entity @Table(name = "subscribed_user") -public class SubscribedUser extends Auditable{ +public class SubscribedUser extends Auditable { @Id @GeneratedValue(strategy = GenerationType.IDENTITY) diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v2/AppUser.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v2/AppUser.java new file mode 100644 index 0000000..47e31ca --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v2/AppUser.java @@ -0,0 +1,26 @@ +package com.myoa.engineering.crawl.shopping.domain.entity.v2; + +import com.myoa.engineering.crawl.shopping.domain.entity.Auditable; +import jakarta.persistence.*; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; +import lombok.NoArgsConstructor; + +@Getter +@Builder +@NoArgsConstructor +@AllArgsConstructor +@Entity +@Table +public class AppUser extends Auditable { + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + @Column + private String name; + + @Column + private Boolean enabled; +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v2/Article.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v2/Article.java new file mode 100644 index 0000000..a6e7a3e --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v2/Article.java @@ -0,0 +1,63 @@ +package com.myoa.engineering.crawl.shopping.domain.entity.v2; + +import com.myoa.engineering.crawl.shopping.domain.entity.Auditable; +import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget; +import jakarta.persistence.*; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; +import lombok.NoArgsConstructor; + +import java.time.ZonedDateTime; + +@Getter +@Builder +@NoArgsConstructor +@AllArgsConstructor +@Entity +@Table +public class Article extends Auditable { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + @Column + private Long articleId; + + @Column + @Enumerated(EnumType.STRING) + private CrawlTarget crawlTarget; + + @Column + private String boardName; + + @Column + private String articleUrl; + + @Column + private String title; + + @Column + private Integer hit; + + @Column + private Integer recommended; + + @Column + private ZonedDateTime registeredAt; + + public Article update(Article article) { + this.boardName = article.getBoardName(); + this.articleUrl = article.getArticleUrl(); + this.title = article.getTitle(); + this.hit = article.getHit(); + this.recommended = article.getRecommended(); + return this; + } + + public Article updateCrawlTarget(CrawlTarget crawlTarget) { + this.crawlTarget = crawlTarget; + return this; + } +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v2/SubscribedKeyword.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v2/SubscribedKeyword.java new file mode 100644 index 0000000..e859e9f --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v2/SubscribedKeyword.java @@ -0,0 +1,30 @@ +package com.myoa.engineering.crawl.shopping.domain.entity.v2; + +import com.myoa.engineering.crawl.shopping.domain.entity.Auditable; +import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget; +import jakarta.persistence.*; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; +import lombok.NoArgsConstructor; + +@Getter +@Builder +@NoArgsConstructor +@AllArgsConstructor +@Entity +@Table +public class SubscribedKeyword extends Auditable { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + @Column + private String keyword; + + @Column + @Enumerated(EnumType.STRING) + private CrawlTarget crawlTarget; + +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/model/v2/ArticleModel.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/model/v2/ArticleModel.java new file mode 100644 index 0000000..b1435ff --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/model/v2/ArticleModel.java @@ -0,0 +1,27 @@ +package com.myoa.engineering.crawl.shopping.domain.model.v2; + +import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; +import lombok.NoArgsConstructor; + +import java.time.ZonedDateTime; + +@Getter +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class ArticleModel { + + private Long id; + private Long articleId; + private CrawlTarget crawlTarget; + private String boardName; + private String articleUrl; + private String title; + private Integer hit; + private Integer recommended; + private ZonedDateTime registeredAt; + +} diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/FeedParsedResult.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/FeedParsedResult.java similarity index 89% rename from processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/FeedParsedResult.java rename to shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/FeedParsedResult.java index 2e2eca6..f169a3e 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/FeedParsedResult.java +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/FeedParsedResult.java @@ -1,12 +1,13 @@ -package com.myoa.engineering.crawl.ppomppu.processor.dto; +package com.myoa.engineering.crawl.shopping.dto; -import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName; -import java.io.Serializable; -import java.time.Instant; +import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName; import lombok.Builder; import lombok.Getter; import lombok.NoArgsConstructor; +import java.io.Serializable; +import java.time.Instant; + /** * FeedParsedResult * diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/PpomppuArticleDTO.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/PpomppuArticleDTO.java similarity index 70% rename from processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/PpomppuArticleDTO.java rename to shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/PpomppuArticleDTO.java index af2e1a8..ce8d484 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/PpomppuArticleDTO.java +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/PpomppuArticleDTO.java @@ -1,4 +1,4 @@ -package com.myoa.engineering.crawl.ppomppu.processor.dto; +package com.myoa.engineering.crawl.shopping.dto; /** * PpomppuArticle diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/PpomppuArticleTransformer.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/PpomppuArticleTransformer.java similarity index 72% rename from processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/PpomppuArticleTransformer.java rename to shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/PpomppuArticleTransformer.java index ede7623..e0531ad 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/PpomppuArticleTransformer.java +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/PpomppuArticleTransformer.java @@ -1,38 +1,38 @@ -package com.myoa.engineering.crawl.ppomppu.processor.dto; +package com.myoa.engineering.crawl.shopping.dto; + +import com.myoa.engineering.crawl.shopping.domain.entity.v1.PpomppuArticle; +import com.myoa.engineering.crawl.shopping.support.dto.SimpleMessageDTO; import java.time.Instant; import java.time.ZoneId; import java.time.format.DateTimeFormatter; import java.util.List; -import java.util.function.Function; import java.util.stream.Collectors; -import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle; -import com.myoa.engineering.crawl.ppomppu.support.dto.SimpleMessageDTO; - /** * PpomppuArticleTransformer + * * @author Shin Woo-jin (woo-jin.shin@linecorp.com) * @since 2021-11-21 - * */ public final class PpomppuArticleTransformer { - private PpomppuArticleTransformer() {} + private PpomppuArticleTransformer() { + } private static final String MESSAGE_FORMAT_V1 = "%s)) `%s` <%s:LINK>"; private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss") .withZone(ZoneId.of("Asia/Seoul")); - +/* public static final Function TRANSFORM_TO_MESSAGE_DTO = entity -> - SimpleMessageDTO.builder() - .requestedAt(Instant.now()) - .publishedAt(entity.getRegisteredAt()) - .title(String.format(MESSAGE_FORMAT_V1, entity.getBoardName().getMenuName(), entity.getTitle())) - .body(entity.getArticleUrl()) - .build(); - + SimpleMessageDTO.builder() + .requestedAt(Instant.now()) + .publishedAt(entity.getRegisteredAt()) + .title(String.format(MESSAGE_FORMAT_V1, entity.getBoardName().getMenuName(), entity.getTitle())) + .body(entity.getArticleUrl()) + .build(); +*/ // https://stackoverflow.com/questions/24882927/using-streams-to-convert-a-list-of-objects-into-a-string-obtained-from-the-tostr public static SimpleMessageDTO transform(List articles) { Instant requestedAt = Instant.now(); diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/constant/WebClientPropertiesUnitName.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/constant/WebClientPropertiesUnitName.java similarity index 83% rename from processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/constant/WebClientPropertiesUnitName.java rename to shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/constant/WebClientPropertiesUnitName.java index 3383138..03615c7 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/dto/constant/WebClientPropertiesUnitName.java +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/constant/WebClientPropertiesUnitName.java @@ -1,4 +1,4 @@ -package com.myoa.engineering.crawl.ppomppu.processor.dto.constant; +package com.myoa.engineering.crawl.shopping.dto.constant; import lombok.AllArgsConstructor; import lombok.Getter; diff --git a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/dto/MessageDTO.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/slack/MessageDTO.java similarity index 76% rename from sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/dto/MessageDTO.java rename to shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/slack/MessageDTO.java index db45e43..018a1fd 100644 --- a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/dto/MessageDTO.java +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/slack/MessageDTO.java @@ -1,4 +1,4 @@ -package com.myoa.engineering.crawl.ppomppu.sender.dto; +package com.myoa.engineering.crawl.shopping.dto.slack; import java.io.Serializable; diff --git a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/dto/SlackMessageDTO.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/slack/SlackMessageDTO.java similarity index 93% rename from sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/dto/SlackMessageDTO.java rename to shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/slack/SlackMessageDTO.java index faf7ef5..6a63b14 100644 --- a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/dto/SlackMessageDTO.java +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/slack/SlackMessageDTO.java @@ -1,4 +1,4 @@ -package com.myoa.engineering.crawl.ppomppu.sender.dto; +package com.myoa.engineering.crawl.shopping.dto.slack; import com.fasterxml.jackson.annotation.JsonProperty; diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/event/ArticleUpsertEvent.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/event/ArticleUpsertEvent.java new file mode 100644 index 0000000..e58b59d --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/event/ArticleUpsertEvent.java @@ -0,0 +1,12 @@ +package com.myoa.engineering.crawl.shopping.event; + +import org.springframework.context.ApplicationEvent; + +public class ArticleUpsertEvent extends ApplicationEvent { + + public ArticleUpsertEvent(Object source) { + super(source); + } +} + + diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/event/ArticleUpsertEventCommand.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/event/ArticleUpsertEventCommand.java new file mode 100644 index 0000000..fb2d709 --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/event/ArticleUpsertEventCommand.java @@ -0,0 +1,19 @@ +package com.myoa.engineering.crawl.shopping.event; + +import com.myoa.engineering.crawl.shopping.domain.model.v2.ArticleModel; +import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget; +import lombok.Getter; + +import java.util.List; + +@Getter +@Deprecated +public class ArticleUpsertEventCommand { + private final List articles; + private final CrawlTarget crawlTarget; + + public ArticleUpsertEventCommand(List articles, CrawlTarget crawlTarget) { + this.articles = articles; + this.crawlTarget = crawlTarget; + } +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/event/handler/ArticleUpsertEventListener.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/event/handler/ArticleUpsertEventListener.java new file mode 100644 index 0000000..846a641 --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/event/handler/ArticleUpsertEventListener.java @@ -0,0 +1,23 @@ +package com.myoa.engineering.crawl.shopping.event.handler; + +import com.myoa.engineering.crawl.shopping.event.ArticleUpsertEvent; +import com.myoa.engineering.crawl.shopping.service.SubscribedKeywordQueryService; +import org.ahocorasick.trie.Trie; +import org.springframework.context.event.EventListener; +import org.springframework.stereotype.Component; + +@Component +public class ArticleUpsertEventListener { + + private final SubscribedKeywordQueryService subscribedKeywordQueryService; + + public ArticleUpsertEventListener(SubscribedKeywordQueryService subscribedKeywordQueryService) { + this.subscribedKeywordQueryService = subscribedKeywordQueryService; + } + + @EventListener + public void handleArticleUpsertEvent(ArticleUpsertEvent event) { + + System.out.println("event = " + event); + } +} diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/client/MessageSenderAPIClient.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/MessageSenderAPIClient.java similarity index 63% rename from processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/client/MessageSenderAPIClient.java rename to shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/MessageSenderAPIClient.java index 155214f..38ff6d1 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/client/MessageSenderAPIClient.java +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/MessageSenderAPIClient.java @@ -1,22 +1,7 @@ -package com.myoa.engineering.crawl.ppomppu.processor.infrastructure.client; - -import org.springframework.core.ParameterizedTypeReference; -import org.springframework.http.HttpHeaders; -import org.springframework.http.MediaType; -import org.springframework.stereotype.Component; -import org.springframework.web.reactive.function.client.WebClient; -import org.springframework.web.reactive.function.client.WebClientRequestException; - -import com.myoa.engineering.crawl.ppomppu.processor.dto.constant.WebClientPropertiesUnitName; -import com.myoa.engineering.crawl.ppomppu.support.dto.SimpleMessageDTO; -import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebClientFilterFactory; -import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebFluxExchangeStragiesFactory; -import com.myoa.engineering.crawl.ppomppu.support.webclient.properties.WebClientProperties; -import com.myoa.engineering.crawl.ppomppu.support.webclient.properties.WebClientProperties.WebClientPropertiesUnit; +package com.myoa.engineering.crawl.shopping.infra.client; import lombok.extern.slf4j.Slf4j; -import reactor.core.publisher.Mono; -import reactor.core.scheduler.Schedulers; +import org.springframework.stereotype.Component; /** * PpomppuNotifierSenderAPIClient @@ -27,7 +12,7 @@ import reactor.core.scheduler.Schedulers; @Slf4j @Component public class MessageSenderAPIClient { - +/* private final WebClient webClient; public MessageSenderAPIClient(WebClientProperties webClientProperties) { @@ -53,4 +38,6 @@ public class MessageSenderAPIClient { return Mono.empty(); }); } + + */ } \ No newline at end of file diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/fmkorea/FmkoreaBoardClient.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/fmkorea/FmkoreaBoardClient.java new file mode 100644 index 0000000..1d0f0e0 --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/fmkorea/FmkoreaBoardClient.java @@ -0,0 +1,12 @@ +package com.myoa.engineering.crawl.shopping.infra.client.fmkorea; + +import org.springframework.cloud.openfeign.FeignClient; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; + +@FeignClient(value = "fmkorea-board-client", url = "https://fmkorea.com") +public interface FmkoreaBoardClient { + + @GetMapping("{boardLink}") + String getBoardHtml(@PathVariable("boardLink") String boardLink); +} diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/client/PpomppuBoardFeedRetriever.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/ppomppu/PpomppuBoardClient.java similarity index 62% rename from processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/client/PpomppuBoardFeedRetriever.java rename to shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/ppomppu/PpomppuBoardClient.java index 9db4dab..1ab1b8e 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/client/PpomppuBoardFeedRetriever.java +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/ppomppu/PpomppuBoardClient.java @@ -1,16 +1,7 @@ -package com.myoa.engineering.crawl.ppomppu.processor.infrastructure.client; +package com.myoa.engineering.crawl.shopping.infra.client.ppomppu; -import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebClientFilterFactory; -import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebFluxExchangeStragiesFactory; -import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName; import lombok.extern.slf4j.Slf4j; - -import org.springframework.core.ParameterizedTypeReference; import org.springframework.stereotype.Component; -import org.springframework.web.reactive.function.client.WebClient; -import org.springframework.web.reactive.function.client.WebClientRequestException; -import reactor.core.publisher.Mono; -import reactor.core.scheduler.Schedulers; /** * PpomppuBoardFeedRetriever @@ -20,11 +11,11 @@ import reactor.core.scheduler.Schedulers; */ @Slf4j @Component -public class PpomppuBoardFeedRetriever { - +public class PpomppuBoardClient { +/* private final WebClient webClient; - public PpomppuBoardFeedRetriever(WebClient.Builder webClientBuilder) { + public PpomppuBoardClient(WebClient.Builder webClientBuilder) { this.webClient = webClientBuilder.baseUrl(PpomppuBoardName.PPOMPPU_URL) .exchangeStrategies(WebFluxExchangeStragiesFactory.ofTextHtml()) .filter(WebClientFilterFactory.logRequest()) @@ -43,5 +34,5 @@ public class PpomppuBoardFeedRetriever { }); // .doOnNext(e -> log.info("[getHtml] {}", e)); } - +*/ } diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/ppomppu/PpomppuBoardClientV2.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/ppomppu/PpomppuBoardClientV2.java new file mode 100644 index 0000000..287dbae --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/ppomppu/PpomppuBoardClientV2.java @@ -0,0 +1,17 @@ +package com.myoa.engineering.crawl.shopping.infra.client.ppomppu; + +import org.springframework.cloud.openfeign.FeignClient; +import org.springframework.cloud.openfeign.SpringQueryMap; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestParam; + +import java.util.Map; + +@FeignClient(value = "ppomppu-board-client", url = "https://www.ppomppu.co.kr") +public interface PpomppuBoardClientV2 { + + @GetMapping("{boardLink}") + String getBoardHtml(@PathVariable("boardLink") String boardLink, + @SpringQueryMap Map params); +} diff --git a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/infrastructure/client/MongeShoppingBotSlackMessageSender.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/slack/MongeShoppingBotSlackMessageSender.java similarity index 58% rename from sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/infrastructure/client/MongeShoppingBotSlackMessageSender.java rename to shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/slack/MongeShoppingBotSlackMessageSender.java index 6f5b352..68501b1 100644 --- a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/infrastructure/client/MongeShoppingBotSlackMessageSender.java +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/slack/MongeShoppingBotSlackMessageSender.java @@ -1,22 +1,25 @@ -package com.myoa.engineering.crawl.ppomppu.sender.infrastructure.client; +package com.myoa.engineering.crawl.shopping.infra.client.slack; -import com.myoa.engineering.crawl.ppomppu.sender.configuration.properties.SlackSecretProperties; -import com.myoa.engineering.crawl.ppomppu.sender.configuration.properties.SlackSecretProperties.SlackSecretPropertiesUnit; -import com.myoa.engineering.crawl.ppomppu.sender.dto.SlackMessageDTO; +import com.myoa.engineering.crawl.shopping.configuration.slack.properties.SlackSecretProperties; +import com.myoa.engineering.crawl.shopping.dto.slack.SlackMessageDTO; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; @Slf4j @Component -public class MongeShoppingBotSlackMessageSender extends SlackMessageSender { +public class MongeShoppingBotSlackMessageSender { - private static final String SLACK_SECRET_UNIT_NAME = "monge_shopping_bot"; + private static final String SLACK_SECRET_UNIT_NAME = "shopping-crawler"; - private final SlackSecretPropertiesUnit slackProperties; + private final SlackSecretProperties.SlackSecretPropertiesUnit slackProperties; + private final SlackAPIClient slackAPIClient; + private final String token; - public MongeShoppingBotSlackMessageSender(SlackSecretProperties slackSecretProperties) { - super(slackSecretProperties.find(SLACK_SECRET_UNIT_NAME).getToken()); + public MongeShoppingBotSlackMessageSender(SlackAPIClient slackAPIClient, + SlackSecretProperties slackSecretProperties) { + this.slackAPIClient = slackAPIClient; this.slackProperties = slackSecretProperties.find(SLACK_SECRET_UNIT_NAME); + this.token = slackProperties.getToken(); } public SlackMessageDTO ofMessageTemplate() { diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/slack/SlackAPIClient.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/slack/SlackAPIClient.java new file mode 100644 index 0000000..0144288 --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/slack/SlackAPIClient.java @@ -0,0 +1,11 @@ +package com.myoa.engineering.crawl.shopping.infra.client.slack; + +import org.springframework.cloud.openfeign.FeignClient; +import org.springframework.web.bind.annotation.PostMapping; + +@FeignClient(value = "slack-api-client", url = "https://slack.com") +public interface SlackAPIClient { + + @PostMapping("/api/v1/messages/sendMessage/messengers/slack") + String sendMessage(); +} diff --git a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/infrastructure/client/SlackMessageSender.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/slack/SlackMessageSender.java similarity index 68% rename from sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/infrastructure/client/SlackMessageSender.java rename to shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/slack/SlackMessageSender.java index c0f2efb..c95d70c 100644 --- a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/infrastructure/client/SlackMessageSender.java +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/slack/SlackMessageSender.java @@ -1,17 +1,6 @@ -package com.myoa.engineering.crawl.ppomppu.sender.infrastructure.client; - -import org.springframework.http.HttpHeaders; -import org.springframework.http.MediaType; -import org.springframework.web.reactive.function.client.WebClient; -import org.springframework.web.reactive.function.client.WebClientRequestException; - -import com.myoa.engineering.crawl.ppomppu.sender.dto.SlackMessageDTO; -import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebClientFilterFactory; -import com.myoa.engineering.crawl.ppomppu.support.webclient.factory.WebFluxExchangeStragiesFactory; +package com.myoa.engineering.crawl.shopping.infra.client.slack; import lombok.extern.slf4j.Slf4j; -import reactor.core.publisher.Mono; -import reactor.core.scheduler.Schedulers; /** * SlackMessageSender @@ -20,8 +9,9 @@ import reactor.core.scheduler.Schedulers; * @since 2021-09-08 */ @Slf4j -public class SlackMessageSender implements MessageSender { +public class SlackMessageSender { /* implements MessageSender {*/ +/* private static final String SLACK_API_URL = "https://slack.com/api"; private final WebClient webClient; @@ -50,4 +40,6 @@ public class SlackMessageSender implements MessageSender { }); } + */ + } diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/repository/BaseScanRepository.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/repository/BaseScanRepository.java new file mode 100644 index 0000000..04447d8 --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/repository/BaseScanRepository.java @@ -0,0 +1,4 @@ +package com.myoa.engineering.crawl.shopping.infra.repository; + +public interface BaseScanRepository { +} diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/repository/PpomppuArticleRepository.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/repository/v1/PpomppuArticleRepository.java similarity index 58% rename from processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/repository/PpomppuArticleRepository.java rename to shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/repository/v1/PpomppuArticleRepository.java index 0b727d9..885d1ce 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/repository/PpomppuArticleRepository.java +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/repository/v1/PpomppuArticleRepository.java @@ -1,6 +1,6 @@ -package com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository; +package com.myoa.engineering.crawl.shopping.infra.repository.v1; -import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle; +import com.myoa.engineering.crawl.shopping.domain.entity.v1.PpomppuArticle; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Repository; diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/repository/PpomppuBoardFeedStatusRepository.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/repository/v1/PpomppuBoardFeedStatusRepository.java similarity index 58% rename from processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/repository/PpomppuBoardFeedStatusRepository.java rename to shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/repository/v1/PpomppuBoardFeedStatusRepository.java index e84a077..0e87f92 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/infrastructure/repository/PpomppuBoardFeedStatusRepository.java +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/repository/v1/PpomppuBoardFeedStatusRepository.java @@ -1,11 +1,12 @@ -package com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository; +package com.myoa.engineering.crawl.shopping.infra.repository.v1; -import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuBoardFeedStatus; -import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName; -import java.util.Optional; +import com.myoa.engineering.crawl.shopping.domain.entity.v1.PpomppuBoardFeedStatus; +import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Repository; +import java.util.Optional; + @Repository public interface PpomppuBoardFeedStatusRepository extends JpaRepository { diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/repository/v2/AppUserRepository.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/repository/v2/AppUserRepository.java new file mode 100644 index 0000000..6579b4a --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/repository/v2/AppUserRepository.java @@ -0,0 +1,9 @@ +package com.myoa.engineering.crawl.shopping.infra.repository.v2; + +import com.myoa.engineering.crawl.shopping.domain.entity.v2.AppUser; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.stereotype.Repository; + +@Repository +public interface AppUserRepository extends JpaRepository { +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/repository/v2/ArticleRepository.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/repository/v2/ArticleRepository.java new file mode 100644 index 0000000..1d80b33 --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/repository/v2/ArticleRepository.java @@ -0,0 +1,16 @@ +package com.myoa.engineering.crawl.shopping.infra.repository.v2; + +import com.myoa.engineering.crawl.shopping.domain.entity.v2.Article; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.stereotype.Repository; + +import java.util.Collection; +import java.util.List; +import java.util.Optional; + +@Repository +public interface ArticleRepository extends JpaRepository { + + Optional
findByArticleId(Long articleId); + List
findByArticleIdIn(Collection articleId); +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/repository/v2/SubscribedKeywordRepository.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/repository/v2/SubscribedKeywordRepository.java new file mode 100644 index 0000000..6870073 --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/repository/v2/SubscribedKeywordRepository.java @@ -0,0 +1,14 @@ +package com.myoa.engineering.crawl.shopping.infra.repository.v2; + +import com.myoa.engineering.crawl.shopping.domain.entity.v2.SubscribedKeyword; +import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.stereotype.Repository; + +import java.util.List; + +@Repository +public interface SubscribedKeywordRepository extends JpaRepository { + + List findByCrawlTarget(CrawlTarget crawlTarget); +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/scheduler/ParseEventEmitter.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/scheduler/ParseEventEmitter.java new file mode 100644 index 0000000..ae65851 --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/scheduler/ParseEventEmitter.java @@ -0,0 +1,27 @@ +package com.myoa.engineering.crawl.shopping.scheduler; + +import com.myoa.engineering.crawl.shopping.crawlhandler.CrawlHandler; +import lombok.extern.slf4j.Slf4j; +import org.springframework.scheduling.annotation.EnableScheduling; +import org.springframework.stereotype.Component; + +import java.util.List; + +@Slf4j +@Component +@EnableScheduling +public class ParseEventEmitter { + private final List crawlHandlers; + + + public ParseEventEmitter(List crawlHandlers) { + this.crawlHandlers = crawlHandlers; + } + + // @Scheduled(cron = "0 0/5 * * * ?") + public void emit() { + log.info("[emitDomesticBoard] trigger fired!"); + crawlHandlers.forEach(CrawlHandler::handle); + + } +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/ArticleCommandService.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/ArticleCommandService.java new file mode 100644 index 0000000..fe8f22d --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/ArticleCommandService.java @@ -0,0 +1,81 @@ +package com.myoa.engineering.crawl.shopping.service; + +import com.myoa.engineering.crawl.shopping.domain.entity.v2.Article; +import com.myoa.engineering.crawl.shopping.domain.model.v2.ArticleModel; +import com.myoa.engineering.crawl.shopping.event.ArticleUpsertEvent; +import com.myoa.engineering.crawl.shopping.infra.repository.v2.ArticleRepository; +import jakarta.transaction.Transactional; +import org.springframework.context.ApplicationEventPublisher; +import org.springframework.stereotype.Service; + +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; + +@Service +public class ArticleCommandService { + + private final ArticleRepository articleRepository; + private final ApplicationEventPublisher applicationEventPublisher; + + public ArticleCommandService(ArticleRepository articleRepository, + ApplicationEventPublisher applicationEventPublisher) { + this.articleRepository = articleRepository; + this.applicationEventPublisher = applicationEventPublisher; + } + + + @Transactional + public void upsert(List
articles) { + Map articleMap = articles.stream() + .collect(Collectors.toMap(Article::getArticleId, e -> e)); + + List
saved = articleRepository.findByArticleIdIn(articleMap.keySet()); + List
updated = saved.stream() + .filter(e -> articleMap.containsKey(e.getArticleId())) + .map(e -> e.update(articleMap.get(e.getArticleId()))) + .peek(e -> articleMap.remove(e.getArticleId())) + .sorted(Comparator.comparing(Article::getArticleId)) + .toList(); + + List
newArticles = articleMap.values() + .stream() + .sorted(Comparator.comparing(Article::getArticleId)) + .toList(); + + articleRepository.saveAll(updated); + articleRepository.saveAll(newArticles); + + publish(newArticles); + } + + private void publish(List
articles) { + List articleModels = + articles.stream() + .map(transformer) + .toList(); + applicationEventPublisher.publishEvent(new ArticleUpsertEvent(articleModels)); + } + + @Transactional + public void upsert(Article article) { + Article saved = articleRepository.findByArticleId(article.getArticleId()) + .orElse(article); + articleRepository.save(saved); + } + + public static Function transformer = + article -> ArticleModel.builder() + .id(article.getId()) + .articleId(article.getArticleId()) + .crawlTarget(article.getCrawlTarget()) + .boardName(article.getBoardName()) + .articleUrl(article.getArticleUrl()) + .title(article.getTitle()) + .hit(article.getHit()) + .recommended(article.getRecommended()) + .registeredAt(article.getRegisteredAt()) + .build(); +} diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/MessageSenderService.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/MessageSenderService.java similarity index 57% rename from processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/MessageSenderService.java rename to shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/MessageSenderService.java index 2a6d0ac..4671c4f 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/MessageSenderService.java +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/MessageSenderService.java @@ -1,15 +1,7 @@ -package com.myoa.engineering.crawl.ppomppu.processor.service; - -import java.util.List; - -import org.springframework.stereotype.Service; - -import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle; -import com.myoa.engineering.crawl.ppomppu.processor.dto.PpomppuArticleTransformer; -import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.client.MessageSenderAPIClient; +package com.myoa.engineering.crawl.shopping.service; import lombok.extern.slf4j.Slf4j; -import reactor.core.publisher.Mono; +import org.springframework.stereotype.Service; /** * MessageSenderService @@ -21,18 +13,20 @@ import reactor.core.publisher.Mono; @Service public class MessageSenderService { + /* private final MessageSenderAPIClient messageSenderAPIClient; public MessageSenderService(MessageSenderAPIClient messageSenderAPIClient) { this.messageSenderAPIClient = messageSenderAPIClient; } - public Mono sendMessageToSlack(PpomppuArticle article) { + public String sendMessageToSlack(PpomppuArticle article) { return messageSenderAPIClient.sendMessageToSlack(PpomppuArticleTransformer.TRANSFORM_TO_MESSAGE_DTO.apply(article)); } - public Mono sendMessageToSlack(List articles) { + public String sendMessageToSlack(List articles) { return messageSenderAPIClient.sendMessageToSlack(PpomppuArticleTransformer.transform(articles)); } + */ } diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/PpomppuArticleService.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/PpomppuArticleService.java similarity index 84% rename from processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/PpomppuArticleService.java rename to shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/PpomppuArticleService.java index ad210bb..53997a7 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/PpomppuArticleService.java +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/PpomppuArticleService.java @@ -1,20 +1,18 @@ -package com.myoa.engineering.crawl.ppomppu.processor.service; +package com.myoa.engineering.crawl.shopping.service; + +import com.myoa.engineering.crawl.shopping.domain.entity.v1.PpomppuArticle; +import com.myoa.engineering.crawl.shopping.domain.entity.v1.PpomppuBoardFeedStatus; +import com.myoa.engineering.crawl.shopping.infra.repository.v1.PpomppuArticleRepository; +import com.myoa.engineering.crawl.shopping.infra.repository.v1.PpomppuBoardFeedStatusRepository; +import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; import java.util.List; import java.util.Optional; import java.util.stream.Collectors; -import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; - -import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle; -import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuBoardFeedStatus; -import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository.PpomppuArticleRepository; -import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.repository.PpomppuBoardFeedStatusRepository; -import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName; - -import lombok.extern.slf4j.Slf4j; - @Slf4j @Service public class PpomppuArticleService { diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/PpomppuFeedService.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/PpomppuFeedService.java similarity index 82% rename from processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/PpomppuFeedService.java rename to shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/PpomppuFeedService.java index b8a3568..f5a4110 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/service/PpomppuFeedService.java +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/PpomppuFeedService.java @@ -1,16 +1,12 @@ -package com.myoa.engineering.crawl.ppomppu.processor.service; +package com.myoa.engineering.crawl.shopping.service; -import com.myoa.engineering.crawl.ppomppu.processor.domain.PpomppuArticle; -import com.myoa.engineering.crawl.ppomppu.processor.dto.PpomppuArticleParser; -import com.myoa.engineering.crawl.ppomppu.processor.infrastructure.client.PpomppuBoardFeedRetriever; -import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName; -import java.util.List; +import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName; import lombok.extern.slf4j.Slf4j; import org.jsoup.Jsoup; import org.jsoup.nodes.Element; import org.springframework.stereotype.Component; -import reactor.core.publisher.Flux; -import reactor.core.publisher.Mono; + +import java.util.List; /** * PpomppuFeedService @@ -21,7 +17,7 @@ import reactor.core.publisher.Mono; @Slf4j @Component public class PpomppuFeedService { - +/* private final PpomppuBoardFeedRetriever ppomppuBoardFeedRetriever; public PpomppuFeedService(PpomppuBoardFeedRetriever ppomppuBoardFeedRetriever) { @@ -56,4 +52,6 @@ public class PpomppuFeedService { private PpomppuArticle convertFromElement(Element element) { return PpomppuArticleParser.toArticle(element.getElementsByTag("td")); } + + */ } diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/SubscribedKeywordQueryService.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/SubscribedKeywordQueryService.java new file mode 100644 index 0000000..9aa461e --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/SubscribedKeywordQueryService.java @@ -0,0 +1,26 @@ +package com.myoa.engineering.crawl.shopping.service; + +import com.myoa.engineering.crawl.shopping.domain.entity.v2.SubscribedKeyword; +import com.myoa.engineering.crawl.shopping.infra.repository.v2.SubscribedKeywordRepository; +import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget; +import org.springframework.stereotype.Service; + +import java.util.List; + +@Service +public class SubscribedKeywordQueryService { + + private final SubscribedKeywordRepository subscribedKeywordRepository; + + public SubscribedKeywordQueryService(SubscribedKeywordRepository subscribedKeywordRepository) { + this.subscribedKeywordRepository = subscribedKeywordRepository; + } + + public List findAll() { + return subscribedKeywordRepository.findAll(); + } + + public List findByCrawlTarget(CrawlTarget crawlTarget) { + return subscribedKeywordRepository.findByCrawlTarget(crawlTarget); + } +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/util/DateTimeUtils.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/util/DateTimeUtils.java new file mode 100644 index 0000000..3ec9d5a --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/util/DateTimeUtils.java @@ -0,0 +1,27 @@ +package com.myoa.engineering.crawl.shopping.util; + +import java.time.*; +import java.time.format.DateTimeFormatter; + +public final class DateTimeUtils { + + private static final DateTimeFormatter FORMATTER_HHMMss = DateTimeFormatter.ofPattern("HH:mm:ss"); + private static final ZoneId ZONE_ASIA_SEOUL = ZoneId.of("Asia/Seoul"); + + private DateTimeUtils() { + } + + public static ZonedDateTime parse(String HHMMss) { + try { + LocalTime time = LocalTime.parse(HHMMss, FORMATTER_HHMMss); + LocalDateTime dateTime = LocalDateTime.of(LocalDate.now(), time); + if (dateTime.isAfter(LocalDateTime.now())) { + dateTime = dateTime.minusDays(1); + } + return dateTime.atZone(ZONE_ASIA_SEOUL); + } catch (Exception e) { + return null; + } + } + +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/util/NumberUtils.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/util/NumberUtils.java new file mode 100644 index 0000000..6efae60 --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/util/NumberUtils.java @@ -0,0 +1,32 @@ +package com.myoa.engineering.crawl.shopping.util; + +public final class NumberUtils { + + private NumberUtils() { + } + + public static Integer parseInt(String value) { + return parseInt(value, null); + } + + public static Integer parseInt(String value, Integer defaultValue) { + try { + return Integer.parseInt(value); + } catch (NumberFormatException e) { + return defaultValue; + } + } + + public static Long parseLong(String value) { + return parseLong(value, null); + } + + public static Long parseLong(String value, Long defaultValue) { + try { + return Long.parseLong(value); + } catch (NumberFormatException e) { + return defaultValue; + } + } + +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/util/ObjectMapperFactory.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/util/ObjectMapperFactory.java new file mode 100644 index 0000000..0785ccb --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/util/ObjectMapperFactory.java @@ -0,0 +1,62 @@ +package com.myoa.engineering.crawl.shopping.util; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.core.json.JsonReadFeature; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.databind.json.JsonMapper; +import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import com.fasterxml.jackson.module.paramnames.ParameterNamesModule; + +public final class ObjectMapperFactory { + + public static final ObjectMapper DEFAULT_MAPPER; + public static final ObjectMapper REDIS_MAPPER; + @Deprecated + public static final ObjectMapper LOGGING_MAPPER; + + static { + DEFAULT_MAPPER = initDefaultMapper(); + REDIS_MAPPER = initRedisMapper(); + LOGGING_MAPPER = JsonMapper.builder() + .serializationInclusion(JsonInclude.Include.NON_EMPTY) + .build(); + } + + private ObjectMapperFactory() { + } + + private static ObjectMapper initDefaultMapper() { + return JsonMapper.builder() + .configure(JsonReadFeature.ALLOW_UNQUOTED_FIELD_NAMES.mappedFeature(), true) + .configure(JsonReadFeature.ALLOW_UNESCAPED_CONTROL_CHARS.mappedFeature(), true) + .disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES) + .disable(SerializationFeature.FAIL_ON_EMPTY_BEANS) + .disable(SerializationFeature.FAIL_ON_UNWRAPPED_TYPE_IDENTIFIERS) + .enable(DeserializationFeature.READ_UNKNOWN_ENUM_VALUES_AS_NULL) + .disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS) + .serializationInclusion(JsonInclude.Include.NON_NULL) + .build() + .registerModule(new ParameterNamesModule()) + .registerModule(new Jdk8Module()) + .registerModule(new JavaTimeModule()); + } + + private static ObjectMapper initRedisMapper() { + return JsonMapper.builder() + .configure(JsonReadFeature.ALLOW_UNQUOTED_FIELD_NAMES.mappedFeature(), true) + .configure(JsonReadFeature.ALLOW_UNESCAPED_CONTROL_CHARS.mappedFeature(), true) + .disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES) + .disable(SerializationFeature.FAIL_ON_EMPTY_BEANS) + .disable(SerializationFeature.FAIL_ON_UNWRAPPED_TYPE_IDENTIFIERS) + .enable(DeserializationFeature.READ_UNKNOWN_ENUM_VALUES_AS_NULL) + .disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS) + .serializationInclusion(JsonInclude.Include.NON_NULL) + .build() + .registerModule(new ParameterNamesModule()) + .registerModule(new Jdk8Module()) + .registerModule(new JavaTimeModule()); + } +} diff --git a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/util/ObjectUtil.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/util/ObjectUtils.java similarity index 91% rename from processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/util/ObjectUtil.java rename to shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/util/ObjectUtils.java index 2f03f12..765100b 100644 --- a/processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/util/ObjectUtil.java +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/util/ObjectUtils.java @@ -3,7 +3,7 @@ * LINE Corporation PROPRIETARY/CONFIDENTIAL. Use is subject to license terms. */ -package com.myoa.engineering.crawl.ppomppu.processor.util; +package com.myoa.engineering.crawl.shopping.util; import java.util.Arrays; import java.util.Collection; @@ -14,9 +14,9 @@ import java.util.Collection; * @author Shin Woo-jin (lp12254@linecorp.com) * @since 2019-10-28 */ -public final class ObjectUtil { +public final class ObjectUtils { - private ObjectUtil() { + private ObjectUtils() { } /** @@ -58,7 +58,7 @@ public final class ObjectUtil { * @return Is there objects array has null? */ public static boolean hasNullObject(Object... args) { - return Arrays.stream(args).anyMatch(ObjectUtil::isNullObject); + return Arrays.stream(args).anyMatch(ObjectUtils::isNullObject); } /** @@ -72,7 +72,7 @@ public final class ObjectUtil { * @return Is there objects array has null? */ public static boolean hasAllObject(Object... args) { - return Arrays.stream(args).noneMatch(ObjectUtil::isNullObject); + return Arrays.stream(args).noneMatch(ObjectUtils::isNullObject); } /** @@ -86,7 +86,7 @@ public final class ObjectUtil { * @return Is there null all of given objects? */ public static boolean hasAllNullObjects(final Object... args) { - return Arrays.stream(args).allMatch(ObjectUtil::isNullObject); + return Arrays.stream(args).allMatch(ObjectUtils::isNullObject); } /** diff --git a/shopping-crawler/src/main/resources/application-development.yml b/shopping-crawler/src/main/resources/application-development.yml new file mode 100644 index 0000000..d4ef084 --- /dev/null +++ b/shopping-crawler/src/main/resources/application-development.yml @@ -0,0 +1,12 @@ +spring: + config: + activate: + on-profile: development +# import: +# - "configserver:http://192.168.0.100:20085" + + +server: + port: 20080 + + # import: optional:configserver:http://localhost:11080 # can be start up even config server was not found. \ No newline at end of file diff --git a/shopping-crawler/src/main/resources/application-local.yml b/shopping-crawler/src/main/resources/application-local.yml new file mode 100644 index 0000000..7737491 --- /dev/null +++ b/shopping-crawler/src/main/resources/application-local.yml @@ -0,0 +1,12 @@ +spring: + config: + activate: + on-profile: local + import: + - classpath:/datasource/local.yml + - classpath:/slack/local.yml + +server: + port: 20080 + + # import: optional:configserver:http://localhost:11080 # can be start up even config server was not found. \ No newline at end of file diff --git a/sender/src/main/resources/application-production.yml b/shopping-crawler/src/main/resources/application-production.yml similarity index 100% rename from sender/src/main/resources/application-production.yml rename to shopping-crawler/src/main/resources/application-production.yml diff --git a/shopping-crawler/src/main/resources/application.yml b/shopping-crawler/src/main/resources/application.yml new file mode 100644 index 0000000..a489600 --- /dev/null +++ b/shopping-crawler/src/main/resources/application.yml @@ -0,0 +1,28 @@ +spring: + application: + name: crawler-shopping + main: + allow-bean-definition-overriding: true + profiles: + active: ${SPRING_ACTIVE_PROFILE:local} + group: + local: "local,datasource-local,webclient-local" + development: "development,datasource-development,webclient-development" + production: "production, datasource-production,webclient-production" + freemarker: + enabled: false + cloud: + config: + enabled: false + +server: + port: 20080 + error: + whitelabel: + enabled: false + +management: + endpoints: + web: + exposure: + include: refresh diff --git a/shopping-crawler/src/main/resources/logback-spring.xml b/shopping-crawler/src/main/resources/logback-spring.xml new file mode 100644 index 0000000..907a4f5 --- /dev/null +++ b/shopping-crawler/src/main/resources/logback-spring.xml @@ -0,0 +1,15 @@ + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/sender/src/main/resources/logback/component/logback-file.xml b/shopping-crawler/src/main/resources/logback/component/logback-file.xml similarity index 100% rename from sender/src/main/resources/logback/component/logback-file.xml rename to shopping-crawler/src/main/resources/logback/component/logback-file.xml diff --git a/sender/src/main/resources/logback/logback-development.xml b/shopping-crawler/src/main/resources/logback/logback-development.xml similarity index 100% rename from sender/src/main/resources/logback/logback-development.xml rename to shopping-crawler/src/main/resources/logback/logback-development.xml diff --git a/sender/src/main/resources/logback/logback-production.xml b/shopping-crawler/src/main/resources/logback/logback-production.xml similarity index 100% rename from sender/src/main/resources/logback/logback-production.xml rename to shopping-crawler/src/main/resources/logback/logback-production.xml diff --git a/shopping-crawler/src/test/java/com/myoa/engineering/crawl/shopping/crawlhandler/parser/PpomppuArticleParserV2Test.java b/shopping-crawler/src/test/java/com/myoa/engineering/crawl/shopping/crawlhandler/parser/PpomppuArticleParserV2Test.java new file mode 100644 index 0000000..c39869e --- /dev/null +++ b/shopping-crawler/src/test/java/com/myoa/engineering/crawl/shopping/crawlhandler/parser/PpomppuArticleParserV2Test.java @@ -0,0 +1,118 @@ +package com.myoa.engineering.crawl.shopping.crawlhandler.parser; + +import com.myoa.engineering.crawl.shopping.domain.entity.v1.PpomppuArticle; +import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName; +import com.myoa.engineering.crawl.shopping.util.DateTimeUtils; +import com.myoa.engineering.crawl.shopping.util.NumberUtils; +import com.myoa.engineering.crawl.shopping.util.TestDataUtils; +import io.micrometer.core.instrument.util.StringUtils; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; +import org.junit.jupiter.api.Test; + +import java.time.ZonedDateTime; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +class PpomppuArticleParserV2Test { + + @Test + void test1() { + String data = TestDataUtils.fileToString("testdata/zboard/file1.html"); + Document document = Jsoup.parse(data); + Elements trList = document.getElementById("revolution_main_table").getElementsByTag("tr"); + + List articles = trList.stream() + .filter(this::isRealArticle) + .map(this::parse) + .toList(); + System.out.println(articles.size()); + System.out.println(articles.get(0)); + System.out.println(articles.get(1)); + } + + private boolean isRealArticle(Element tr) { + Elements tdList = tr.getElementsByTag("td"); + if (tdList.size() != 6) { + return false; + } + + if (!hasOnlyNumeric(tdList.get(0))) { + return false; + } + + return true; + } + + + Pattern pattern_numeric = Pattern.compile("\\d+"); + + private boolean hasOnlyNumeric(Element td) { + return pattern_numeric.matcher(td.text()).matches(); + } + + public PpomppuArticle parse(Element tr) { + Elements tdList = tr.getElementsByTag("td"); + Long articleId = Long.parseLong(tdList.get(0).text()); + + String title = tdList.get(2).text(); + String articleUrl = parseArticleUrl(tdList.get(2).getElementsByTag("a").attr("href")); + PpomppuBoardName boardName = parseBoardName(title); + Integer recommended = parseRecommended(tdList.get(4)); + Integer hit = NumberUtils.parseInt(tdList.get(5).text(), 0); + + + ZonedDateTime registeredAt = DateTimeUtils.parse(tdList.get(3).text()); + + return PpomppuArticle.builder() + .articleId(articleId) + .title(title) + .boardName(boardName) + .articleUrl(articleUrl) + .recommended(recommended) + .hit(hit) + .registeredAt(registeredAt) + .build(); + } + + public Long parseArticleId(Element td) { + return Long.parseLong(td.text().trim()); + } + + public static Integer parseRecommended(Element td) { + final String voteString = td.text(); + if (StringUtils.isEmpty(voteString)) { + return null; + } + + final int voteUp = Integer.parseInt(td.text().split(" - ")[0]); + final int voteDown = Integer.parseInt(td.text().split(" - ")[1]); + int recommended = voteUp - voteDown; + return recommended; + } + + public static String parseArticleUrl(String data) { + return PpomppuBoardName.ofViewPageUrl(data); + } + + + Pattern patternBoardName = Pattern.compile("\\[(.+?)\\]"); + + public PpomppuBoardName parseBoardName(String fullTitle) { + Matcher matcher = patternBoardName.matcher(fullTitle); + String lastMatched = null; + while (matcher.find()) { + lastMatched = matcher.group(1); + } + return PpomppuBoardName.ofBoardName(lastMatched, true); + } + + @Test + void test2() { + PpomppuBoardName boardName = parseBoardName("[자사몰]푸마 메쉬 폼스트라이프 [에디션] 5종 [세트] (18,220원/무료)6 [의류/잡화]"); + System.out.println(boardName); + } +} \ No newline at end of file diff --git a/shopping-crawler/src/test/java/com/myoa/engineering/crawl/shopping/event/handler/ArticleUpsertEventListenerTest.java b/shopping-crawler/src/test/java/com/myoa/engineering/crawl/shopping/event/handler/ArticleUpsertEventListenerTest.java new file mode 100644 index 0000000..375fe80 --- /dev/null +++ b/shopping-crawler/src/test/java/com/myoa/engineering/crawl/shopping/event/handler/ArticleUpsertEventListenerTest.java @@ -0,0 +1,81 @@ +package com.myoa.engineering.crawl.shopping.event.handler; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.ahocorasick.trie.Emit; +import org.ahocorasick.trie.Trie; +import org.jeasy.random.EasyRandom; +import org.jeasy.random.EasyRandomParameters; +import org.junit.jupiter.api.Test; + +import java.util.Collection; +import java.util.List; + +class ArticleUpsertEventListenerTest { + + @Test + public void test1() { + + List baseData = List.of( + "[공홈]베베숲 시그니처 위드 블루 20팩 (25,990원/무료)2 [기타]", + "[현대H몰]10주년 스페셜 에디션 봉고데기 40mm 세트 (67,640원/무료)3 [가전/가구]", + "[SSG]필립스 면도기 칫솔 기획전( 437,000원~/무료)1 [기타]", + "(티몬)제스프리 골드키위 중대과 1.8kg내외 (카페,토페 14,823원/무배)3 [식품/건강]", + "[공식몰]연세생활건강 당뇨영양식 24팩 (33,500원/유배)1 [식품/건강]", + "[인터파크]크리스탈라이트 아이스티 에이드 온더고 60개입 멀티팩(17,800원/무료)6 [기타]", + "[인팍쇼핑]샤카웨어 액티브 맥스 헤비웨이트 10종 택 1 (7,350원/무료)7 [의류/잡화]" + ); + + Trie trie = Trie.builder() + .ignoreCase() + .addKeyword("블루") + .addKeyword("봉고") + .build(); + + List> list = baseData.stream() + .map(trie::parseText) + .toList(); + + System.out.println(list); + } + + @Test + void test2() throws InterruptedException { + EasyRandomParameters paramsKeyword = new EasyRandomParameters() + .stringLengthRange(2, 10); + EasyRandom GEN_KEYWORD = new EasyRandom(paramsKeyword); + + EasyRandomParameters paramsSentence = new EasyRandomParameters() + .stringLengthRange(100, 1000); + EasyRandom GEN_SENTENCE = new EasyRandom(paramsSentence); + + Trie.TrieBuilder trieBuilder = Trie.builder().ignoreOverlaps(); + + GEN_KEYWORD.objects(TestA.class, 1000) + .map(TestA::getFieldA) + .forEach(trieBuilder::addKeyword); + Trie trie = trieBuilder.build(); + + List> listOfEmits = GEN_SENTENCE.objects(TestA.class, 2000) + .map(TestA::getFieldA) + .map(trie::parseText) + .toList(); + + for(Collection emits : listOfEmits) { + System.out.println(emits); + System.out.println("--------------------------"); + } + + } + + + @Data + @AllArgsConstructor + @NoArgsConstructor + public class TestA { + private String fieldA; + + } + +} \ No newline at end of file diff --git a/shopping-crawler/src/test/java/com/myoa/engineering/crawl/shopping/util/TestDataUtils.java b/shopping-crawler/src/test/java/com/myoa/engineering/crawl/shopping/util/TestDataUtils.java new file mode 100644 index 0000000..e5ff1fe --- /dev/null +++ b/shopping-crawler/src/test/java/com/myoa/engineering/crawl/shopping/util/TestDataUtils.java @@ -0,0 +1,62 @@ +package com.myoa.engineering.crawl.shopping.util; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.springframework.lang.NonNull; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Scanner; +import java.util.stream.Collectors; + +public final class TestDataUtils { + + private static final ObjectMapper OBJECT_MAPPER; + + static { + OBJECT_MAPPER = ObjectMapperFactory.DEFAULT_MAPPER; + } + + private TestDataUtils() { + } + + public static T inputStreamToObject(@NonNull String resourcePath, + @NonNull Class clazz) throws IOException { + return OBJECT_MAPPER.readValue(fileToString(resourcePath), clazz); + } + + public static List inputStreamsToList(@NonNull Class clazz, + @NonNull String... resourcePath) throws IOException { + return inputStreamsToList(Arrays.asList(resourcePath), clazz); + } + + public static List inputStreamsToList(@NonNull List resourcePath, + @NonNull Class clazz) throws IOException { + return resourcePath.stream() + .map(e -> { + try { + return OBJECT_MAPPER.readValue(fileToString(e), clazz); + } catch (JsonProcessingException jsonProcessingException) { + jsonProcessingException.printStackTrace(); + return null; + } + }) + .collect(Collectors.toList()); + } + + public static List inputStreamToList(@NonNull String resourcePath, + @NonNull Class clazz) throws IOException { + return (List) OBJECT_MAPPER.readValue( + fileToString(resourcePath), + OBJECT_MAPPER.getTypeFactory().constructCollectionType(List.class, clazz)); + } + + public static String fileToString(@NonNull String resourcePath) { + + final ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); + final Scanner s = new Scanner(classLoader.getResourceAsStream(resourcePath)).useDelimiter("\\A"); + return s.hasNext() ? s.next() : ""; + } + +} diff --git a/shopping-crawler/src/test/resources/logback-development.xml b/shopping-crawler/src/test/resources/logback-development.xml new file mode 100644 index 0000000..ab50ad9 --- /dev/null +++ b/shopping-crawler/src/test/resources/logback-development.xml @@ -0,0 +1,12 @@ + + + + + + + + + + + + diff --git a/shopping-crawler/src/test/resources/logback-spring.xml b/shopping-crawler/src/test/resources/logback-spring.xml new file mode 100644 index 0000000..04ccbd8 --- /dev/null +++ b/shopping-crawler/src/test/resources/logback-spring.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/shopping-crawler/src/test/resources/testdata/zboard/file1.html b/shopping-crawler/src/test/resources/testdata/zboard/file1.html new file mode 100644 index 0000000..2de3e57 --- /dev/null +++ b/shopping-crawler/src/test/resources/testdata/zboard/file1.html @@ -0,0 +1,1100 @@ + + + + + + + + + + + + + + + + 뽐뿌 - 뽐뿌게시판 + + + + + + + + + + + + + + + + + + + + +
+
+
+
+ +
+
+
+ +
+
+
+
 
+
+
+
+ 뽐뿌 + 뽐뿌 +
+ +
+
+
+
+ + +
+ +
+ + +
+
+ +
+ +
+ + +
+ + + + +
+ 로그인 + 회원가입|아이디비번찾기 +
+
+
+ +
+ +
+

뽐뿌게시판 입니다.

+ +
+
+ 가성비 좋은 상품과 쇼핑정보를 공유하는 공간입니다. [뽐뿌게시판 이용규칙] +
+ + + + + + + + + + + + + + + + + + + +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
   번호     글쓴이  제목 등록일 추천    조회   
공지관리자3[필독] 제목에는 특수 문자 사용을 절대 금지합니다.- 2010년 2월 24일0 4324677
알림관리자아이콘알리익스프레스 상품은 [알리뽐뿌]에 공유해 주세요!
알림관리자아이콘[키워드 알림] 관심 키워드 등록하고, 앱push 알림 받자!
537951매력적인뽐뿌 + +
+
+ [지마켓]천연 향수 비누 사봉파퓨메 135g 3개+사은품 (9,950원/무료)1 +
+ [기타] +
1069
537950123aaa + +
+
+ [지마켓]순면 국내생산 무지티3장 (유클12,410원/무배) +
+ [의류/잡화] +
1508
537949유유자적루 + +
+
+ [네이버-미루몰] 벨루아체 올인원 포맨 4종 (29,900원/무료)5 +
+ [화장품] +
1201
537947illilliiill + +
+
+ [티몬]역대가 핫딜! 슈퍼대디 세이프 77평량 아기물티슈 (17,991원/무배)74  +
+ [기타] +
9 - 0 14917
5379461회제공량 + +
+
+ [CGV씨네샵]타카라토미 버즈라이트이어 49,000 / 3,0006 +
+ [기타] +
9522
537945가을말차라떼 + +
+
+ [위메프]샤오미 스마트 선풍기 7세대 1X 업그레이드/Type-C 충전 포트 (43,160원/무배)16 +
+ [기타] +
1 - 0 17666
537944거북반 + +
+
+ [무신사]FCMM 풋볼 프로 스트레치 윈드브레이커 (32,364원/무료)3 +
+ [의류/잡화] +
13039
537942jimin1120 + +
+
+ 북유럽 스타일 양손 스프볼 도자기 스프그릇 소스접시 짱구그릇 라면그릇 만두접시 특가전(8,400원~/무료)23 +
+ [기타] +
2 - 0 5767
537941너구리사랑 + +
+
+ [롯데온]제스프리 골드키위, 토레타, 가자미밥상, 쿡반 새우볶음밥(다양/무료)20 +
+ [식품/건강] +
16 - 0 10050
537940깡패건달 + +
+
+ [자사몰]푸마 메쉬 폼스트라이프 에디션 5종 세트 (18,220원/무료)6 +
+ [의류/잡화] +
4 - 0 7273
537939심의켄 + +
+
+ [위메프]오리온 비쵸비 초코비스킷5pX8개 외 치즈톡/감자톡/오징어톡,도도한 나쵸(13,220원/무료)10 +
+ [식품/건강] +
8 - 0 7724
537938벌꿀오소리는강해 + +
+
+ [네이버]추억의 오란다 250g x2봉(6,200원/무배)9 +
+ [식품/건강] +
5892
537937매카aa + +
+
+ [G마켓]휴대폰 젤리케이스1+1 갤럭시,아이폰 (3,610원/무배)5 +
+ [기타] +
5186
537936나무007 + +
+
+ [CJ온스타일]동원 지중해 황도 410g 12캔(16,110/무료)44 +
+ [식품/건강] +
15 - 0 13237
537935IilliillIi + +
+
+ [네이버]맥주효모 비오틴 아누카사과 프리미엄 1박스 (1개월분) (34,900원/무료)2  +
+ [식품/건강] +
3597
537934최군어뭉 + +
+
+ [위메프]델리만쥬 붕어빵/핫도그 600g 2봉 (8,700/무료)15 +
+ [식품/건강] +
6 - 0 6264
537933무념무상해 + +
+
+ [인터파크]산리오 미니 말랑스티커/케이크하우스 외 다수 (2,970원부터/3개 이상 주문시 무료) +
+ [육아] +
1 - 0 6198
537931p1ano + +
+
+ [위메프]비오틴 울트라 4개월분 (4,900원/무배)8  +
+ [식품/건강] +
3 - 0 6065
537930킹갓제네럴MZ + +
+
+ [위메프]아스트로글라이드젤 148ml 1+1 (27,470원/무료배송)44 +
+ [기타] +
1 - 0 16634
537929꿉지 + +
+
+ [옥션]다농바이오 가무치소주 43도 375ml(32,900원/무료)3 +
+ [식품/건강] +
1 - 0 5952
쇼핑뽐뿌 아이콘아스라슬 + +
+
+ [카카오톡 스토어] 캘빈클라인 코튼 반팔티 15,900원 외 폴로반팔티, 반바지, PK카라티, 원피스, (무배,무교,무반) + 256 +
+
+ 		56 - 05325
쇼핑뽐뿌 아이콘LG전자쿠팡공식대행... + +
+
+ [쿠팡] LG 그램 주말 특가 찬스! 와우 즉시할인+농협카드 추가 할인+내일 배송 도착 + 246 +
+
+ 		57 - 03128
+
+

+ 쇼핑포럼뽐뿌상담실 아이콘더 많은 쇼핑 정보와 구매 전 자유로운 쇼핑 이야기는 쇼핑포럼에서! +

+
+
핫딜 아이콘 + 쿠팡 + + +
+
+ 나이키 에브리데이 쿠션크루 트레이닝삭스 장목양말 1세트 3켤레입 (9,490원) +
+
+ 		7
쇼핑포럼 아이콘 + ll427ll + + +
+
+ [알리]펩시제로 라임향 355ml 24캔(13,582원/무료) +
+ [식품/건강]
+ 		907
쇼핑포럼 아이콘 + HBAF + + +
+
+  국산 갈치 1마리 39800원 +
+ [식품/건강]
+ 		1 - 010894
쇼핑포럼 아이콘 + 시뻘건맛 + + +
+
+  남자 집에서 입을 반바지 통풍잘되는거 있을까요? +
+ [의류/잡화]
+ 		10935
+
+
+ +
+
+
+
+
+
+ + + + +
+ +
+
+

+ 실시간 질렀어요 +
    +
  1. 1
  2. 2
+

+ +
    +
  • +
    • +
  • +
    • +
+
+ +
+

+ 최근 내가 본 상품 +
    +
+

+
    +
  • 최근 내가 본 상품이 없습니다.
+
+
+ + + + + + +
+ +  -새로고침  -다음페이지   +
+ + + + + + + +
+ + 1 2 3 4 5 6 7 8 9 10 다음 +
+ + + + + + + + + + + + +
+ + + + + +
+ + + + + + + + + +
+
+ + + + +
+ +
+ + +
+ + + + + + + +
+
+ + + +
+ +
+ + 사이트소개 + 운영참여 + + + 게시중단요청 + + 이용약관 + 개인정보처리방침 + 청소년보호정책 + 불법촬영물등 신고 + 광고/제휴 + 메일문의 + + + 뽐뿌채용 + + 다크모드 +
+ + + + + +
+
+
+

뽐뿌는 쇼핑 정보 커뮤니티입니다. 상품 주문, 배송 및 환불의 의무와 책임은 각 판매 업체에 있습니다.

+

(주)뽐뿌커뮤니케이션 / 대표이사 : 고재성 / 사업자등록번호 : 117-81-66008 / 통신판매업신고번호 : 제2015-서울영등포-1441호

+

주소 : 서울특별시 영등포구 당산로 41길 당산 SK V1센터 W동 1411호 + / 팩스 : 02-6280-7029 / 사업자회원 FAQ

+ +
+ + Copyright 2005-2024 ppomppu. All rights reserved +
+ +
+
+ + + + + + + + + + + + + + + + + diff --git a/support/build.gradle b/support/build.gradle index 81d2bcc..53479a9 100644 --- a/support/build.gradle +++ b/support/build.gradle @@ -3,7 +3,7 @@ dependencies { // https://projectreactor.io/docs/core/release/reference/#debug-activate annotationProcessor 'org.projectlombok:lombok' - implementation 'org.springframework.boot:spring-boot-starter-webflux' + implementation 'org.springframework.boot:spring-boot-starter-web' } diff --git a/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/dto/code/PpomppuBoardName.java b/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/dto/code/PpomppuBoardName.java deleted file mode 100644 index d66bdc0..0000000 --- a/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/dto/code/PpomppuBoardName.java +++ /dev/null @@ -1,51 +0,0 @@ -package com.myoa.engineering.crawl.ppomppu.support.dto.code; - -import lombok.AllArgsConstructor; -import lombok.Getter; - -/** - * PpomppuBoardName - * - * @author Shin Woo-jin (woo-jin.shin@linecorp.com) - * @since 2021-09-05 - */ -@Getter -@AllArgsConstructor -public enum PpomppuBoardName { - PPOMPPU_DOMESTIC_ALL("/zboard/zboard.php?id=ppomppu", "전체", false), - PPOMPPU_DOMESTIC_ETC("/zboard/zboard.php?id=ppomppu&category=1", "기타", true), - PPOMPPU_DOMESTIC_COMPUTER("/zboard/zboard.php?id=ppomppu&category=4", "컴퓨터", true), - PPOMPPU_DOMESTIC_DIGITAL("/zboard/zboard.php?id=ppomppu&category=5", "디지털", true), - PPOMPPU_DOMESTIC_FOOD("/zboard/zboard.php?id=ppomppu&category=6", "식품/건강", true), - PPOMPPU_DOMESTIC_BOOK("/zboard/zboard.php?id=ppomppu&category=8", "서적", true), - PPOMPPU_DOMESTIC_APPLIANCES("/zboard/zboard.php?id=ppomppu&category=9", "가전/가구", true), - PPOMPPU_DOMESTIC_PARENTING("/zboard/zboard.php?id=ppomppu&category=10", "육아", true), - PPOMPPU_DOMESTIC_GIFTCARD("/zboard/zboard.php?id=ppomppu&category=11", "상품권", true), - PPOMPPU_DOMESTIC_CLOTHES("/zboard/zboard.php?id=ppomppu&category=12", "의류/잡화", true), - PPOMPPU_DOMESTIC_COSMETIC("/zboard/zboard.php?id=ppomppu&category=13", "화장품", true), - PPOMPPU_DOMESTIC_OUTDOOR("/zboard/zboard.php?id=ppomppu&category=15", "등산/캠핑", true), - PPOMPPU_OVERSEA_ALL("/zboard/zboard.php?id=ppomppu4", "전체", false), - PPOMPPU_OVERSEA_ETC("/zboard/zboard.php?id=ppomppu4&category=1", "기타", true), - PPOMPPU_OVERSEA_APPLIANCES("/zboard/zboard.php?id=ppomppu4&category=7", "가전", true), - PPOMPPU_OVERSEA_TVAV("/zboard/zboard.php?id=ppomppu4&category=8", "TV/영상", true), - PPOMPPU_OVERSEA_COMPUTER("/zboard/zboard.php?id=ppomppu4&category=3", "컴퓨터", true), - PPOMPPU_OVERSEA_DIGITAL("/zboard/zboard.php?id=ppomppu4&category=4", "디지털", true), - PPOMPPU_OVERSEA_MOBILEACCESSORY("/zboard/zboard.php?id=ppomppu4&category=9", "액세서리", true), - PPOMPPU_OVERSEA_CLOTHES("/zboard/zboard.php?id=ppomppu4&category=5", "의류/잡화", true), - PPOMPPU_OVERSEA_WATCH("/zboard/zboard.php?id=ppomppu4&category=2", "시계", true), - PPOMPPU_OVERSEA_SHOES("/zboard/zboard.php?id=ppomppu4&category=11", "신발", true), - PPOMPPU_OVERSEA_FOOD("/zboard/zboard.php?id=ppomppu4&category=10", "식품/건강", true), - PPOMPPU_OVERSEA_PARENTING("/zboard/zboard.php?id=ppomppu4&category=6", "육아", true), - ; - - private String resourcePath; - private String menuName; - private boolean crawlWithDefaultTimer; - - public static final String PPOMPPU_URL = "https://www.ppomppu.co.kr"; - - public static String ofViewPageUrl(String articleUrl) { - return PPOMPPU_URL + "/zboard/" + articleUrl; - } - -} diff --git a/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/webclient/PpomppuNotifierWebClientConfiguration.java b/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/webclient/PpomppuNotifierWebClientConfiguration.java deleted file mode 100644 index b907331..0000000 --- a/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/webclient/PpomppuNotifierWebClientConfiguration.java +++ /dev/null @@ -1,19 +0,0 @@ -package com.myoa.engineering.crawl.ppomppu.support.webclient; - -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.context.annotation.ComponentScan; -import org.springframework.context.annotation.Configuration; - -/** - * PpomppuNotifierWebClientConfiguration - * @author Shin Woo-jin (woo-jin.shin@linecorp.com) - * @since 2021-11-18 - * - */ -@Configuration -@ConditionalOnProperty(value = "webclient.init", havingValue = "true") -@ComponentScan(basePackageClasses = WebClientBaseScan.class) -public class PpomppuNotifierWebClientConfiguration { - - -} diff --git a/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/webclient/WebClientBaseScan.java b/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/webclient/WebClientBaseScan.java deleted file mode 100644 index 00acb82..0000000 --- a/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/webclient/WebClientBaseScan.java +++ /dev/null @@ -1,4 +0,0 @@ -package com.myoa.engineering.crawl.ppomppu.support.webclient; - -public interface WebClientBaseScan { -} diff --git a/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/dto/APIResponse.java b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/dto/APIResponse.java similarity index 96% rename from support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/dto/APIResponse.java rename to support/src/main/java/com/myoa/engineering/crawl/shopping/support/dto/APIResponse.java index 94df264..3c20af2 100644 --- a/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/dto/APIResponse.java +++ b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/dto/APIResponse.java @@ -1,4 +1,4 @@ -package com.myoa.engineering.crawl.ppomppu.support.dto; +package com.myoa.engineering.crawl.shopping.support.dto; import java.io.Serializable; import java.util.Map; diff --git a/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/dto/APIResponseError.java b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/dto/APIResponseError.java similarity index 96% rename from support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/dto/APIResponseError.java rename to support/src/main/java/com/myoa/engineering/crawl/shopping/support/dto/APIResponseError.java index 38801f9..a73502f 100644 --- a/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/dto/APIResponseError.java +++ b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/dto/APIResponseError.java @@ -1,4 +1,4 @@ -package com.myoa.engineering.crawl.ppomppu.support.dto; +package com.myoa.engineering.crawl.shopping.support.dto; import java.io.Serializable; import java.util.HashMap; diff --git a/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/dto/SimpleMessageDTO.java b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/dto/SimpleMessageDTO.java similarity index 93% rename from support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/dto/SimpleMessageDTO.java rename to support/src/main/java/com/myoa/engineering/crawl/shopping/support/dto/SimpleMessageDTO.java index 1a2c026..033e97e 100644 --- a/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/dto/SimpleMessageDTO.java +++ b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/dto/SimpleMessageDTO.java @@ -1,4 +1,4 @@ - package com.myoa.engineering.crawl.ppomppu.support.dto; + package com.myoa.engineering.crawl.shopping.support.dto; import java.io.Serializable; import java.time.Instant; diff --git a/support/src/main/java/com/myoa/engineering/crawl/shopping/support/dto/constant/CrawlTarget.java b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/dto/constant/CrawlTarget.java new file mode 100644 index 0000000..11ba3e5 --- /dev/null +++ b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/dto/constant/CrawlTarget.java @@ -0,0 +1,13 @@ +package com.myoa.engineering.crawl.shopping.support.dto.constant; + +import lombok.AllArgsConstructor; +import lombok.Getter; + +@Getter +@AllArgsConstructor +public enum CrawlTarget { + PPOMPPU_DOMESTIC, + PPOMPPU_OVERSEA, + FMKOREA, + ; +} diff --git a/support/src/main/java/com/myoa/engineering/crawl/shopping/support/dto/constant/PpomppuBoardName.java b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/dto/constant/PpomppuBoardName.java new file mode 100644 index 0000000..cd6ac23 --- /dev/null +++ b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/dto/constant/PpomppuBoardName.java @@ -0,0 +1,62 @@ +package com.myoa.engineering.crawl.shopping.support.dto.constant; + +import lombok.AllArgsConstructor; +import lombok.Getter; + +import java.util.Arrays; + +/** + * PpomppuBoardName + * + * @author Shin Woo-jin (woo-jin.shin@linecorp.com) + * @since 2021-09-05 + */ +@Getter +@AllArgsConstructor +public enum PpomppuBoardName { + UNKNOWN("", "UNKNOWN", false, false), + PPOMPPU_DOMESTIC_ALL("id=ppomppu", "전체", false, true), + PPOMPPU_DOMESTIC_ETC("id=ppomppu&category=1", "기타", true, true), + PPOMPPU_DOMESTIC_COMPUTER("id=ppomppu&category=4", "컴퓨터", true, true), + PPOMPPU_DOMESTIC_DIGITAL("id=ppomppu&category=5", "디지털", true, true), + PPOMPPU_DOMESTIC_FOOD("id=ppomppu&category=6", "식품/건강", true, true), + PPOMPPU_DOMESTIC_BOOK("id=ppomppu&category=8", "서적", true, true), + PPOMPPU_DOMESTIC_APPLIANCES("id=ppomppu&category=9", "가전/가구", true, true), + PPOMPPU_DOMESTIC_PARENTING("id=ppomppu&category=10", "육아", true, true), + PPOMPPU_DOMESTIC_GIFTCARD("id=ppomppu&category=11", "상품권", true, true), + PPOMPPU_DOMESTIC_CLOTHES("id=ppomppu&category=12", "의류/잡화", true, true), + PPOMPPU_DOMESTIC_COSMETIC("id=ppomppu&category=13", "화장품", true, true), + PPOMPPU_DOMESTIC_OUTDOOR("id=ppomppu&category=15", "등산/캠핑", true, true), + PPOMPPU_OVERSEA_ALL("id=ppomppu4", "전체", false, false), + PPOMPPU_OVERSEA_ETC("id=ppomppu4&category=1", "기타", true, false), + PPOMPPU_OVERSEA_APPLIANCES("id=ppomppu4&category=7", "가전", true, false), + PPOMPPU_OVERSEA_TVAV("id=ppomppu4&category=8", "TV/영상", true, false), + PPOMPPU_OVERSEA_COMPUTER("id=ppomppu4&category=3", "컴퓨터", true, false), + PPOMPPU_OVERSEA_DIGITAL("id=ppomppu4&category=4", "디지털", true, false), + PPOMPPU_OVERSEA_MOBILEACCESSORY("id=ppomppu4&category=9", "액세서리", false, true), + PPOMPPU_OVERSEA_CLOTHES("id=ppomppu4&category=5", "의류/잡화", true, false), + PPOMPPU_OVERSEA_WATCH("id=ppomppu4&category=2", "시계", true, false), + PPOMPPU_OVERSEA_SHOES("id=ppomppu4&category=11", "신발", true, false), + PPOMPPU_OVERSEA_FOOD("id=ppomppu4&category=10", "식품/건강", true, false), + PPOMPPU_OVERSEA_PARENTING("id=ppomppu4&category=6", "육아", true, false), + ; + + private String resourcePath; + private String menuName; + private boolean crawlWithDefaultTimer; + private boolean isDomestic; + + public static final String PPOMPPU_URL = "https://www.ppomppu.co.kr"; + + public static String ofViewPageUrl(String articleUrl) { + return PPOMPPU_URL + "/zboard/" + articleUrl; + } + + public static PpomppuBoardName ofBoardName(String boardName, boolean isDomestic) { + return Arrays.stream(values()) + .filter(e -> e.getMenuName().equals(boardName) && e.isDomestic() == isDomestic) + .findFirst() + .orElse(UNKNOWN); + } + +} diff --git a/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/util/NumberUtil.java b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/util/NumberUtil.java similarity index 52% rename from support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/util/NumberUtil.java rename to support/src/main/java/com/myoa/engineering/crawl/shopping/support/util/NumberUtil.java index a6a0031..1b36154 100644 --- a/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/util/NumberUtil.java +++ b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/util/NumberUtil.java @@ -1,4 +1,4 @@ -package com.myoa.engineering.crawl.ppomppu.support.util; +package com.myoa.engineering.crawl.shopping.support.util; public final class NumberUtil { diff --git a/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/util/ObjectMapperFactory.java b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/util/ObjectMapperFactory.java similarity index 98% rename from support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/util/ObjectMapperFactory.java rename to support/src/main/java/com/myoa/engineering/crawl/shopping/support/util/ObjectMapperFactory.java index fcad823..48797bf 100644 --- a/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/util/ObjectMapperFactory.java +++ b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/util/ObjectMapperFactory.java @@ -1,4 +1,4 @@ -package com.myoa.engineering.crawl.ppomppu.support.util; +package com.myoa.engineering.crawl.shopping.support.util; import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.core.JsonParser.Feature; diff --git a/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/util/ObjectUtil.java b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/util/ObjectUtil.java similarity index 79% rename from support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/util/ObjectUtil.java rename to support/src/main/java/com/myoa/engineering/crawl/shopping/support/util/ObjectUtil.java index e2d9863..49322ac 100644 --- a/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/util/ObjectUtil.java +++ b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/util/ObjectUtil.java @@ -1,4 +1,4 @@ -package com.myoa.engineering.crawl.ppomppu.support.util; +package com.myoa.engineering.crawl.shopping.support.util; public final class ObjectUtil { diff --git a/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/util/WebRequestUtil.java b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/util/WebRequestUtil.java similarity index 88% rename from support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/util/WebRequestUtil.java rename to support/src/main/java/com/myoa/engineering/crawl/shopping/support/util/WebRequestUtil.java index 9204689..00675df 100644 --- a/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/util/WebRequestUtil.java +++ b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/util/WebRequestUtil.java @@ -1,4 +1,4 @@ -package com.myoa.engineering.crawl.ppomppu.support.util; +package com.myoa.engineering.crawl.shopping.support.util; /** * WebUtil diff --git a/support/src/main/java/com/myoa/engineering/crawl/shopping/support/webclient/WebClientBaseScan.java b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/webclient/WebClientBaseScan.java new file mode 100644 index 0000000..0695933 --- /dev/null +++ b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/webclient/WebClientBaseScan.java @@ -0,0 +1,4 @@ +package com.myoa.engineering.crawl.shopping.support.webclient; + +public interface WebClientBaseScan { +} diff --git a/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/webclient/factory/WebClientFilterFactory.java b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/webclient/factory/WebClientFilterFactory.java similarity index 81% rename from support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/webclient/factory/WebClientFilterFactory.java rename to support/src/main/java/com/myoa/engineering/crawl/shopping/support/webclient/factory/WebClientFilterFactory.java index 89d1ec9..cd8c29e 100644 --- a/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/webclient/factory/WebClientFilterFactory.java +++ b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/webclient/factory/WebClientFilterFactory.java @@ -1,10 +1,6 @@ -package com.myoa.engineering.crawl.ppomppu.support.webclient.factory; +package com.myoa.engineering.crawl.shopping.support.webclient.factory; import lombok.extern.slf4j.Slf4j; -import org.springframework.web.reactive.function.client.ClientRequest; -import org.springframework.web.reactive.function.client.ClientResponse; -import org.springframework.web.reactive.function.client.ExchangeFilterFunction; -import reactor.core.publisher.Mono; /** * WebClientFilterFactory @@ -15,6 +11,7 @@ import reactor.core.publisher.Mono; @Slf4j public final class WebClientFilterFactory { + /* private WebClientFilterFactory() {} public static ExchangeFilterFunction logRequest() { @@ -44,5 +41,6 @@ public final class WebClientFilterFactory { } return Mono.just(clientResponse); } + */ } diff --git a/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/webclient/factory/WebFluxExchangeStragiesFactory.java b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/webclient/factory/WebFluxExchangeStragiesFactory.java similarity index 82% rename from support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/webclient/factory/WebFluxExchangeStragiesFactory.java rename to support/src/main/java/com/myoa/engineering/crawl/shopping/support/webclient/factory/WebFluxExchangeStragiesFactory.java index 6558638..a14ab39 100644 --- a/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/webclient/factory/WebFluxExchangeStragiesFactory.java +++ b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/webclient/factory/WebFluxExchangeStragiesFactory.java @@ -1,12 +1,4 @@ -package com.myoa.engineering.crawl.ppomppu.support.webclient.factory; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.myoa.engineering.crawl.ppomppu.support.util.ObjectMapperFactory; - -import org.springframework.http.codec.json.Jackson2JsonDecoder; -import org.springframework.http.codec.json.Jackson2JsonEncoder; -import org.springframework.util.MimeTypeUtils; -import org.springframework.web.reactive.function.client.ExchangeStrategies; +package com.myoa.engineering.crawl.shopping.support.webclient.factory; /** * WebFluxExchangeStragiesFactory @@ -15,7 +7,7 @@ import org.springframework.web.reactive.function.client.ExchangeStrategies; * @since 2021-09-08 */ public final class WebFluxExchangeStragiesFactory { - +/* private WebFluxExchangeStragiesFactory() {} public static ExchangeStrategies ofDefault() { @@ -46,5 +38,5 @@ public final class WebFluxExchangeStragiesFactory { }) .build(); } - +*/ } diff --git a/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/webclient/properties/WebClientProperties.java b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/webclient/properties/WebClientProperties.java similarity index 93% rename from support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/webclient/properties/WebClientProperties.java rename to support/src/main/java/com/myoa/engineering/crawl/shopping/support/webclient/properties/WebClientProperties.java index 64b89f8..6b893f8 100644 --- a/support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/webclient/properties/WebClientProperties.java +++ b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/webclient/properties/WebClientProperties.java @@ -1,4 +1,4 @@ -package com.myoa.engineering.crawl.ppomppu.support.webclient.properties; +package com.myoa.engineering.crawl.shopping.support.webclient.properties; import java.util.ArrayList; import java.util.List; diff --git a/support/src/main/resources/datasource/local.yml b/support/src/main/resources/datasource/local.yml new file mode 100644 index 0000000..2814984 --- /dev/null +++ b/support/src/main/resources/datasource/local.yml @@ -0,0 +1,42 @@ +spring: + jpa: + open-in-view: false + hibernate: + ddl-auto: create + datasource: + # driver-class-name: com.mysql.cj.jdbc.Driver + driver-class-name: org.h2.Driver + url: jdbc:h2:mem:crawler-shopping;DB_CLOSE_DELAY=-1 + hikari: + minimum:idle: 5 + maximum-pool-size: 10 + idle-timeout: 600000 + validation-timeout: 5000 + connection-timeout: 5000 + max-lifetime: 1800000 + auto-commit: false + h2: + console: + enabled: true + path: /h2 + port: 20082 + +datasource: + init: true + units: + - unit-name: crawler-shopping + schema-name: crawler-shopping + db-connection-url: jdbc:h2:mem:crawler-shopping + is-simple-connection-url: true + driver-class-name: org.h2.Driver + username: sa + password: sa + +hibernate: + units: + - unit-name: crawler-shopping + dialect: org.hibernate.dialect.H2Dialect + format-sql: true + show-sql: true + hbm2ddl-auto: create + disable-auto-commit: true diff --git a/support/src/main/resources/development/database.yml b/support/src/main/resources/development/database.yml index a4af932..3f64bae 100644 --- a/support/src/main/resources/development/database.yml +++ b/support/src/main/resources/development/database.yml @@ -6,7 +6,7 @@ spring: datasource: # driver-class-name: com.mysql.cj.jdbc.Driver driver-class-name: org.h2.Driver - url: jdbc:h2:mem:ppomppu-local;DB_CLOSE_DELAY=-1 + url: jdbc:h2:mem:crawler-shopping;DB_CLOSE_DELAY=-1 hikari: minimum:idle: 5 maximum-pool-size: 10 @@ -24,18 +24,17 @@ spring: datasource: init: true units: - - unit-name: ppomppu - schema-name: ppomppu-local - db-connection-url: jdbc:h2:mem:ppomppu-local + - unit-name: crawler-shopping + schema-name: crawler-shopping + db-connection-url: jdbc:h2:mem:crawler-shopping simple-connection-url: true hibernate: units: - - unit-name: ppomppu + - unit-name: crawler-shopping dialect: org.hibernate.dialect.H2Dialect format-sql: true show-sql: true hbm2ddl-auto: create disable-auto-commit: true - diff --git a/support/src/main/resources/slack/local.yml b/support/src/main/resources/slack/local.yml new file mode 100644 index 0000000..8bc78cd --- /dev/null +++ b/support/src/main/resources/slack/local.yml @@ -0,0 +1,8 @@ +slack: + bot: + units: + - bot-unit-name: shopping-crawler + username: "몽이 탈호구봇" + icon-emoji: ":monge_big:" + channel: "notify_shopping" + token: "xoxb-2688454277126-2695026012277-YKlaeoSBY42NtF6Teh4z7dLK" diff --git a/sender/build.gradle b/writer/build.gradle similarity index 100% rename from sender/build.gradle rename to writer/build.gradle diff --git a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/controller/MessageSenderAPIController.java b/writer/src/main/java/com/myoa/engineering/crawl/ppomppu/writer/controller/MessageSenderAPIController.java similarity index 79% rename from sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/controller/MessageSenderAPIController.java rename to writer/src/main/java/com/myoa/engineering/crawl/ppomppu/writer/controller/MessageSenderAPIController.java index 479d101..8a2ca3f 100644 --- a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/controller/MessageSenderAPIController.java +++ b/writer/src/main/java/com/myoa/engineering/crawl/ppomppu/writer/controller/MessageSenderAPIController.java @@ -1,8 +1,8 @@ -package com.myoa.engineering.crawl.ppomppu.sender.controller; +package com.myoa.engineering.crawl.ppomppu.writer.controller; -import com.myoa.engineering.crawl.ppomppu.sender.infrastructure.client.MongeShoppingBotSlackMessageSender; -import com.myoa.engineering.crawl.ppomppu.support.dto.APIResponse; -import com.myoa.engineering.crawl.ppomppu.support.dto.SimpleMessageDTO; +import com.myoa.engineering.crawl.ppomppu.writer.infrastructure.client.MongeShoppingBotSlackMessageSender; +import com.myoa.engineering.crawl.shopping.support.dto.APIResponse; +import com.myoa.engineering.crawl.shopping.support.dto.SimpleMessageDTO; import lombok.extern.slf4j.Slf4j; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; diff --git a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/controller/TestAPIController.java b/writer/src/main/java/com/myoa/engineering/crawl/ppomppu/writer/controller/TestAPIController.java similarity index 86% rename from sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/controller/TestAPIController.java rename to writer/src/main/java/com/myoa/engineering/crawl/ppomppu/writer/controller/TestAPIController.java index 28192e1..8ec7e82 100644 --- a/sender/src/main/java/com/myoa/engineering/crawl/ppomppu/sender/controller/TestAPIController.java +++ b/writer/src/main/java/com/myoa/engineering/crawl/ppomppu/writer/controller/TestAPIController.java @@ -1,6 +1,6 @@ -package com.myoa.engineering.crawl.ppomppu.sender.controller; +package com.myoa.engineering.crawl.ppomppu.writer.controller; -import com.myoa.engineering.crawl.ppomppu.sender.infrastructure.client.MongeShoppingBotSlackMessageSender; +import com.myoa.engineering.crawl.ppomppu.writer.infrastructure.client.MongeShoppingBotSlackMessageSender; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; diff --git a/sender/src/main/resources/application-development.yml b/writer/src/main/resources/application-development.yml similarity index 100% rename from sender/src/main/resources/application-development.yml rename to writer/src/main/resources/application-development.yml diff --git a/sender/src/main/resources/application-local.yml b/writer/src/main/resources/application-local.yml similarity index 100% rename from sender/src/main/resources/application-local.yml rename to writer/src/main/resources/application-local.yml diff --git a/writer/src/main/resources/application-production.yml b/writer/src/main/resources/application-production.yml new file mode 100644 index 0000000..4ff32ed --- /dev/null +++ b/writer/src/main/resources/application-production.yml @@ -0,0 +1,6 @@ +spring: + config: + activate: + on-profile: production + import: + - "configserver:http://ppn-config-server:20080" diff --git a/sender/src/main/resources/application.yml b/writer/src/main/resources/application.yml similarity index 96% rename from sender/src/main/resources/application.yml rename to writer/src/main/resources/application.yml index 3a184fe..32c6df7 100644 --- a/sender/src/main/resources/application.yml +++ b/writer/src/main/resources/application.yml @@ -1,6 +1,6 @@ spring: application: - name: ppn-sender + name: ppn-writer main: allow-bean-definition-overriding: true profiles: diff --git a/sender/src/main/resources/logback-spring.xml b/writer/src/main/resources/logback-spring.xml similarity index 100% rename from sender/src/main/resources/logback-spring.xml rename to writer/src/main/resources/logback-spring.xml diff --git a/writer/src/main/resources/logback/component/logback-file.xml b/writer/src/main/resources/logback/component/logback-file.xml new file mode 100644 index 0000000..65e54db --- /dev/null +++ b/writer/src/main/resources/logback/component/logback-file.xml @@ -0,0 +1,23 @@ + + + + + ${DIRECTORY}/${LOG_FILE_BASE}_log + + ${DIRECTORY}/${LOG_FILE_BASE}_log.%d{yyyyMMdd}.%i + + 1000MB + + 60 + + + ${FILE_LOG_PATTERN} + ${IMMEDIATE_FLUSH} + + + + 1024 + + + diff --git a/writer/src/main/resources/logback/logback-development.xml b/writer/src/main/resources/logback/logback-development.xml new file mode 100644 index 0000000..458e3d8 --- /dev/null +++ b/writer/src/main/resources/logback/logback-development.xml @@ -0,0 +1,19 @@ + + + + + + + + + + + + + + + + + + + diff --git a/writer/src/main/resources/logback/logback-production.xml b/writer/src/main/resources/logback/logback-production.xml new file mode 100644 index 0000000..f824e41 --- /dev/null +++ b/writer/src/main/resources/logback/logback-production.xml @@ -0,0 +1,19 @@ + + + + + + + + + + + + + + + + + + +