diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..3f1f94d --- /dev/null +++ b/.gitignore @@ -0,0 +1,42 @@ +HELP.md +.gradle +build/ +!gradle/wrapper/gradle-wrapper.jar +!**/src/main/**/build/ +!**/src/test/**/build/ + +### STS ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans +.sts4-cache +bin/ +!**/src/main/**/bin/ +!**/src/test/**/bin/ + +### IntelliJ IDEA ### +.idea +*.iws +*.iml +*.ipr +out/ +!**/src/main/**/out/ +!**/src/test/**/out/ + +### NetBeans ### +/nbproject/private/ +/nbbuild/ +/dist/ +/nbdist/ +/.nb-gradle/ + +### VS Code ### +.vscode/ + +temppassword.yml +data.sql +**/src/main/resources/slack +**/src/main/resources/datasource \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..2071b23 --- /dev/null +++ b/LICENSE @@ -0,0 +1,9 @@ +MIT License + +Copyright (c) + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/PpomppuNotifier_DB.sql b/PpomppuNotifier_DB.sql new file mode 100644 index 0000000..6b06b1e --- /dev/null +++ b/PpomppuNotifier_DB.sql @@ -0,0 +1,46 @@ +create table "ppomppu_article" +( + "id" bigint generated by default as identity, + "article_id" bigint, + "article_url" varchar(255), + "board_name" integer, + "hit" integer, + "recommended" integer, + "registered_at" timestamp, + "title" varchar(255), + primary key ("id") +) + +create table "ppomppu_board_feed_status" +( + "id" bigint generated by default as identity, + "board_name" integer, + "latest_parsed_article_id" bigint, + "updated_at" timestamp, + primary key ("id") +) + +create table "published_history" +( + "id" bigint generated by default as identity, + "board_name_list" varchar(255), + "published_at" timestamp, + "user_id" bigint, + primary key ("id") +) + +create table "subscribed_board" +( + "id" bigint generated by default as identity, + "board_name" integer, + "user_id" bigint, + primary key ("id") +) + +create table "subscribed_user" +( + "id" bigint generated by default as identity, + "registered_at" timestamp, + "user_id" bigint, + primary key ("id") +) \ No newline at end of file diff --git a/PpomppuNotifier_ERD.puml b/PpomppuNotifier_ERD.puml new file mode 100644 index 0000000..c8f4e8b --- /dev/null +++ b/PpomppuNotifier_ERD.puml @@ -0,0 +1,54 @@ +@startuml +'https://plantuml.com/sequence-diagram + +class SubscribedUser { +- id ++ user_id ++ registered_at ++ created_at ++ modified_at +} + +class SubscribedBoard { +- id +- user_id ++ board_name ++ created_at ++ modified_at +} + +class PublishedHisotry { +- id ++ user_id ++ board_name_list ++ published_at ++ created_at ++ modified_at +} + +class PpomppuArticle { +- id ++ article_id ++ board_name ++ article_url ++ title ++ recommended ++ hit ++ registered_at ++ created_at ++ modified_at +} + +class PpomppuBoardFeedStatus { +- id ++ board_name ++ latest_parsed_article_id ++ updated_at ++ created_at ++ modified_at +} + +SubscribedUser --o{ SubscribedBoard +SubscribedUser --o{ PublishedHisotry + +@enduml \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..9ab58a2 --- /dev/null +++ b/README.md @@ -0,0 +1,2 @@ +# PpomppuNotifier + diff --git a/build.gradle b/build.gradle new file mode 100644 index 0000000..118203c --- /dev/null +++ b/build.gradle @@ -0,0 +1,70 @@ +plugins { + id 'java' + id 'idea' + id 'org.springframework.boot' version '2.5.4' + id 'io.spring.dependency-management' version '1.0.11.RELEASE' +} + +group = 'com.myoa.engineering.crawl.shopping' +version = '1.1.1' +sourceCompatibility = JavaVersion.VERSION_17 + +configurations { + compileOnly { + extendsFrom annotationProcessor + } +} + +repositories { + mavenCentral() +} + +allprojects { + group = 'com.myoa.engineering.crawl.shopping' + version = '1.1.1' + + apply plugin: 'java' + apply plugin: 'idea' + apply plugin: 'org.springframework.boot' + apply plugin: 'io.spring.dependency-management' + + repositories { + mavenCentral() + maven { + url "https://plugins.gradle.org/m2/" + } + } + + ext { + set('springCloudVersion', "2020.0.4") + } + + dependencyManagement { + imports { + mavenBom "org.springframework.cloud:spring-cloud-dependencies:${springCloudVersion}" + } + } + + +/* task initSourceFolders { + sourceSets*.java.srcDirs*.each { + if( !it.exists() ) { + it.mkdirs() + } + } + + sourceSets*.resources.srcDirs*.each { + if( !it.exists() ) { + it.mkdirs() + } + } + }*/ +} + +jar { + enabled = false +} + +bootJar { + enabled = false +} \ No newline at end of file diff --git a/copy.bat b/copy.bat new file mode 100644 index 0000000..23f8003 --- /dev/null +++ b/copy.bat @@ -0,0 +1,3 @@ +xcopy /y .\processor\build\libs\*.jar .\ +xcopy /y .\receiver\build\libs\*.jar .\ +xcopy /y .\sender\build\libs\*.jar .\ \ No newline at end of file diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 0000000..7454180 Binary files /dev/null and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 0000000..6677206 --- /dev/null +++ b/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,6 @@ +#Sun Apr 28 23:47:38 KST 2024 +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-8.5-bin.zip +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/gradlew b/gradlew new file mode 100755 index 0000000..744e882 --- /dev/null +++ b/gradlew @@ -0,0 +1,185 @@ +#!/usr/bin/env sh + +# +# Copyright 2015 the original author or authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +############################################################################## +## +## Gradle start up script for UN*X +## +############################################################################## + +# Attempt to set APP_HOME +# Resolve links: $0 may be a link +PRG="$0" +# Need this for relative symlinks. +while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "$PRG"`"/$link" + fi +done +SAVED="`pwd`" +cd "`dirname \"$PRG\"`/" >/dev/null +APP_HOME="`pwd -P`" +cd "$SAVED" >/dev/null + +APP_NAME="Gradle" +APP_BASE_NAME=`basename "$0"` + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD="maximum" + +warn () { + echo "$*" +} + +die () { + echo + echo "$*" + echo + exit 1 +} + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "`uname`" in + CYGWIN* ) + cygwin=true + ;; + Darwin* ) + darwin=true + ;; + MSYS* | MINGW* ) + msys=true + ;; + NONSTOP* ) + nonstop=true + ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD="java" + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then + MAX_FD_LIMIT=`ulimit -H -n` + if [ $? -eq 0 ] ; then + if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then + MAX_FD="$MAX_FD_LIMIT" + fi + ulimit -n $MAX_FD + if [ $? -ne 0 ] ; then + warn "Could not set maximum file descriptor limit: $MAX_FD" + fi + else + warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" + fi +fi + +# For Darwin, add options to specify how the application appears in the dock +if $darwin; then + GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" +fi + +# For Cygwin or MSYS, switch paths to Windows format before running java +if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then + APP_HOME=`cygpath --path --mixed "$APP_HOME"` + CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` + + JAVACMD=`cygpath --unix "$JAVACMD"` + + # We build the pattern for arguments to be converted via cygpath + ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` + SEP="" + for dir in $ROOTDIRSRAW ; do + ROOTDIRS="$ROOTDIRS$SEP$dir" + SEP="|" + done + OURCYGPATTERN="(^($ROOTDIRS))" + # Add a user-defined pattern to the cygpath arguments + if [ "$GRADLE_CYGPATTERN" != "" ] ; then + OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" + fi + # Now convert the arguments - kludge to limit ourselves to /bin/sh + i=0 + for arg in "$@" ; do + CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` + CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option + + if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition + eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` + else + eval `echo args$i`="\"$arg\"" + fi + i=`expr $i + 1` + done + case $i in + 0) set -- ;; + 1) set -- "$args0" ;; + 2) set -- "$args0" "$args1" ;; + 3) set -- "$args0" "$args1" "$args2" ;; + 4) set -- "$args0" "$args1" "$args2" "$args3" ;; + 5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; + 6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; + 7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; + 8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; + 9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; + esac +fi + +# Escape application args +save () { + for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done + echo " " +} +APP_ARGS=`save "$@"` + +# Collect all arguments for the java command, following the shell quoting and substitution rules +eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" + +exec "$JAVACMD" "$@" diff --git a/gradlew.bat b/gradlew.bat new file mode 100644 index 0000000..107acd3 --- /dev/null +++ b/gradlew.bat @@ -0,0 +1,89 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@if "%DEBUG%" == "" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%" == "" set DIRNAME=. +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if "%ERRORLEVEL%" == "0" goto execute + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto execute + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* + +:end +@rem End local scope for the variables with windows NT shell +if "%ERRORLEVEL%"=="0" goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 +exit /b 1 + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/settings.gradle b/settings.gradle new file mode 100644 index 0000000..ab70af1 --- /dev/null +++ b/settings.gradle @@ -0,0 +1,14 @@ +rootProject.name = 'PpomppuNotifier' + +//include 'reader' +//project(':reader').projectDir = "$rootDir/reader" as File +//include 'processor' +//project(':processor').projectDir = "$rootDir/processor" as File +//include 'writer' +//project(':writer').projectDir = "$rootDir/writer" as File + +include 'shopping-crawler' +project(':shopping-crawler').projectDir = "$rootDir/shopping-crawler" as File + +include 'support' +project(':support').projectDir = "$rootDir/support" as File diff --git a/shopping-crawler/.jqwik-database b/shopping-crawler/.jqwik-database new file mode 100644 index 0000000..711006c Binary files /dev/null and b/shopping-crawler/.jqwik-database differ diff --git a/shopping-crawler/build.gradle b/shopping-crawler/build.gradle new file mode 100644 index 0000000..1aa8ad3 --- /dev/null +++ b/shopping-crawler/build.gradle @@ -0,0 +1,41 @@ +dependencies { + developmentOnly 'org.springframework.boot:spring-boot-devtools' + runtimeOnly 'com.h2database:h2' + runtimeOnly 'com.mysql:mysql-connector-j:8.4.0' + compileOnly 'org.projectlombok:lombok' + + implementation project(':support') + // https://projectreactor.io/docs/core/release/reference/#debug-activate + implementation("org.springframework.boot:spring-boot-starter-web") { + exclude group: "org.springframework.boot", module: "spring-boot-starter-tomcat" + } + + implementation("org.springframework.boot:spring-boot-starter-undertow") { + exclude group: "io.undertow", module: "undertow-websockets-jsr" + } + implementation 'org.springframework.boot:spring-boot-starter-data-jpa' + implementation 'org.springframework.boot:spring-boot-configuration-processor' + implementation 'org.springframework.cloud:spring-cloud-starter-config' + implementation 'org.springframework.boot:spring-boot-starter-actuator' + implementation 'org.jsoup:jsoup:1.17.2' + implementation 'com.h2database:h2:2.2.224' + implementation "org.springframework.cloud:spring-cloud-starter-openfeign" + implementation "io.github.openfeign:feign-hc5" + implementation 'org.ahocorasick:ahocorasick:0.6.3' + + annotationProcessor 'org.springframework.boot:spring-boot-configuration-processor' + annotationProcessor 'org.projectlombok:lombok' + + testImplementation 'org.springframework.boot:spring-boot-starter-test' + testImplementation('org.assertj:assertj-core:3.25.3') + testImplementation("org.jeasy:easy-random-core:5.0.0") + testCompileOnly 'org.projectlombok:lombok' + testAnnotationProcessor('org.projectlombok:lombok') +} + +test { + useJUnitPlatform() + testLogging { + events "passed", "skipped", "failed" + } +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/ShoppingCrawlerApplication.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/ShoppingCrawlerApplication.java new file mode 100644 index 0000000..71444d8 --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/ShoppingCrawlerApplication.java @@ -0,0 +1,19 @@ +package com.myoa.engineering.crawl.shopping; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.cloud.openfeign.EnableFeignClients; +import org.springframework.scheduling.annotation.EnableScheduling; + +@EnableScheduling +@EnableFeignClients +@EnableConfigurationProperties +@SpringBootApplication +public class ShoppingCrawlerApplication { + + public static void main(String[] args) { + SpringApplication.run(ShoppingCrawlerApplication.class, args); + } + +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/FeignDefaultConfig.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/FeignDefaultConfig.java new file mode 100644 index 0000000..f01a4d4 --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/FeignDefaultConfig.java @@ -0,0 +1,30 @@ +package com.myoa.engineering.crawl.shopping.configuration; + +import feign.Logger; +import feign.RequestInterceptor; +import feign.codec.ErrorDecoder; +import lombok.extern.slf4j.Slf4j; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; + +@Slf4j +@Configuration +public class FeignDefaultConfig { + + @Bean + Logger.Level feignLoggerLevel() { + return Logger.Level.FULL; + } + + + public static final String MIME_TYPE = + MediaType.APPLICATION_JSON_VALUE + ";charset=utf-8"; + + @Bean + public RequestInterceptor requestInterceptor() { + return requestTemplate -> requestTemplate.header(HttpHeaders.CONTENT_TYPE, MIME_TYPE); + } + +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/datasource/H2ConsoleConfiguration.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/datasource/H2ConsoleConfiguration.java new file mode 100644 index 0000000..9f9ab8b --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/datasource/H2ConsoleConfiguration.java @@ -0,0 +1,36 @@ +package com.myoa.engineering.crawl.shopping.configuration.datasource; + +import lombok.extern.slf4j.Slf4j; +import org.h2.tools.Server; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Profile; +import org.springframework.context.event.ContextClosedEvent; +import org.springframework.context.event.ContextRefreshedEvent; +import org.springframework.context.event.EventListener; + +import java.sql.SQLException; + +@Slf4j +@Profile({"datasource-local", "datasource-development"}) +@Configuration +public class H2ConsoleConfiguration { + + private Server webServer; + + @Value("${spring.h2.console.port}") + private String port; + + @EventListener(ContextRefreshedEvent.class) + public void start() throws SQLException { + log.info("starting h2 console"); + this.webServer = Server.createWebServer("-webPort", port, "-tcpAllowOthers").start(); + } + + @EventListener(ContextClosedEvent.class) + public void stop() { + log.info("stopping h2 console"); + this.webServer.stop(); ; + } + +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/datasource/ShoppingCrawlerDatasourceConfiguration.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/datasource/ShoppingCrawlerDatasourceConfiguration.java new file mode 100644 index 0000000..6c5afe3 --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/datasource/ShoppingCrawlerDatasourceConfiguration.java @@ -0,0 +1,126 @@ +package com.myoa.engineering.crawl.shopping.configuration.datasource; + +import com.myoa.engineering.crawl.shopping.configuration.datasource.properties.DatasourceProperties; +import com.myoa.engineering.crawl.shopping.configuration.datasource.properties.HibernateProperties; +import com.myoa.engineering.crawl.shopping.configuration.datasource.properties.HikariProperties; +import com.myoa.engineering.crawl.shopping.domain.entity.BaseScanDomain; +import com.myoa.engineering.crawl.shopping.infra.repository.BaseScanRepository; +import com.zaxxer.hikari.HikariConfig; +import com.zaxxer.hikari.HikariDataSource; +import lombok.NonNull; +import org.hibernate.cfg.AvailableSettings; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.orm.jpa.EntityManagerFactoryBuilder; +import org.springframework.boot.orm.jpa.hibernate.SpringImplicitNamingStrategy; +import org.springframework.boot.orm.jpa.hibernate.SpringPhysicalNamingStrategy; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.data.jpa.repository.config.EnableJpaAuditing; +import org.springframework.data.jpa.repository.config.EnableJpaRepositories; +import org.springframework.orm.jpa.JpaTransactionManager; +import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean; +import org.springframework.transaction.PlatformTransactionManager; + +import javax.persistence.EntityManagerFactory; +import javax.sql.DataSource; +import java.util.Enumeration; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; + +@Configuration +@EnableJpaAuditing +@EnableJpaRepositories(basePackageClasses = BaseScanRepository.class, + entityManagerFactoryRef = "shoppingCrawlerEntityManagerFactory", + transactionManagerRef = "shoppingCrawlerTransactionManager" +) +public class ShoppingCrawlerDatasourceConfiguration { + + private static final String DATA_SOURCE_UNIT_NAME = "crawler-shopping"; + + private final DatasourceProperties dataSourceProeprties; + private final HikariProperties hikariProperties; + private final HibernateProperties hibernateProperties; + + public ShoppingCrawlerDatasourceConfiguration(DatasourceProperties dataSourceProeprties, + HikariProperties hikariProperties, + HibernateProperties hibernateProperties) { + this.dataSourceProeprties = dataSourceProeprties; + this.hikariProperties = hikariProperties; + this.hibernateProperties = hibernateProperties; + } + + @Bean(name = "shoppingCrawlerDataSource") + public DataSource dataSource() { + DatasourceProperties.DataSourcePropertiesUnit dataSourcePropertiesUnit = dataSourceProeprties.find(DATA_SOURCE_UNIT_NAME); + + final HikariConfig hikariConfig = new HikariConfig(); + hikariConfig.setJdbcUrl(dataSourcePropertiesUnit.toCompletedJdbcUrl()); + hikariConfig.setDriverClassName(dataSourcePropertiesUnit.getDriverClassName()); + hikariConfig.setUsername(dataSourcePropertiesUnit.getUsername()); + hikariConfig.setPassword(dataSourcePropertiesUnit.getPassword()); + hikariConfig.setAutoCommit(hikariProperties.getAutoCommit()); + hikariConfig.setMaximumPoolSize(hikariProperties.getMaximumPoolSize()); + hikariConfig.setMinimumIdle(hikariProperties.getMinimumIdle()); + if (hikariProperties.getMaximumPoolSize() > hikariProperties.getMinimumIdle()) { + hikariConfig.setIdleTimeout(hikariProperties.getIdleTimeout()); + } + hikariConfig.setValidationTimeout(hikariProperties.getValidationTimeout()); + hikariConfig.setConnectionTimeout(hikariProperties.getConnectionTimeout()); + hikariConfig.setMaxLifetime(hikariProperties.getMaxLifetime()); + + final DataSource dataSource = new HikariDataSource(hikariConfig); + return dataSource; + } + + @Bean("shoppingCrawlerEntityManagerFactory") + public LocalContainerEntityManagerFactoryBean entityManagerFactory( + EntityManagerFactoryBuilder builder, + @Qualifier("shoppingCrawlerDataSource") DataSource dataSource) { + return builder.dataSource(dataSource) + .packages(BaseScanDomain.class) + .properties(getPropsMap(DATA_SOURCE_UNIT_NAME)) + .build(); + } + + @Bean("shoppingCrawlerTransactionManager") + public PlatformTransactionManager transactionManager( + @Qualifier("shoppingCrawlerEntityManagerFactory") EntityManagerFactory entityManagerFactory) { + return new JpaTransactionManager(entityManagerFactory); + } + + public static Properties getProps(@NonNull HibernateProperties.HibernatePropertiesUnit hibernateProperties) { + Properties properties = new Properties(); + properties.put(AvailableSettings.DIALECT, hibernateProperties.getDialect()); + properties.put(AvailableSettings.FORMAT_SQL, hibernateProperties.getFormatSql()); + properties.put(AvailableSettings.SHOW_SQL, hibernateProperties.getShowSql()); + properties.put(AvailableSettings.HBM2DDL_AUTO, hibernateProperties.getHbm2ddlAuto()); + properties.put(AvailableSettings.CONNECTION_PROVIDER_DISABLES_AUTOCOMMIT, hibernateProperties.getDisableAutoCommit()); + properties.put(AvailableSettings.IMPLICIT_NAMING_STRATEGY, SpringImplicitNamingStrategy.class.getName()); + properties.put(AvailableSettings.PHYSICAL_NAMING_STRATEGY, SpringPhysicalNamingStrategy.class.getName()); + properties.put(AvailableSettings.GENERATE_STATISTICS, "false"); +// properties.put(AvailableSettings.GLOBALLY_QUOTED_IDENTIFIERS, "true"); +// properties.put(AvailableSettings.GLOBALLY_QUOTED_IDENTIFIERS_SKIP_COLUMN_DEFINITIONS, "true"); + properties.put(AvailableSettings.STATEMENT_BATCH_SIZE, "20"); + properties.put(AvailableSettings.ORDER_INSERTS, "true"); + properties.put(AvailableSettings.ORDER_UPDATES, "true"); + properties.put(AvailableSettings.BATCH_VERSIONED_DATA, "true"); +// properties.put(AvailableSettings.JPA_ID_GENERATOR_GLOBAL_SCOPE_COMPLIANCE, "false"); + return properties; + } + + public Map getPropsMap(@NonNull String unitName) { + return convertPropertiestoMaps(getProps(hibernateProperties.find(unitName))); + } + + public Map convertPropertiestoMaps(Properties properties) { + Map propertiesMap = new HashMap<>(); + + for (Enumeration e = properties.propertyNames(); e.hasMoreElements(); ) { + String key = (String) e.nextElement(); + propertiesMap.put(key, properties.getProperty(key)); + } + return propertiesMap; + } + +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/datasource/properties/DatasourceProperties.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/datasource/properties/DatasourceProperties.java new file mode 100644 index 0000000..3a1e583 --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/datasource/properties/DatasourceProperties.java @@ -0,0 +1,45 @@ +package com.myoa.engineering.crawl.shopping.configuration.datasource.properties; + +import com.myoa.engineering.crawl.shopping.support.util.ObjectUtil; +import lombok.Data; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +import java.util.List; + +@Component +@Data +@ConfigurationProperties(prefix = "datasource") +public class DatasourceProperties { + + private List units; + + @Data + public static class DataSourcePropertiesUnit { + + private String unitName; + private String schemaName; + private String connectionParameters; + private String dbConnectionUrl; + private Boolean isSimpleConnectionUrl; + private String username; + private String password; + private String driverClassName; + + public String toCompletedJdbcUrl() { + if (ObjectUtil.isEmpty(isSimpleConnectionUrl) || isSimpleConnectionUrl == false) { + return String.format("%s/%s?%s", dbConnectionUrl, schemaName, connectionParameters); + } + return dbConnectionUrl; + } + } + + public DataSourcePropertiesUnit find(String unitName) { + return units.stream() + .filter(e -> e.getUnitName().equals(unitName)) + .findFirst() + .orElseThrow( + () -> new IllegalArgumentException(this.getClass().getName() + ": unitName Not found. " + unitName)); + } + +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/datasource/properties/HibernateProperties.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/datasource/properties/HibernateProperties.java new file mode 100644 index 0000000..cbdfa49 --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/datasource/properties/HibernateProperties.java @@ -0,0 +1,39 @@ +package com.myoa.engineering.crawl.shopping.configuration.datasource.properties; + +import lombok.Getter; +import lombok.Setter; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +import java.util.List; + +@Component +@Setter +@Getter +@ConfigurationProperties(prefix = "hibernate") +public class HibernateProperties { + + private List units; + + @Getter + @Setter + public static class HibernatePropertiesUnit { + + private String unitName; + private String dialect; + private String formatSql; + private String showSql; + private String hbm2ddlAuto; + private String disableAutoCommit; + + } + + public HibernatePropertiesUnit find(String unitName) { + return units.stream() + .filter(x -> x.getUnitName().equals(unitName)) + .findFirst() + .orElseThrow( + () -> new IllegalArgumentException(this.getClass().getName() + ": unitName Not found. " + unitName)); + } + +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/datasource/properties/HikariProperties.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/datasource/properties/HikariProperties.java new file mode 100644 index 0000000..47516db --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/datasource/properties/HikariProperties.java @@ -0,0 +1,22 @@ +package com.myoa.engineering.crawl.shopping.configuration.datasource.properties; + +import lombok.Getter; +import lombok.Setter; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +@Component +@Setter +@Getter +@ConfigurationProperties(prefix = "spring.datasource.hikari") +public class HikariProperties { + + private Integer minimumIdle; + private Integer maximumPoolSize; + private Integer idleTimeout; + private Integer validationTimeout; + private Integer connectionTimeout; + private Integer maxLifetime; + private Boolean autoCommit; + +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/slack/properties/SlackSecretProperties.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/slack/properties/SlackSecretProperties.java new file mode 100644 index 0000000..691f141 --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/configuration/slack/properties/SlackSecretProperties.java @@ -0,0 +1,32 @@ +package com.myoa.engineering.crawl.shopping.configuration.slack.properties; + +import java.util.List; +import lombok.Data; +import lombok.Getter; +import lombok.Setter; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +@Data +@Component +@ConfigurationProperties(prefix = "slack.bot") +public class SlackSecretProperties { + + private List units; + + @Data + public static class SlackSecretPropertiesUnit { + private String botUnitName; + private String username; + private String iconEmoji; + private String channel; + private String token; + } + + public SlackSecretPropertiesUnit find(String botUnitName) { + return units.stream() + .filter(e -> e.getBotUnitName().equals(botUnitName)) + .findFirst() + .orElseThrow(() -> new IllegalArgumentException("not found bot unit name : " + botUnitName)); + } +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/controller/TestAPIController.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/controller/TestAPIController.java new file mode 100644 index 0000000..7992a38 --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/controller/TestAPIController.java @@ -0,0 +1,22 @@ +package com.myoa.engineering.crawl.shopping.controller; + +import com.myoa.engineering.crawl.shopping.crawlhandler.PpomppuCrawlDomesticHandler; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +@RestController +@RequestMapping("/api/v1/exploit") +public class TestAPIController { + + private final PpomppuCrawlDomesticHandler ppomppuCrawlDomesticHandler; + + public TestAPIController(PpomppuCrawlDomesticHandler ppomppuCrawlDomesticHandler) { + this.ppomppuCrawlDomesticHandler = ppomppuCrawlDomesticHandler; + } + + @GetMapping("/triggers") + public void triggerExploit() { + ppomppuCrawlDomesticHandler.handle(); + } +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/crawlhandler/CrawlHandler.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/crawlhandler/CrawlHandler.java new file mode 100644 index 0000000..8b60ff0 --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/crawlhandler/CrawlHandler.java @@ -0,0 +1,10 @@ +package com.myoa.engineering.crawl.shopping.crawlhandler; + +import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget; + +public interface CrawlHandler { + + CrawlTarget getCrawlTarget(); + + void handle(); +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/crawlhandler/FmkoreaCrawlHandler.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/crawlhandler/FmkoreaCrawlHandler.java new file mode 100644 index 0000000..bfb6938 --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/crawlhandler/FmkoreaCrawlHandler.java @@ -0,0 +1,18 @@ +package com.myoa.engineering.crawl.shopping.crawlhandler; + +import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; + +@Slf4j +@Component +public class FmkoreaCrawlHandler implements CrawlHandler { + @Override + public CrawlTarget getCrawlTarget() { + return CrawlTarget.FMKOREA; + } + + @Override + public void handle() { + } +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/crawlhandler/PpomppuCrawlDomesticHandler.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/crawlhandler/PpomppuCrawlDomesticHandler.java new file mode 100644 index 0000000..541c969 --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/crawlhandler/PpomppuCrawlDomesticHandler.java @@ -0,0 +1,58 @@ +package com.myoa.engineering.crawl.shopping.crawlhandler; + +import com.myoa.engineering.crawl.shopping.crawlhandler.parser.PpomppuArticleParserV2; +import com.myoa.engineering.crawl.shopping.domain.entity.v2.Article; +import com.myoa.engineering.crawl.shopping.infra.client.ppomppu.PpomppuBoardClientV2; +import com.myoa.engineering.crawl.shopping.service.ArticleCommandService; +import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget; +import org.springframework.stereotype.Component; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Stream; + +@Component +public class PpomppuCrawlDomesticHandler implements CrawlHandler { + + private final PpomppuBoardClientV2 ppomppuBoardClient; + private final PpomppuArticleParserV2 ppomppuArticleParserV2; + private final ArticleCommandService articleCommandService; + + public PpomppuCrawlDomesticHandler(PpomppuBoardClientV2 ppomppuBoardClient, + PpomppuArticleParserV2 ppomppuArticleParserV2, + ArticleCommandService articleCommandService) { + this.ppomppuBoardClient = ppomppuBoardClient; + this.ppomppuArticleParserV2 = ppomppuArticleParserV2; + this.articleCommandService = articleCommandService; + } + + @Override + public CrawlTarget getCrawlTarget() { + return CrawlTarget.PPOMPPU_DOMESTIC; + } + + @Override + public void handle() { + + String boardHtmlPage1 = ppomppuBoardClient.getBoardHtml("/zboard/zboard.php", generateRequestParams(1)); + List
parsedPage1 = ppomppuArticleParserV2.parse(boardHtmlPage1); + + String boardHtmlPage2 = ppomppuBoardClient.getBoardHtml("/zboard/zboard.php", generateRequestParams(2)); + List
parsedPage2 = ppomppuArticleParserV2.parse(boardHtmlPage2); + + List
merged = Stream.of(parsedPage1, parsedPage2) + .flatMap(List::stream) + .map(e -> e.updateCrawlTarget(getCrawlTarget())) + .toList(); + + articleCommandService.upsert(merged); + } + + private Map generateRequestParams(int pageId) { + Map params = new HashMap<>(); + params.put("id", "ppomppu"); + params.put("page", String.valueOf(pageId)); + return params; + } +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/crawlhandler/parser/PpomppuArticleParserV2.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/crawlhandler/parser/PpomppuArticleParserV2.java new file mode 100644 index 0000000..850f8a1 --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/crawlhandler/parser/PpomppuArticleParserV2.java @@ -0,0 +1,115 @@ +package com.myoa.engineering.crawl.shopping.crawlhandler.parser; + +import com.myoa.engineering.crawl.shopping.domain.entity.v2.Article; +import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName; +import com.myoa.engineering.crawl.shopping.util.DateTimeUtils; +import com.myoa.engineering.crawl.shopping.util.NumberUtils; +import io.micrometer.core.instrument.util.StringUtils; +import lombok.extern.slf4j.Slf4j; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; +import org.springframework.stereotype.Component; + +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +@Slf4j +@Component +public final class PpomppuArticleParserV2 { + + private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yy.MM.dd HH:mm:ss") + .withZone(ZoneId.of("Asia/Seoul")); + + private PpomppuArticleParserV2() { + } + + public List
parse(String html) { + Elements trElements = converHtmlToTrElements(html); + return trElements.stream() + .filter(this::isRealArticle) + .map(this::parse) + .toList(); + + } + + private Elements converHtmlToTrElements(String data) { + Document document = Jsoup.parse(data); + Elements trList = document.getElementById("revolution_main_table").getElementsByTag("tr"); + return trList; + } + + private boolean isRealArticle(Element tr) { + Elements tdList = tr.getElementsByTag("td"); + if (tdList.size() != 6) { + return false; + } + + if (!hasOnlyNumeric(tdList.get(0))) { + return false; + } + + return true; + } + + private Pattern pattern_numeric = Pattern.compile("\\d+"); + + private boolean hasOnlyNumeric(Element td) { + return pattern_numeric.matcher(td.text()).matches(); + } + + private Article parse(Element tr) { + Elements tdList = tr.getElementsByTag("td"); + Long articleId = Long.parseLong(tdList.get(0).text()); + + String title = tdList.get(2).text(); + String articleUrl = parseArticleUrl(tdList.get(2).getElementsByTag("a").attr("href")); + String boardName = parseBoardName(title); + Integer recommended = parseRecommended(tdList.get(4)); + Integer hit = NumberUtils.parseInt(tdList.get(5).text(), 0); + ZonedDateTime registeredAt = DateTimeUtils.parse(tdList.get(3).text()); + + return Article.builder() + .articleId(articleId) + .title(title) + .boardName(boardName) + .articleUrl(articleUrl) + .recommended(recommended) + .hit(hit) + .registeredAt(registeredAt) + .build(); + } + + public Integer parseRecommended(Element td) { + final String voteString = td.text(); + if (StringUtils.isEmpty(voteString)) { + return null; + } + + final int voteUp = Integer.parseInt(td.text().split(" - ")[0]); + final int voteDown = Integer.parseInt(td.text().split(" - ")[1]); + int recommended = voteUp - voteDown; + return recommended; + } + + public static String parseArticleUrl(String data) { + return PpomppuBoardName.ofViewPageUrl(data); + } + + Pattern patternBoardName = Pattern.compile("\\[(.+?)\\]"); + + public String parseBoardName(String fullTitle) { + Matcher matcher = patternBoardName.matcher(fullTitle); + String lastMatched = null; + while (matcher.find()) { + lastMatched = matcher.group(1); + } + return lastMatched; + } + +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/Auditable.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/Auditable.java new file mode 100644 index 0000000..59b51a4 --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/Auditable.java @@ -0,0 +1,28 @@ +package com.myoa.engineering.crawl.shopping.domain.entity; + +import lombok.Getter; +import org.springframework.data.annotation.CreatedDate; +import org.springframework.data.annotation.LastModifiedDate; +import org.springframework.data.jpa.domain.support.AuditingEntityListener; + +import javax.persistence.Column; +import javax.persistence.EntityListeners; +import javax.persistence.MappedSuperclass; +import java.io.Serializable; +import java.time.Instant; + +@Getter +@MappedSuperclass +@EntityListeners(AuditingEntityListener.class) +public abstract class Auditable implements Serializable { + private static final long serialVersionUID = -7105030870015828551L; + + @Column + @CreatedDate + private Instant createdAt; + + @Column + @LastModifiedDate + private Instant modifiedAt; + +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/BaseScanDomain.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/BaseScanDomain.java new file mode 100644 index 0000000..bddd3dd --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/BaseScanDomain.java @@ -0,0 +1,10 @@ +package com.myoa.engineering.crawl.shopping.domain.entity; + +/** + * BaseScanDomain + * + * @author Shin Woo-jin (woozu.shin@kakaoent.com) + * @since 2021-09-08 + */ +public interface BaseScanDomain { +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v1/PpomppuArticle.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v1/PpomppuArticle.java new file mode 100644 index 0000000..e7dee7c --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v1/PpomppuArticle.java @@ -0,0 +1,66 @@ +package com.myoa.engineering.crawl.shopping.domain.entity.v1; + +import com.myoa.engineering.crawl.shopping.domain.entity.Auditable; +import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName; +import lombok.Builder; +import lombok.Getter; +import lombok.NoArgsConstructor; + +import javax.persistence.*; +import java.time.Instant; + +@Getter +@NoArgsConstructor +@Entity +@Table(name = "ppomppu_article") +public class PpomppuArticle extends Auditable { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + @Column + private Long articleId; + + @Column + @Enumerated(EnumType.STRING) + private PpomppuBoardName boardName; + + @Column + private String articleUrl; + + @Column + private String thumbnailUrl; + + @Column + private String title; + + @Column + private Integer hit; + + @Column + private Integer recommended; + + @Column + private Instant registeredAt; + + @Builder + public PpomppuArticle(Long id, Long articleId, PpomppuBoardName boardName, String articleUrl, + String thumbnailUrl, String title, Integer recommended, Integer hit, + Instant registeredAt) { + this.id = id; + this.articleId = articleId; + this.boardName = boardName; + this.articleUrl = articleUrl; + this.thumbnailUrl = thumbnailUrl; + this.title = title; + this.recommended = recommended; + this.hit = hit; + this.registeredAt = registeredAt; + } + + public PpomppuArticle updateBoardName(PpomppuBoardName boardName) { + this.boardName = boardName; + return this; + } +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v1/PpomppuBoardFeedStatus.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v1/PpomppuBoardFeedStatus.java new file mode 100644 index 0000000..18ec466 --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v1/PpomppuBoardFeedStatus.java @@ -0,0 +1,52 @@ +package com.myoa.engineering.crawl.shopping.domain.entity.v1; + +import com.myoa.engineering.crawl.shopping.domain.entity.Auditable; +import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName; +import lombok.Builder; +import lombok.Getter; +import lombok.NoArgsConstructor; + +import javax.persistence.*; +import java.time.Instant; + +@Getter +@NoArgsConstructor +@Entity +@Table(name = "ppomppu_board_feed_status") +public class PpomppuBoardFeedStatus extends Auditable { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + @Column + private Long latestParsedArticleId; + + @Column + @Enumerated(EnumType.STRING) + private PpomppuBoardName boardName; + + @Column + private Instant updatedAt; + + public static PpomppuBoardFeedStatus of(PpomppuBoardName boardName, Long latestArticleId) { + return PpomppuBoardFeedStatus.builder() + .boardName(boardName) + .latestParsedArticleId(latestArticleId) + .updatedAt(Instant.now()) + .build(); + } + + public void updateArticleId(Long latestArticleId) { + this.updatedAt = Instant.now(); + this.latestParsedArticleId = latestArticleId; + } + + @Builder + public PpomppuBoardFeedStatus(Long id, Long latestParsedArticleId, PpomppuBoardName boardName, Instant updatedAt) { + this.id = id; + this.latestParsedArticleId = latestParsedArticleId; + this.boardName = boardName; + this.updatedAt = updatedAt; + } +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v1/PublishedHistory.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v1/PublishedHistory.java new file mode 100644 index 0000000..d821fbf --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v1/PublishedHistory.java @@ -0,0 +1,29 @@ +package com.myoa.engineering.crawl.shopping.domain.entity.v1; + +import com.myoa.engineering.crawl.shopping.domain.entity.Auditable; +import lombok.Getter; +import lombok.NoArgsConstructor; + +import javax.persistence.*; +import java.time.Instant; + +@Getter +@NoArgsConstructor +@Entity +@Table(name = "published_history") +public class PublishedHistory extends Auditable { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + @Column + private Long userId; + + @Column + private String boardNameList; + + @Column + private Instant publishedAt; + +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v1/SubscribedBoard.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v1/SubscribedBoard.java new file mode 100644 index 0000000..0ed69da --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v1/SubscribedBoard.java @@ -0,0 +1,27 @@ +package com.myoa.engineering.crawl.shopping.domain.entity.v1; + +import com.myoa.engineering.crawl.shopping.domain.entity.Auditable; +import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName; +import lombok.Getter; +import lombok.NoArgsConstructor; + +import javax.persistence.*; + +@Getter +@NoArgsConstructor +@Entity +@Table(name = "subscribed_board") +public class SubscribedBoard extends Auditable { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + @Column + private Long userId; + + @Column + @Enumerated(EnumType.STRING) + private PpomppuBoardName boardName; + +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v1/SubscribedUser.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v1/SubscribedUser.java new file mode 100644 index 0000000..12e7669 --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v1/SubscribedUser.java @@ -0,0 +1,26 @@ +package com.myoa.engineering.crawl.shopping.domain.entity.v1; + +import com.myoa.engineering.crawl.shopping.domain.entity.Auditable; +import lombok.Getter; +import lombok.NoArgsConstructor; + +import javax.persistence.*; +import java.time.Instant; + +@Getter +@NoArgsConstructor +@Entity +@Table(name = "subscribed_user") +public class SubscribedUser extends Auditable { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + @Column + private Long userId; + + @Column + private Instant registeredAt; + +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v2/AppUser.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v2/AppUser.java new file mode 100644 index 0000000..a5d0959 --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v2/AppUser.java @@ -0,0 +1,29 @@ +package com.myoa.engineering.crawl.shopping.domain.entity.v2; + +import com.myoa.engineering.crawl.shopping.domain.entity.Auditable; +import javax.persistence.*; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; +import lombok.NoArgsConstructor; + +@Getter +@Builder +@NoArgsConstructor +@AllArgsConstructor +@Entity +@Table +public class AppUser extends Auditable { + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + @Column + private String name; + + @Column + private String slackId; + + @Column + private Boolean enabled; +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v2/Article.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v2/Article.java new file mode 100644 index 0000000..4021435 --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v2/Article.java @@ -0,0 +1,63 @@ +package com.myoa.engineering.crawl.shopping.domain.entity.v2; + +import com.myoa.engineering.crawl.shopping.domain.entity.Auditable; +import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget; +import javax.persistence.*; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; +import lombok.NoArgsConstructor; + +import java.time.ZonedDateTime; + +@Getter +@Builder +@NoArgsConstructor +@AllArgsConstructor +@Entity +@Table +public class Article extends Auditable { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + @Column + private Long articleId; + + @Column + @Enumerated(EnumType.STRING) + private CrawlTarget crawlTarget; + + @Column + private String boardName; + + @Column + private String articleUrl; + + @Column + private String title; + + @Column + private Integer hit; + + @Column + private Integer recommended; + + @Column + private ZonedDateTime registeredAt; + + public Article update(Article article) { + this.boardName = article.getBoardName(); + this.articleUrl = article.getArticleUrl(); + this.title = article.getTitle(); + this.hit = article.getHit(); + this.recommended = article.getRecommended(); + return this; + } + + public Article updateCrawlTarget(CrawlTarget crawlTarget) { + this.crawlTarget = crawlTarget; + return this; + } +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v2/SubscribedKeyword.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v2/SubscribedKeyword.java new file mode 100644 index 0000000..b3b2969 --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/entity/v2/SubscribedKeyword.java @@ -0,0 +1,33 @@ +package com.myoa.engineering.crawl.shopping.domain.entity.v2; + +import com.myoa.engineering.crawl.shopping.domain.entity.Auditable; +import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget; +import javax.persistence.*; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; +import lombok.NoArgsConstructor; + +@Getter +@Builder +@NoArgsConstructor +@AllArgsConstructor +@Entity +@Table +public class SubscribedKeyword extends Auditable { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + @Column + private String keyword; + + @Column + @Enumerated(EnumType.STRING) + private CrawlTarget crawlTarget; + + @Column + private String userId; + +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/model/UserNotifyModel.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/model/UserNotifyModel.java new file mode 100644 index 0000000..36a261a --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/model/UserNotifyModel.java @@ -0,0 +1,24 @@ +package com.myoa.engineering.crawl.shopping.domain.model; + +import com.myoa.engineering.crawl.shopping.domain.model.v2.ArticleModel; +import lombok.*; + +import java.util.List; + + +@ToString +@Getter +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class UserNotifyModel { + private String userId; + private List articles; + + public static UserNotifyModel of(String userId, List articles) { + return UserNotifyModel.builder() + .userId(userId) + .articles(articles) + .build(); + } +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/model/v2/AppUserModel.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/model/v2/AppUserModel.java new file mode 100644 index 0000000..8c22c7d --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/model/v2/AppUserModel.java @@ -0,0 +1,26 @@ +package com.myoa.engineering.crawl.shopping.domain.model.v2; + +import com.myoa.engineering.crawl.shopping.domain.entity.v2.AppUser; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; + +@Getter +@Builder +@AllArgsConstructor +public class AppUserModel { + + private Long id; + private String name; + private String slackId; + private Boolean enabled; + + public static AppUserModel from(AppUser entity) { + return AppUserModel.builder() + .id(entity.getId()) + .name(entity.getName()) + .slackId(entity.getSlackId()) + .enabled(entity.getEnabled()) + .build(); + } +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/model/v2/ArticleModel.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/model/v2/ArticleModel.java new file mode 100644 index 0000000..ec7626b --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/model/v2/ArticleModel.java @@ -0,0 +1,25 @@ +package com.myoa.engineering.crawl.shopping.domain.model.v2; + +import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget; +import lombok.*; + +import java.time.ZonedDateTime; + +@ToString +@Getter +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class ArticleModel { + + private Long id; + private Long articleId; + private CrawlTarget crawlTarget; + private String boardName; + private String articleUrl; + private String title; + private Integer hit; + private Integer recommended; + private ZonedDateTime registeredAt; + +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/model/v2/SubscribedKeywordAggregatedModel.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/model/v2/SubscribedKeywordAggregatedModel.java new file mode 100644 index 0000000..a6fa5dd --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/domain/model/v2/SubscribedKeywordAggregatedModel.java @@ -0,0 +1,27 @@ +package com.myoa.engineering.crawl.shopping.domain.model.v2; + +import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget; +import com.myoa.engineering.crawl.shopping.util.AhoCorasickUtils; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; +import org.ahocorasick.trie.Trie; + +import java.util.List; + +@Getter +@Builder +@AllArgsConstructor +public class SubscribedKeywordAggregatedModel { + private final Trie ahoCorasickTrie; + private final String userId; + private final CrawlTarget crawlTarget; + + public static SubscribedKeywordAggregatedModel of(String userId, CrawlTarget crawlTarget, List keywords) { + return SubscribedKeywordAggregatedModel.builder() + .userId(userId) + .crawlTarget(crawlTarget) + .ahoCorasickTrie(AhoCorasickUtils.generateTrie(keywords)) + .build(); + } +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/FeedParsedResult.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/FeedParsedResult.java new file mode 100644 index 0000000..f169a3e --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/FeedParsedResult.java @@ -0,0 +1,46 @@ +package com.myoa.engineering.crawl.shopping.dto; + +import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName; +import lombok.Builder; +import lombok.Getter; +import lombok.NoArgsConstructor; + +import java.io.Serializable; +import java.time.Instant; + +/** + * FeedParsedResult + * + * @author Shin Woo-jin (woozu.shin@kakaoent.com) + * @since 2021-09-08 + */ +@Getter +@NoArgsConstructor +public class FeedParsedResult implements Serializable { + + private static final long serialVersionUID = -3771310078623481348L; + + private PpomppuBoardName boardName; + private Instant requestedAt; + private Instant processedAt; + + @Builder + public FeedParsedResult(PpomppuBoardName boardName, Instant requestedAt, Instant processedAt) { + this.boardName = boardName; + this.requestedAt = requestedAt; + this.processedAt = processedAt; + } + + public static FeedParsedResult of(PpomppuBoardName boardName) { + return FeedParsedResult.builder() + .boardName(boardName) + .requestedAt(Instant.now()) + .build(); + } + + public FeedParsedResult done() { + this.processedAt = Instant.now(); + return this; + } + +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/PpomppuArticleParseDTO.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/PpomppuArticleParseDTO.java new file mode 100644 index 0000000..497d746 --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/PpomppuArticleParseDTO.java @@ -0,0 +1,65 @@ +package com.myoa.engineering.crawl.shopping.dto; + +import com.myoa.engineering.crawl.shopping.domain.entity.v1.PpomppuArticle; +import com.myoa.engineering.crawl.shopping.support.util.DateUtil; +import lombok.Builder; +import lombok.Getter; +import lombok.NoArgsConstructor; + +import java.time.Instant; + +/** + * PpomppuArticleParseDTO + * + * @author Shin Woo-jin (woozu.shin@kakaoent.com) + * @since 2021-09-08 + */ +@Getter +@NoArgsConstructor +public class PpomppuArticleParseDTO { + + private String id; + private String articleId; + private String boardName; + private String articleUrl; + private String thumbnailUrl; + private String title; + private String hit; + private Integer recommended; + private String registeredAt; + + @Builder + public PpomppuArticleParseDTO(String id, String articleId, String boardName, String articleUrl, + String thumbnailUrl, String title, String hit, Integer recommended, + String registeredAt) { + this.id = id; + this.articleId = articleId; + this.boardName = boardName; + this.articleUrl = articleUrl; + this.thumbnailUrl = thumbnailUrl; + this.title = title; + this.hit = hit; + this.recommended = recommended; + this.registeredAt = registeredAt; + } + + public boolean isInValidated() { + return articleId == null || articleId.isEmpty() + || hit == null || hit.isEmpty(); + } + + public PpomppuArticle convert() { + if (isInValidated()) { + throw new IllegalArgumentException("PpomppuArticleParseDTO was invalidated"); + } + return PpomppuArticle.builder() + .articleId(Long.parseLong(articleId)) + .title(title) + .articleUrl(articleUrl) + .thumbnailUrl(thumbnailUrl) + .recommended(recommended) + .hit(Integer.parseInt(hit)) + .registeredAt(DateUtil.DATE_TIME_FORMATTER.parse(registeredAt, Instant::from)) + .build(); + } +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/PpomppuArticleTransformer.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/PpomppuArticleTransformer.java new file mode 100644 index 0000000..504b036 --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/PpomppuArticleTransformer.java @@ -0,0 +1,71 @@ +package com.myoa.engineering.crawl.shopping.dto; + +import com.myoa.engineering.crawl.shopping.domain.entity.v1.PpomppuArticle; +import com.myoa.engineering.crawl.shopping.support.dto.BlockMessageDTO; +import com.myoa.engineering.crawl.shopping.support.dto.SimpleMessageDTO; +import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName; +import com.myoa.engineering.crawl.shopping.support.util.DateUtil; + +import java.time.Instant; +import java.util.List; +import java.util.function.Function; +import java.util.stream.Collectors; + +/** + * PpomppuArticleTransformer + * + * @author Shin Woo-jin (woo-jin.shin@linecorp.com) + * @since 2021-11-21 + */ +public final class PpomppuArticleTransformer { + + private PpomppuArticleTransformer() { + } + + private static final String MESSAGE_FORMAT_V1 = "%s)) <%s|LINK> `%s` "; + private static final String MESSAGE_FORMAT_V2 = "%s *<%s|LINK>*\n%s"; + private static final String TITLE_FORMAT_V1 = "_*:hearts: %s | %s*_"; + + public static final Function TRANSFORM_TO_MESSAGE_DTO = article -> + SimpleMessageDTO.builder() + .requestedAt(Instant.now()) + .publishedAt(article.getRegisteredAt()) + .title(String.format(MESSAGE_FORMAT_V1, + article.getBoardName().getMenuName(), article.getArticleUrl(), + article.getTitle())) + .body(article.getArticleUrl()) + .build(); + + // https://stackoverflow.com/questions/24882927/using-streams-to-convert-a-list-of-objects-into-a-string-obtained-from-the-tostr + public static SimpleMessageDTO transformToSimpleMessage(List articles) { + Instant requestedAt = Instant.now(); + String body = articles.stream() + .map(PpomppuArticleTransformer::convertToInlineMessage) + .collect(Collectors.joining("\n\n")); + return SimpleMessageDTO.builder() + .requestedAt(requestedAt) + .title(DateUtil.DATE_TIME_FORMATTER.format(requestedAt)) + .body(body) + .build(); + } + + public static BlockMessageDTO transformToBlockMessage(PpomppuBoardName boardName, List articles) { + Instant requestedAt = Instant.now(); + List body = articles.stream() + .map(e -> BlockMessageDTO.createBlock(convertToInlineMessage(e), + e.getThumbnailUrl())) + .collect(Collectors.toList()); + return BlockMessageDTO.builder() + .requestedAt(requestedAt) + .title(String.format(TITLE_FORMAT_V1, + boardName.getMenuName(), + DateUtil.DATE_TIME_FORMATTER.format(requestedAt))) + .blocks(body) + .build(); + } + + public static String convertToInlineMessage(PpomppuArticle article) { + return String.format(MESSAGE_FORMAT_V2, + article.getBoardName().getMenuName(), article.getArticleUrl(), article.getTitle()); + } +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/constant/WebClientPropertiesUnitName.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/constant/WebClientPropertiesUnitName.java new file mode 100644 index 0000000..03615c7 --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/constant/WebClientPropertiesUnitName.java @@ -0,0 +1,19 @@ +package com.myoa.engineering.crawl.shopping.dto.constant; + +import lombok.AllArgsConstructor; +import lombok.Getter; + +/** + * WebClientPropertiesUnitName + * @author Shin Woo-jin (woo-jin.shin@linecorp.com) + * @since 2021-11-18 + * + */ +@Getter +@AllArgsConstructor +public enum WebClientPropertiesUnitName { + PPOMPPU_NOTIFIER_SENDER_API("ppn-sender-api"), + ; + + private String unitName; +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/slack/MessageDTO.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/slack/MessageDTO.java new file mode 100644 index 0000000..018a1fd --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/slack/MessageDTO.java @@ -0,0 +1,12 @@ +package com.myoa.engineering.crawl.shopping.dto.slack; + +import java.io.Serializable; + +/** + * MessageDTO + * @author Shin Woo-jin (woo-jin.shin@linecorp.com) + * @since 2021-11-14 + * + */ +public interface MessageDTO extends Serializable { +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/slack/SlackBaseMessageBlock.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/slack/SlackBaseMessageBlock.java new file mode 100644 index 0000000..4f41cba --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/slack/SlackBaseMessageBlock.java @@ -0,0 +1,47 @@ +package com.myoa.engineering.crawl.shopping.dto.slack; + +import com.fasterxml.jackson.annotation.JsonInclude; + +import lombok.Builder; +import lombok.Getter; +import lombok.NoArgsConstructor; + +/** + * MessageBlock + * @author Shin Woo-jin (woo-jin.shin@linecorp.com) + * @since 2021-11-30 + * + */ +@Getter +@NoArgsConstructor +@JsonInclude(JsonInclude.Include.NON_NULL) +public class SlackBaseMessageBlock implements SlackMessageBlock { + private static final long serialVersionUID = 1597984001727808419L; + + private SlackMessageBlockType type; + private String text; + + @Builder + private SlackBaseMessageBlock(SlackMessageBlockType type, String text) { + this.type = type; + this.text = text; + } + + public static SlackBaseMessageBlock ofMarkDown(String message) { + return SlackBaseMessageBlock.builder() + .type(SlackMessageBlockType.MARKDOWN) + .text(message) + .build(); + } + + public static SlackBaseMessageBlock ofDivider() { + return SlackBaseMessageBlock.builder() + .type(SlackMessageBlockType.DIVIDER) + .build(); + } + + @Override + public String getType() { + return type.getType(); + } +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/slack/SlackImageMessageBlock.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/slack/SlackImageMessageBlock.java new file mode 100644 index 0000000..8ec1e51 --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/slack/SlackImageMessageBlock.java @@ -0,0 +1,49 @@ +package com.myoa.engineering.crawl.shopping.dto.slack; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; + +import lombok.Builder; +import lombok.Getter; +import lombok.NoArgsConstructor; + +/** + * SlackImageMessageBlock + * @author Shin Woo-jin (woo-jin.shin@linecorp.com) + * @since 2021-11-30 + * + */ +@Getter +@NoArgsConstructor +@JsonInclude(JsonInclude.Include.NON_NULL) +public class SlackImageMessageBlock implements SlackMessageBlock { + private static final long serialVersionUID = 1597984001727808419L; + + private SlackMessageBlockType type; + + @JsonProperty(value = "image_url", required = true) + private String imageUrl; + + @JsonProperty(value = "alt_text", required = true) + private String altText; + + @Builder + private SlackImageMessageBlock(SlackMessageBlockType type, String imageUrl, String altText) { + this.type = type; + this.imageUrl = imageUrl; + this.altText = altText; + } + + public static SlackImageMessageBlock of(String imageUrl, String altText) { + return SlackImageMessageBlock.builder() + .type(SlackMessageBlockType.IMAGE) + .imageUrl(imageUrl) + .altText(altText) + .build(); + } + + @Override + public String getType() { + return type.getType(); + } +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/slack/SlackMessageBlock.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/slack/SlackMessageBlock.java new file mode 100644 index 0000000..59c9748 --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/slack/SlackMessageBlock.java @@ -0,0 +1,15 @@ +package com.myoa.engineering.crawl.shopping.dto.slack; + +import java.io.Serializable; + +/** + * SlackMessageBlock + * @author Shin Woo-jin (woo-jin.shin@linecorp.com) + * @since 2021-12-01 + * + */ +public interface SlackMessageBlock extends Serializable { + + String getType(); + +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/slack/SlackMessageBlockType.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/slack/SlackMessageBlockType.java new file mode 100644 index 0000000..a3cdc1b --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/slack/SlackMessageBlockType.java @@ -0,0 +1,22 @@ +package com.myoa.engineering.crawl.shopping.dto.slack; + +import lombok.AllArgsConstructor; +import lombok.Getter; + +/** + * BlockType + * @author Shin Woo-jin (woo-jin.shin@linecorp.com) + * @since 2021-11-30 + * + */ +@Getter +@AllArgsConstructor +public enum SlackMessageBlockType { + SECTION("section"), + MARKDOWN("mrkdwn"), + DIVIDER("divider"), + IMAGE("image"), + ; + + private String type; +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/slack/SlackMessageDTO.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/slack/SlackMessageDTO.java new file mode 100644 index 0000000..ba79116 --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/slack/SlackMessageDTO.java @@ -0,0 +1,61 @@ +package com.myoa.engineering.crawl.shopping.dto.slack; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.myoa.engineering.crawl.shopping.support.dto.BlockMessageDTO; +import lombok.Builder; +import lombok.Getter; +import lombok.NoArgsConstructor; + +import java.util.List; + +/** + * SlackMessageDTO + * + * @author Shin Woo-jin (woo-jin.shin@linecorp.com) + * @since 2021-11-14 + */ +@Getter +@NoArgsConstructor +@JsonInclude(JsonInclude.Include.NON_NULL) +public class SlackMessageDTO implements MessageDTO { + + private final static long serialVersionUID = 4737608709660494713L; + + private String text; + private String channel; + private String username; + private List blocks; + + @JsonProperty("icon_emoji") + private String iconEmoji; + + @Builder + public SlackMessageDTO(String text, String channel, String username, + List blocks, String iconEmoji) { + this.text = text; + this.channel = channel; + this.username = username; + this.blocks = blocks; + this.iconEmoji = iconEmoji; + } + + public void applyText(String text) { + this.text = text; + } + + public void addSectionBlock(BlockMessageDTO.Block block) { + SlackSectionMessageBlock slackSectionMessageBlock = SlackSectionMessageBlock.ofMarkDown(block.getText()); + slackSectionMessageBlock.applyImageaccessory(block.getImageUrl(), block.getAltText()); + addBlock(slackSectionMessageBlock); + } + + public void addSectionBlock(String rawBlockMessage) { + addBlock(SlackSectionMessageBlock.ofMarkDown(rawBlockMessage)); + } + + public void addBlock(SlackMessageBlock block) { + blocks.add(block); + } + +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/slack/SlackSectionMessageBlock.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/slack/SlackSectionMessageBlock.java new file mode 100644 index 0000000..651d77f --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/dto/slack/SlackSectionMessageBlock.java @@ -0,0 +1,48 @@ +package com.myoa.engineering.crawl.shopping.dto.slack; + +import com.fasterxml.jackson.annotation.JsonInclude; +import lombok.Builder; +import lombok.Getter; +import lombok.NoArgsConstructor; + +/** + * SectionBlock + * + * @author Shin Woo-jin (woo-jin.shin@linecorp.com) + * @since 2021-11-30 + */ +@Getter +@NoArgsConstructor +@JsonInclude(JsonInclude.Include.NON_NULL) +public class SlackSectionMessageBlock implements SlackMessageBlock { + private static final long serialVersionUID = -7600944576753160168L; + + private SlackMessageBlockType type; + private SlackBaseMessageBlock text; + private SlackImageMessageBlock accessory; + + @Builder + private SlackSectionMessageBlock(SlackMessageBlockType type, SlackBaseMessageBlock text, + SlackImageMessageBlock accessory) { + this.type = type; + this.text = text; + this.accessory = accessory; + } + + public static SlackSectionMessageBlock ofMarkDown(String message) { + return SlackSectionMessageBlock.builder() + .type(SlackMessageBlockType.SECTION) + .text(SlackBaseMessageBlock.ofMarkDown(message)) + .build(); + } + + public SlackSectionMessageBlock applyImageaccessory(String imageUrl, String altText) { + this.accessory = SlackImageMessageBlock.of(imageUrl, altText); + return this; + } + + @Override + public String getType() { + return type.getType(); + } +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/event/ArticleUpsertEvent.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/event/ArticleUpsertEvent.java new file mode 100644 index 0000000..f7cbe79 --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/event/ArticleUpsertEvent.java @@ -0,0 +1,13 @@ +package com.myoa.engineering.crawl.shopping.event; + +import com.myoa.engineering.crawl.shopping.domain.model.v2.ArticleModel; +import org.springframework.context.ApplicationEvent; + +import java.util.List; + +public class ArticleUpsertEvent extends ApplicationEvent { + + public ArticleUpsertEvent(List source) { + super(source); + } +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/event/ArticleUpsertEventPayload.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/event/ArticleUpsertEventPayload.java new file mode 100644 index 0000000..2fab09b --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/event/ArticleUpsertEventPayload.java @@ -0,0 +1,15 @@ +package com.myoa.engineering.crawl.shopping.event; + +import com.myoa.engineering.crawl.shopping.domain.model.v2.ArticleModel; +import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget; +import lombok.AllArgsConstructor; +import lombok.Getter; + +import java.util.List; + +@Getter +@AllArgsConstructor +public class ArticleUpsertEventPayload { + private final List articles; + private final CrawlTarget crawlTarget; +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/event/handler/ArticleUpsertEventListener.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/event/handler/ArticleUpsertEventListener.java new file mode 100644 index 0000000..f0ac31c --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/event/handler/ArticleUpsertEventListener.java @@ -0,0 +1,86 @@ +package com.myoa.engineering.crawl.shopping.event.handler; + +import com.myoa.engineering.crawl.shopping.domain.model.UserNotifyModel; +import com.myoa.engineering.crawl.shopping.domain.model.v2.AppUserModel; +import com.myoa.engineering.crawl.shopping.domain.model.v2.ArticleModel; +import com.myoa.engineering.crawl.shopping.domain.model.v2.SubscribedKeywordAggregatedModel; +import com.myoa.engineering.crawl.shopping.event.ArticleUpsertEvent; +import com.myoa.engineering.crawl.shopping.service.AppUserQueryService; +import com.myoa.engineering.crawl.shopping.service.SubscribedKeywordCacheService; +import com.myoa.engineering.crawl.shopping.service.UserNotifyService; +import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget; +import org.springframework.context.event.EventListener; +import org.springframework.stereotype.Component; + +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; + +@Component +public class ArticleUpsertEventListener { + + private final SubscribedKeywordCacheService subscribedKeywordCacheService; + private final AppUserQueryService appUserQueryService; + private final UserNotifyService userNotifyService; + + + public ArticleUpsertEventListener(SubscribedKeywordCacheService subscribedKeywordCacheService, + AppUserQueryService appUserQueryService, UserNotifyService userNotifyService) { + this.subscribedKeywordCacheService = subscribedKeywordCacheService; + this.appUserQueryService = appUserQueryService; + this.userNotifyService = userNotifyService; + } + + + @EventListener + public void handleArticleUpsertEvent(ArticleUpsertEvent event) { + Map> articleMap = + ((List) event.getSource()).stream() + .collect(Collectors.groupingBy(ArticleModel::getCrawlTarget)); + List appUsers = appUserQueryService.findAll(); + + appUsers.stream() + .filter(AppUserModel::getEnabled) + .map(user -> { + List filteredArticles = handleAhoCorasick(articleMap) + .apply(subscribedKeywordCacheService.getSubscribedKeywordsCached(user.getName())); + return UserNotifyModel.of(user.getName(), filteredArticles); + }) + .forEach(this::notifyMessage); + + } + + private Function, List> handleAhoCorasick( + Map> articleMap) { + return userTrieModel -> { + return userTrieModel + .entrySet() + .stream().filter(e -> articleMap.containsKey(e.getKey())) + .map((entry) -> filterAhocorasick(articleMap.get(entry.getKey()), entry.getValue())) + .flatMap(List::stream) + .toList(); +// return UserNotifyModel.of(userTrieModel.values().stream().findFirst().get().getUserId(), +// filteredArticle); + }; + } + + private List filterAhocorasick(List articles, + SubscribedKeywordAggregatedModel trieModel) { + return articles.stream() + .filter(article -> !trieModel.getAhoCorasickTrie() + .parseText(article.getTitle()) + .isEmpty()) + .toList(); + //ArticleUpsertEventListener::printArticle + } + + private void notifyMessage(UserNotifyModel article) { + System.out.println("article = " + article); + if (article.getArticles().isEmpty()){ + return; + } + userNotifyService.notify("์•ˆ๋…• " + article.getUserId() + "\n" + article.getArticles()); + } + +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/fmkorea/FmkoreaBoardClient.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/fmkorea/FmkoreaBoardClient.java new file mode 100644 index 0000000..1d0f0e0 --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/fmkorea/FmkoreaBoardClient.java @@ -0,0 +1,12 @@ +package com.myoa.engineering.crawl.shopping.infra.client.fmkorea; + +import org.springframework.cloud.openfeign.FeignClient; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; + +@FeignClient(value = "fmkorea-board-client", url = "https://fmkorea.com") +public interface FmkoreaBoardClient { + + @GetMapping("{boardLink}") + String getBoardHtml(@PathVariable("boardLink") String boardLink); +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/ppomppu/PpomppuBoardClient.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/ppomppu/PpomppuBoardClient.java new file mode 100644 index 0000000..1ab1b8e --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/ppomppu/PpomppuBoardClient.java @@ -0,0 +1,38 @@ +package com.myoa.engineering.crawl.shopping.infra.client.ppomppu; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; + +/** + * PpomppuBoardFeedRetriever + * + * @author Shin Woo-jin (woozu.shin@kakaoent.com) + * @since 2021-09-08 + */ +@Slf4j +@Component +public class PpomppuBoardClient { +/* + private final WebClient webClient; + + public PpomppuBoardClient(WebClient.Builder webClientBuilder) { + this.webClient = webClientBuilder.baseUrl(PpomppuBoardName.PPOMPPU_URL) + .exchangeStrategies(WebFluxExchangeStragiesFactory.ofTextHtml()) + .filter(WebClientFilterFactory.logRequest()) + .filter(WebClientFilterFactory.logResponse()) + .build(); + } + + public Mono getHtml(String uri) { + return webClient.get() + .uri(uri) + .exchangeToMono(e -> e.bodyToMono(String.class)) + .publishOn(Schedulers.boundedElastic()) + .onErrorResume(WebClientRequestException.class, t -> { + log.info("Exception occured, ignoring. : {}", t.getClass().getSimpleName()); + return Mono.empty(); + }); + // .doOnNext(e -> log.info("[getHtml] {}", e)); + } +*/ +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/ppomppu/PpomppuBoardClientV2.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/ppomppu/PpomppuBoardClientV2.java new file mode 100644 index 0000000..287dbae --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/ppomppu/PpomppuBoardClientV2.java @@ -0,0 +1,17 @@ +package com.myoa.engineering.crawl.shopping.infra.client.ppomppu; + +import org.springframework.cloud.openfeign.FeignClient; +import org.springframework.cloud.openfeign.SpringQueryMap; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestParam; + +import java.util.Map; + +@FeignClient(value = "ppomppu-board-client", url = "https://www.ppomppu.co.kr") +public interface PpomppuBoardClientV2 { + + @GetMapping("{boardLink}") + String getBoardHtml(@PathVariable("boardLink") String boardLink, + @SpringQueryMap Map params); +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/slack/SlackAPIClient.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/slack/SlackAPIClient.java new file mode 100644 index 0000000..1d1ed89 --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/slack/SlackAPIClient.java @@ -0,0 +1,17 @@ +package com.myoa.engineering.crawl.shopping.infra.client.slack; + +import com.myoa.engineering.crawl.shopping.configuration.FeignDefaultConfig; +import com.myoa.engineering.crawl.shopping.dto.slack.SlackMessageDTO; +import org.springframework.cloud.openfeign.FeignClient; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestHeader; + +@FeignClient(value = "slack-api-client", url = "https://slack.com/api", + configuration = FeignDefaultConfig.class) +public interface SlackAPIClient { + + @PostMapping("/chat.postMessage") + String sendMessage(@RequestBody SlackMessageDTO message, + @RequestHeader("Authorization") String token); +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/repository/BaseScanRepository.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/repository/BaseScanRepository.java new file mode 100644 index 0000000..04447d8 --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/repository/BaseScanRepository.java @@ -0,0 +1,4 @@ +package com.myoa.engineering.crawl.shopping.infra.repository; + +public interface BaseScanRepository { +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/repository/v1/PpomppuArticleRepository.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/repository/v1/PpomppuArticleRepository.java new file mode 100644 index 0000000..885d1ce --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/repository/v1/PpomppuArticleRepository.java @@ -0,0 +1,10 @@ +package com.myoa.engineering.crawl.shopping.infra.repository.v1; + +import com.myoa.engineering.crawl.shopping.domain.entity.v1.PpomppuArticle; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.stereotype.Repository; + +@Repository +public interface PpomppuArticleRepository extends JpaRepository { + +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/repository/v1/PpomppuBoardFeedStatusRepository.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/repository/v1/PpomppuBoardFeedStatusRepository.java new file mode 100644 index 0000000..0e87f92 --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/repository/v1/PpomppuBoardFeedStatusRepository.java @@ -0,0 +1,15 @@ +package com.myoa.engineering.crawl.shopping.infra.repository.v1; + +import com.myoa.engineering.crawl.shopping.domain.entity.v1.PpomppuBoardFeedStatus; +import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.stereotype.Repository; + +import java.util.Optional; + +@Repository +public interface PpomppuBoardFeedStatusRepository extends JpaRepository { + + Optional findByBoardName(PpomppuBoardName boardName); + +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/repository/v2/AppUserRepository.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/repository/v2/AppUserRepository.java new file mode 100644 index 0000000..6579b4a --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/repository/v2/AppUserRepository.java @@ -0,0 +1,9 @@ +package com.myoa.engineering.crawl.shopping.infra.repository.v2; + +import com.myoa.engineering.crawl.shopping.domain.entity.v2.AppUser; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.stereotype.Repository; + +@Repository +public interface AppUserRepository extends JpaRepository { +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/repository/v2/ArticleRepository.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/repository/v2/ArticleRepository.java new file mode 100644 index 0000000..1d80b33 --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/repository/v2/ArticleRepository.java @@ -0,0 +1,16 @@ +package com.myoa.engineering.crawl.shopping.infra.repository.v2; + +import com.myoa.engineering.crawl.shopping.domain.entity.v2.Article; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.stereotype.Repository; + +import java.util.Collection; +import java.util.List; +import java.util.Optional; + +@Repository +public interface ArticleRepository extends JpaRepository { + + Optional
findByArticleId(Long articleId); + List
findByArticleIdIn(Collection articleId); +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/repository/v2/SubscribedKeywordRepository.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/repository/v2/SubscribedKeywordRepository.java new file mode 100644 index 0000000..90028c1 --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/repository/v2/SubscribedKeywordRepository.java @@ -0,0 +1,23 @@ +package com.myoa.engineering.crawl.shopping.infra.repository.v2; + +import com.myoa.engineering.crawl.shopping.domain.entity.v2.SubscribedKeyword; +import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.stereotype.Repository; + +import java.util.List; + +@Repository +public interface SubscribedKeywordRepository extends JpaRepository { + + List findByCrawlTarget(CrawlTarget crawlTarget); + + + /* @Query("SELECT new com.myoa.engineering.crawl.shopping.domain.model.v2.SubscribedKeywordUserAggregatedModel(" + + " s.userId, s.keyword, s.crawlTarget) " + + " FROM SubscribedKeyword s GROUP BY s.userId ") + List findGroupByUserId(String userId);*/ + List findByUserIdAndCrawlTarget(String userId, CrawlTarget crawlTarget); + + List findByUserId(String userId); +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/scheduler/ParseEventEmitter.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/scheduler/ParseEventEmitter.java new file mode 100644 index 0000000..ae65851 --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/scheduler/ParseEventEmitter.java @@ -0,0 +1,27 @@ +package com.myoa.engineering.crawl.shopping.scheduler; + +import com.myoa.engineering.crawl.shopping.crawlhandler.CrawlHandler; +import lombok.extern.slf4j.Slf4j; +import org.springframework.scheduling.annotation.EnableScheduling; +import org.springframework.stereotype.Component; + +import java.util.List; + +@Slf4j +@Component +@EnableScheduling +public class ParseEventEmitter { + private final List crawlHandlers; + + + public ParseEventEmitter(List crawlHandlers) { + this.crawlHandlers = crawlHandlers; + } + + // @Scheduled(cron = "0 0/5 * * * ?") + public void emit() { + log.info("[emitDomesticBoard] trigger fired!"); + crawlHandlers.forEach(CrawlHandler::handle); + + } +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/AppUserQueryService.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/AppUserQueryService.java new file mode 100644 index 0000000..c2f1d01 --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/AppUserQueryService.java @@ -0,0 +1,25 @@ +package com.myoa.engineering.crawl.shopping.service; + +import com.myoa.engineering.crawl.shopping.domain.model.v2.AppUserModel; +import com.myoa.engineering.crawl.shopping.infra.repository.v2.AppUserRepository; +import org.springframework.stereotype.Service; + +import java.util.List; + +@Service +public class AppUserQueryService { + + private final AppUserRepository appUserRepository; + + public AppUserQueryService(AppUserRepository appUserRepository) { + this.appUserRepository = appUserRepository; + } + + public List findAll() { + return appUserRepository.findAll() + .stream() + .map(AppUserModel::from) + .toList(); + } + +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/ArticleCommandService.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/ArticleCommandService.java new file mode 100644 index 0000000..44af3db --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/ArticleCommandService.java @@ -0,0 +1,81 @@ +package com.myoa.engineering.crawl.shopping.service; + +import com.myoa.engineering.crawl.shopping.domain.entity.v2.Article; +import com.myoa.engineering.crawl.shopping.domain.model.v2.ArticleModel; +import com.myoa.engineering.crawl.shopping.event.ArticleUpsertEvent; +import com.myoa.engineering.crawl.shopping.infra.repository.v2.ArticleRepository; +import javax.transaction.Transactional; +import org.springframework.context.ApplicationEventPublisher; +import org.springframework.stereotype.Service; + +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; + +@Service +public class ArticleCommandService { + + private final ArticleRepository articleRepository; + private final ApplicationEventPublisher applicationEventPublisher; + + public ArticleCommandService(ArticleRepository articleRepository, + ApplicationEventPublisher applicationEventPublisher) { + this.articleRepository = articleRepository; + this.applicationEventPublisher = applicationEventPublisher; + } + + + @Transactional + public void upsert(List
articles) { + Map articleMap = articles.stream() + .collect(Collectors.toMap(Article::getArticleId, e -> e)); + + List
saved = articleRepository.findByArticleIdIn(articleMap.keySet()); + List
updated = saved.stream() + .filter(e -> articleMap.containsKey(e.getArticleId())) + .map(e -> e.update(articleMap.get(e.getArticleId()))) + .peek(e -> articleMap.remove(e.getArticleId())) + .sorted(Comparator.comparing(Article::getArticleId)) + .toList(); + + List
newArticles = articleMap.values() + .stream() + .sorted(Comparator.comparing(Article::getArticleId)) + .toList(); + + articleRepository.saveAll(updated); + articleRepository.saveAll(newArticles); + + publishEvent(newArticles); + } + + private void publishEvent(List
articles) { + List articleModels = + articles.stream() + .map(transformer) + .toList(); + applicationEventPublisher.publishEvent(new ArticleUpsertEvent(articleModels)); + } + + @Transactional + public void upsert(Article article) { + Article saved = articleRepository.findByArticleId(article.getArticleId()) + .orElse(article); + articleRepository.save(saved); + } + + public static Function transformer = + article -> ArticleModel.builder() + .id(article.getId()) + .articleId(article.getArticleId()) + .crawlTarget(article.getCrawlTarget()) + .boardName(article.getBoardName()) + .articleUrl(article.getArticleUrl()) + .title(article.getTitle()) + .hit(article.getHit()) + .recommended(article.getRecommended()) + .registeredAt(article.getRegisteredAt()) + .build(); +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/PpomppuArticleService.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/PpomppuArticleService.java new file mode 100644 index 0000000..88b89b3 --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/PpomppuArticleService.java @@ -0,0 +1,67 @@ +package com.myoa.engineering.crawl.shopping.service; + +import com.myoa.engineering.crawl.shopping.domain.entity.v1.PpomppuArticle; +import com.myoa.engineering.crawl.shopping.domain.entity.v1.PpomppuBoardFeedStatus; +import com.myoa.engineering.crawl.shopping.infra.repository.v1.PpomppuArticleRepository; +import com.myoa.engineering.crawl.shopping.infra.repository.v1.PpomppuBoardFeedStatusRepository; +import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +@Slf4j +@Service +public class PpomppuArticleService { + + private final PpomppuArticleRepository ppomppuArticleRepository; + + private final PpomppuBoardFeedStatusRepository ppomppuBoardFeedStatusRepository; + + public PpomppuArticleService(PpomppuArticleRepository ppomppuArticleRepository, + PpomppuBoardFeedStatusRepository ppomppuBoardFeedStatusRepository) { + this.ppomppuArticleRepository = ppomppuArticleRepository; + this.ppomppuBoardFeedStatusRepository = ppomppuBoardFeedStatusRepository; + } + + @Transactional(readOnly = true) + public List filterOnlyNewArticles(PpomppuBoardName boardName, List articles) { + Optional boardFeedStatus = ppomppuBoardFeedStatusRepository.findByBoardName(boardName); + Long latestArticleId = boardFeedStatus.map(PpomppuBoardFeedStatus::getLatestParsedArticleId) + .orElse(0L); + + log.info("latestArticleId : {}", latestArticleId); + return articles.stream() + .filter(e -> e.getArticleId().compareTo(latestArticleId) > 0) + .collect(Collectors.toList()); + } + + @Transactional + public List save(PpomppuBoardName boardName, List articles) { + Long latestArticleId = articles.stream() + .map(PpomppuArticle::getArticleId) + .max(Long::compareTo) + .orElse(0L); + + // save PpomppuBoardFeedStatus + Optional boardFeedStatus = ppomppuBoardFeedStatusRepository.findByBoardName(boardName); + log.info("[save] boardName: {}, isPresent?: {}, latestArticleId: {}", + boardName, boardFeedStatus.isPresent(), latestArticleId); + log.info("[save] articles count: {}, article ids: {}", + articles.size(), articles.stream().map(PpomppuArticle::getArticleId).toArray()); + boardFeedStatus.ifPresentOrElse(e -> { + if (latestArticleId.longValue() > 0L) { + e.updateArticleId(latestArticleId); + ppomppuBoardFeedStatusRepository.save(e); + } + }, + () -> ppomppuBoardFeedStatusRepository.save(PpomppuBoardFeedStatus.of(boardName, + latestArticleId))); + + // save real articles. + return ppomppuArticleRepository.saveAll(articles); + } +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/SubscribedKeywordCacheService.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/SubscribedKeywordCacheService.java new file mode 100644 index 0000000..a0fd644 --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/SubscribedKeywordCacheService.java @@ -0,0 +1,41 @@ +package com.myoa.engineering.crawl.shopping.service; + +import com.myoa.engineering.crawl.shopping.domain.entity.v2.SubscribedKeyword; +import com.myoa.engineering.crawl.shopping.domain.model.v2.SubscribedKeywordAggregatedModel; +import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget; +import org.springframework.cache.annotation.Cacheable; +import org.springframework.stereotype.Component; + +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +@Component +public class SubscribedKeywordCacheService { + + private final SubscribedKeywordQueryService subscribedKeywordQueryService; + + public SubscribedKeywordCacheService(SubscribedKeywordQueryService subscribedKeywordQueryService) { + this.subscribedKeywordQueryService = subscribedKeywordQueryService; + } + + @Cacheable(cacheNames = "subscribe.keywords", key = "#userId + '_' + #crawlTarget.name()") + public SubscribedKeywordAggregatedModel getSubscribedKeywordsCached(String userId, CrawlTarget crawlTarget) { + System.out.println("getSubscribedKeywordsCached"); + List keywords = subscribedKeywordQueryService.findByUserWithTarget(userId, crawlTarget) + .stream().map(SubscribedKeyword::getKeyword).toList(); + return SubscribedKeywordAggregatedModel.of(userId, crawlTarget, keywords); + } + + @Cacheable(cacheNames = "subscribe.keywords", key = "#userId") + public Map getSubscribedKeywordsCached(String userId) { + System.out.println("getSubscribedKeywordsCached"); + return subscribedKeywordQueryService.findByUser(userId) + .stream() + .collect(Collectors.groupingBy(SubscribedKeyword::getCrawlTarget, + Collectors.mapping(SubscribedKeyword::getKeyword, Collectors.toList()))) + .entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getKey, e -> SubscribedKeywordAggregatedModel.of(userId, e.getKey(), e.getValue()))); + } + +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/SubscribedKeywordQueryService.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/SubscribedKeywordQueryService.java new file mode 100644 index 0000000..dc06a58 --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/SubscribedKeywordQueryService.java @@ -0,0 +1,30 @@ +package com.myoa.engineering.crawl.shopping.service; + +import com.myoa.engineering.crawl.shopping.domain.entity.v2.SubscribedKeyword; +import com.myoa.engineering.crawl.shopping.infra.repository.v2.SubscribedKeywordRepository; +import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget; +import org.springframework.stereotype.Service; + +import java.util.List; + +@Service +public class SubscribedKeywordQueryService { + + private final SubscribedKeywordRepository subscribedKeywordRepository; + + public SubscribedKeywordQueryService(SubscribedKeywordRepository subscribedKeywordRepository) { + this.subscribedKeywordRepository = subscribedKeywordRepository; + } + + public List findAll() { + return subscribedKeywordRepository.findAll(); + } + + public List findByUserWithTarget(String userId, CrawlTarget crawlTarget) { + return subscribedKeywordRepository.findByUserIdAndCrawlTarget(userId, crawlTarget); + } + + public List findByUser(String userId) { + return subscribedKeywordRepository.findByUserId(userId); + } +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/UserNotifyService.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/UserNotifyService.java new file mode 100644 index 0000000..13486ee --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/service/UserNotifyService.java @@ -0,0 +1,34 @@ +package com.myoa.engineering.crawl.shopping.service; + +import com.myoa.engineering.crawl.shopping.configuration.slack.properties.SlackSecretProperties; +import com.myoa.engineering.crawl.shopping.dto.slack.SlackMessageDTO; +import com.myoa.engineering.crawl.shopping.infra.client.slack.SlackAPIClient; +import org.springframework.stereotype.Service; + +@Service +public class UserNotifyService { + + private static final String SLACK_PROPERTIES_UNIT_NAME = "shopping-crawler"; +// private static final String NOTIFY_CHANNEL_ID = "notify_shopping"; +// private static final String NOTIFY_ICON_EMOJI = ":monge_big:"; +// private static final String NOTIFY_BOT_NAME = "๋ชฝ์ด ํƒˆํ˜ธ๊ตฌ๋ด‡"; + + private final SlackAPIClient slackAPIClient; + private final SlackSecretProperties.SlackSecretPropertiesUnit slackSecretProperties; + + public UserNotifyService(SlackAPIClient slackAPIClient, + SlackSecretProperties slackSecretProperties) { + this.slackAPIClient = slackAPIClient; + this.slackSecretProperties = slackSecretProperties.find(SLACK_PROPERTIES_UNIT_NAME); + } + + public void notify(String message) { + SlackMessageDTO slackMessageDTO = SlackMessageDTO.builder() + .channel(slackSecretProperties.getChannel()) + .text(message) + .iconEmoji(slackSecretProperties.getIconEmoji()) + .username(slackSecretProperties.getUsername()) + .build(); + slackAPIClient.sendMessage(slackMessageDTO, slackSecretProperties.getToken()); + } +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/util/AhoCorasickUtils.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/util/AhoCorasickUtils.java new file mode 100644 index 0000000..1066198 --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/util/AhoCorasickUtils.java @@ -0,0 +1,18 @@ +package com.myoa.engineering.crawl.shopping.util; + +import org.ahocorasick.trie.Trie; + +import java.util.List; + +public final class AhoCorasickUtils { + + private AhoCorasickUtils() { + } + + public static Trie generateTrie(List keywords) { + return Trie.builder() + .addKeywords(keywords) + .ignoreCase() + .build(); + } +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/util/DateTimeUtils.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/util/DateTimeUtils.java new file mode 100644 index 0000000..3ec9d5a --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/util/DateTimeUtils.java @@ -0,0 +1,27 @@ +package com.myoa.engineering.crawl.shopping.util; + +import java.time.*; +import java.time.format.DateTimeFormatter; + +public final class DateTimeUtils { + + private static final DateTimeFormatter FORMATTER_HHMMss = DateTimeFormatter.ofPattern("HH:mm:ss"); + private static final ZoneId ZONE_ASIA_SEOUL = ZoneId.of("Asia/Seoul"); + + private DateTimeUtils() { + } + + public static ZonedDateTime parse(String HHMMss) { + try { + LocalTime time = LocalTime.parse(HHMMss, FORMATTER_HHMMss); + LocalDateTime dateTime = LocalDateTime.of(LocalDate.now(), time); + if (dateTime.isAfter(LocalDateTime.now())) { + dateTime = dateTime.minusDays(1); + } + return dateTime.atZone(ZONE_ASIA_SEOUL); + } catch (Exception e) { + return null; + } + } + +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/util/NumberUtils.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/util/NumberUtils.java new file mode 100644 index 0000000..6efae60 --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/util/NumberUtils.java @@ -0,0 +1,32 @@ +package com.myoa.engineering.crawl.shopping.util; + +public final class NumberUtils { + + private NumberUtils() { + } + + public static Integer parseInt(String value) { + return parseInt(value, null); + } + + public static Integer parseInt(String value, Integer defaultValue) { + try { + return Integer.parseInt(value); + } catch (NumberFormatException e) { + return defaultValue; + } + } + + public static Long parseLong(String value) { + return parseLong(value, null); + } + + public static Long parseLong(String value, Long defaultValue) { + try { + return Long.parseLong(value); + } catch (NumberFormatException e) { + return defaultValue; + } + } + +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/util/ObjectMapperFactory.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/util/ObjectMapperFactory.java new file mode 100644 index 0000000..0785ccb --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/util/ObjectMapperFactory.java @@ -0,0 +1,62 @@ +package com.myoa.engineering.crawl.shopping.util; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.core.json.JsonReadFeature; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.databind.json.JsonMapper; +import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import com.fasterxml.jackson.module.paramnames.ParameterNamesModule; + +public final class ObjectMapperFactory { + + public static final ObjectMapper DEFAULT_MAPPER; + public static final ObjectMapper REDIS_MAPPER; + @Deprecated + public static final ObjectMapper LOGGING_MAPPER; + + static { + DEFAULT_MAPPER = initDefaultMapper(); + REDIS_MAPPER = initRedisMapper(); + LOGGING_MAPPER = JsonMapper.builder() + .serializationInclusion(JsonInclude.Include.NON_EMPTY) + .build(); + } + + private ObjectMapperFactory() { + } + + private static ObjectMapper initDefaultMapper() { + return JsonMapper.builder() + .configure(JsonReadFeature.ALLOW_UNQUOTED_FIELD_NAMES.mappedFeature(), true) + .configure(JsonReadFeature.ALLOW_UNESCAPED_CONTROL_CHARS.mappedFeature(), true) + .disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES) + .disable(SerializationFeature.FAIL_ON_EMPTY_BEANS) + .disable(SerializationFeature.FAIL_ON_UNWRAPPED_TYPE_IDENTIFIERS) + .enable(DeserializationFeature.READ_UNKNOWN_ENUM_VALUES_AS_NULL) + .disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS) + .serializationInclusion(JsonInclude.Include.NON_NULL) + .build() + .registerModule(new ParameterNamesModule()) + .registerModule(new Jdk8Module()) + .registerModule(new JavaTimeModule()); + } + + private static ObjectMapper initRedisMapper() { + return JsonMapper.builder() + .configure(JsonReadFeature.ALLOW_UNQUOTED_FIELD_NAMES.mappedFeature(), true) + .configure(JsonReadFeature.ALLOW_UNESCAPED_CONTROL_CHARS.mappedFeature(), true) + .disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES) + .disable(SerializationFeature.FAIL_ON_EMPTY_BEANS) + .disable(SerializationFeature.FAIL_ON_UNWRAPPED_TYPE_IDENTIFIERS) + .enable(DeserializationFeature.READ_UNKNOWN_ENUM_VALUES_AS_NULL) + .disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS) + .serializationInclusion(JsonInclude.Include.NON_NULL) + .build() + .registerModule(new ParameterNamesModule()) + .registerModule(new Jdk8Module()) + .registerModule(new JavaTimeModule()); + } +} diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/util/ObjectUtils.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/util/ObjectUtils.java new file mode 100644 index 0000000..765100b --- /dev/null +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/util/ObjectUtils.java @@ -0,0 +1,121 @@ +/* + * Copyright (c) 2019 LINE Corporation. All rights reserved. + * LINE Corporation PROPRIETARY/CONFIDENTIAL. Use is subject to license terms. + */ + +package com.myoa.engineering.crawl.shopping.util; + +import java.util.Arrays; +import java.util.Collection; + +/** + * NumberUtils + * + * @author Shin Woo-jin (lp12254@linecorp.com) + * @since 2019-10-28 + */ +public final class ObjectUtils { + + private ObjectUtils() { + } + + /** + * Check if given object is null. + * + * e == object == > false e == null == > true + * + * + * @param e Target object + * @param Unfixed specific type. If you want restrict specific interface, Copy and extend qualifier. + * @return Is null given object? + */ + public static boolean isNullObject(final E e) { + return e == null; + } + + /** + * Check if given object is not null. + * + * e == object == > false e == null == > true + * + * + * @param e Target object + * @param Unfixed specific type. If you want restrict specific interface, Copy and extend qualifier. + * @return Is not null given object? + */ + public static boolean isNotEmpty(final E e) { + return !isNullObject(e); + } + + /** + * Check if there are any null object in given objects. + * + * args == object = > false args == object, object = > false args == null, null, object = > true args + * == null = > true args == null, null = > true + * + * + * @param args Want to check objects that have null. + * @return Is there objects array has null? + */ + public static boolean hasNullObject(Object... args) { + return Arrays.stream(args).anyMatch(ObjectUtils::isNullObject); + } + + /** + * Check given objects are not empty. + * + * args == object = > true args == object, object = > true args == null, null, object = > false args + * == null = > false args == null, null = > false + * + * + * @param args Want to check objects that have null. + * @return Is there objects array has null? + */ + public static boolean hasAllObject(Object... args) { + return Arrays.stream(args).noneMatch(ObjectUtils::isNullObject); + } + + /** + * Check if there are all null object in given objects. + * + * args == object = > false args == object, object = > false args == null, null, object = > false args + * == null = > true args == null, null = > true + * + * + * @param args Want to check objects that have null. + * @return Is there null all of given objects? + */ + public static boolean hasAllNullObjects(final Object... args) { + return Arrays.stream(args).allMatch(ObjectUtils::isNullObject); + } + + /** + * Check if given collection object is null or empty collecton. + * + * e == null = > true e == emptyCollection = > true e == hasElement = > false + * + * + * @param e e is must be Collection object + * @param E is must be extended Collection Class + * @return boolean. given collection is null or empty? + */ + public static > boolean isNullOrEmptyCollection(final E e) { + return e == null || e.isEmpty(); + } + + /** + * Get collection's size. Even it pointed null + * + * @param e e is must be Collection object + * @param E is must be extended Collection Class + * @return integer value. given collection's size. + */ + public static > int getCollectionSize(final E e) { + if (isNullOrEmptyCollection(e)) { + return 0; + } + + return e.size(); + } + +} diff --git a/shopping-crawler/src/main/resources/application-development.yml b/shopping-crawler/src/main/resources/application-development.yml new file mode 100644 index 0000000..e383370 --- /dev/null +++ b/shopping-crawler/src/main/resources/application-development.yml @@ -0,0 +1,12 @@ +spring: + config: + activate: + on-profile: development + import: + - "configserver:http://192.168.0.100:11080" + + +server: + port: 20081 + + # import: optional:configserver:http://localhost:11080 # can be start up even config server was not found. \ No newline at end of file diff --git a/shopping-crawler/src/main/resources/application-local.yml b/shopping-crawler/src/main/resources/application-local.yml new file mode 100644 index 0000000..67afb03 --- /dev/null +++ b/shopping-crawler/src/main/resources/application-local.yml @@ -0,0 +1,18 @@ +spring: + config: + activate: + on-profile: local + import: + - classpath:/datasource/local.yml + - classpath:/slack/local.yml + +server: + port: 20080 + + # import: optional:configserver:http://localhost:11080 # can be start up even config server was not found. + +feign: + client: + config: + default: + loggerLevel: FULL \ No newline at end of file diff --git a/shopping-crawler/src/main/resources/application-production.yml b/shopping-crawler/src/main/resources/application-production.yml new file mode 100644 index 0000000..4ff32ed --- /dev/null +++ b/shopping-crawler/src/main/resources/application-production.yml @@ -0,0 +1,6 @@ +spring: + config: + activate: + on-profile: production + import: + - "configserver:http://ppn-config-server:20080" diff --git a/shopping-crawler/src/main/resources/application.yml b/shopping-crawler/src/main/resources/application.yml new file mode 100644 index 0000000..bfd63e5 --- /dev/null +++ b/shopping-crawler/src/main/resources/application.yml @@ -0,0 +1,28 @@ +spring: + application: + name: crawler-shopping + main: + allow-bean-definition-overriding: true + profiles: + active: ${SPRING_ACTIVE_PROFILE:local} + group: + local: "local,datasource-local,webclient-local" + development: "development,datasource-development,webclient-development" + production: "production,datasource-production,webclient-production" + freemarker: + enabled: false + cloud: + config: + enabled: false + +server: + port: 20080 + error: + whitelabel: + enabled: false + +management: + endpoints: + web: + exposure: + include: refresh,health diff --git a/shopping-crawler/src/main/resources/logback-spring.xml b/shopping-crawler/src/main/resources/logback-spring.xml new file mode 100644 index 0000000..907a4f5 --- /dev/null +++ b/shopping-crawler/src/main/resources/logback-spring.xml @@ -0,0 +1,15 @@ + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/shopping-crawler/src/main/resources/logback/component/logback-file.xml b/shopping-crawler/src/main/resources/logback/component/logback-file.xml new file mode 100644 index 0000000..65e54db --- /dev/null +++ b/shopping-crawler/src/main/resources/logback/component/logback-file.xml @@ -0,0 +1,23 @@ + + + + + ${DIRECTORY}/${LOG_FILE_BASE}_log + + ${DIRECTORY}/${LOG_FILE_BASE}_log.%d{yyyyMMdd}.%i + + 1000MB + + 60 + + + ${FILE_LOG_PATTERN} + ${IMMEDIATE_FLUSH} + + + + 1024 + + + diff --git a/shopping-crawler/src/main/resources/logback/logback-development.xml b/shopping-crawler/src/main/resources/logback/logback-development.xml new file mode 100644 index 0000000..458e3d8 --- /dev/null +++ b/shopping-crawler/src/main/resources/logback/logback-development.xml @@ -0,0 +1,19 @@ + + + + + + + + + + + + + + + + + + + diff --git a/shopping-crawler/src/main/resources/logback/logback-production.xml b/shopping-crawler/src/main/resources/logback/logback-production.xml new file mode 100644 index 0000000..f824e41 --- /dev/null +++ b/shopping-crawler/src/main/resources/logback/logback-production.xml @@ -0,0 +1,19 @@ + + + + + + + + + + + + + + + + + + + diff --git a/shopping-crawler/src/test/java/com/myoa/engineering/crawl/shopping/crawlhandler/parser/PpomppuArticleParserV2Test.java b/shopping-crawler/src/test/java/com/myoa/engineering/crawl/shopping/crawlhandler/parser/PpomppuArticleParserV2Test.java new file mode 100644 index 0000000..eff04e6 --- /dev/null +++ b/shopping-crawler/src/test/java/com/myoa/engineering/crawl/shopping/crawlhandler/parser/PpomppuArticleParserV2Test.java @@ -0,0 +1,6 @@ +package com.myoa.engineering.crawl.shopping.crawlhandler.parser; + +class PpomppuArticleParserV2Test { + + +} \ No newline at end of file diff --git a/shopping-crawler/src/test/java/com/myoa/engineering/crawl/shopping/event/handler/ArticleUpsertEventListenerTest.java b/shopping-crawler/src/test/java/com/myoa/engineering/crawl/shopping/event/handler/ArticleUpsertEventListenerTest.java new file mode 100644 index 0000000..375fe80 --- /dev/null +++ b/shopping-crawler/src/test/java/com/myoa/engineering/crawl/shopping/event/handler/ArticleUpsertEventListenerTest.java @@ -0,0 +1,81 @@ +package com.myoa.engineering.crawl.shopping.event.handler; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.ahocorasick.trie.Emit; +import org.ahocorasick.trie.Trie; +import org.jeasy.random.EasyRandom; +import org.jeasy.random.EasyRandomParameters; +import org.junit.jupiter.api.Test; + +import java.util.Collection; +import java.util.List; + +class ArticleUpsertEventListenerTest { + + @Test + public void test1() { + + List baseData = List.of( + "[๊ณตํ™ˆ]๋ฒ ๋ฒ ์ˆฒ ์‹œ๊ทธ๋‹ˆ์ฒ˜ ์œ„๋“œ ๋ธ”๋ฃจ 20ํŒฉ (25,990์›/๋ฌด๋ฃŒ)2 [๊ธฐํƒ€]", + "[ํ˜„๋Œ€H๋ชฐ]10์ฃผ๋…„ ์ŠคํŽ˜์…œ ์—๋””์…˜ ๋ด‰๊ณ ๋ฐ๊ธฐ 40mm ์„ธํŠธ (67,640์›/๋ฌด๋ฃŒ)3 [๊ฐ€์ „/๊ฐ€๊ตฌ]", + "[SSG]ํ•„๋ฆฝ์Šค ๋ฉด๋„๊ธฐ ์นซ์†” ๊ธฐํš์ „( 437,000์›~/๋ฌด๋ฃŒ)1 [๊ธฐํƒ€]", + "(ํ‹ฐ๋ชฌ)์ œ์Šคํ”„๋ฆฌ ๊ณจ๋“œํ‚ค์œ„ ์ค‘๋Œ€๊ณผ 1.8kg๋‚ด์™ธ (์นดํŽ˜,ํ† ํŽ˜ 14,823์›/๋ฌด๋ฐฐ)3 [์‹ํ’ˆ/๊ฑด๊ฐ•]", + "[๊ณต์‹๋ชฐ]์—ฐ์„ธ์ƒํ™œ๊ฑด๊ฐ• ๋‹น๋‡จ์˜์–‘์‹ 24ํŒฉ (33,500์›/์œ ๋ฐฐ)1 [์‹ํ’ˆ/๊ฑด๊ฐ•]", + "[์ธํ„ฐํŒŒํฌ]ํฌ๋ฆฌ์Šคํƒˆ๋ผ์ดํŠธ ์•„์ด์Šคํ‹ฐ ์—์ด๋“œ ์˜จ๋”๊ณ  60๊ฐœ์ž… ๋ฉ€ํ‹ฐํŒฉ(17,800์›/๋ฌด๋ฃŒ)6 [๊ธฐํƒ€]", + "[์ธํŒ์‡ผํ•‘]์ƒค์นด์›จ์–ด ์•กํ‹ฐ๋ธŒ ๋งฅ์Šค ํ—ค๋น„์›จ์ดํŠธ 10์ข… ํƒ 1 (7,350์›/๋ฌด๋ฃŒ)7 [์˜๋ฅ˜/์žกํ™”]" + ); + + Trie trie = Trie.builder() + .ignoreCase() + .addKeyword("๋ธ”๋ฃจ") + .addKeyword("๋ด‰๊ณ ") + .build(); + + List> list = baseData.stream() + .map(trie::parseText) + .toList(); + + System.out.println(list); + } + + @Test + void test2() throws InterruptedException { + EasyRandomParameters paramsKeyword = new EasyRandomParameters() + .stringLengthRange(2, 10); + EasyRandom GEN_KEYWORD = new EasyRandom(paramsKeyword); + + EasyRandomParameters paramsSentence = new EasyRandomParameters() + .stringLengthRange(100, 1000); + EasyRandom GEN_SENTENCE = new EasyRandom(paramsSentence); + + Trie.TrieBuilder trieBuilder = Trie.builder().ignoreOverlaps(); + + GEN_KEYWORD.objects(TestA.class, 1000) + .map(TestA::getFieldA) + .forEach(trieBuilder::addKeyword); + Trie trie = trieBuilder.build(); + + List> listOfEmits = GEN_SENTENCE.objects(TestA.class, 2000) + .map(TestA::getFieldA) + .map(trie::parseText) + .toList(); + + for(Collection emits : listOfEmits) { + System.out.println(emits); + System.out.println("--------------------------"); + } + + } + + + @Data + @AllArgsConstructor + @NoArgsConstructor + public class TestA { + private String fieldA; + + } + +} \ No newline at end of file diff --git a/shopping-crawler/src/test/java/com/myoa/engineering/crawl/shopping/util/TestDataUtils.java b/shopping-crawler/src/test/java/com/myoa/engineering/crawl/shopping/util/TestDataUtils.java new file mode 100644 index 0000000..e5ff1fe --- /dev/null +++ b/shopping-crawler/src/test/java/com/myoa/engineering/crawl/shopping/util/TestDataUtils.java @@ -0,0 +1,62 @@ +package com.myoa.engineering.crawl.shopping.util; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.springframework.lang.NonNull; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Scanner; +import java.util.stream.Collectors; + +public final class TestDataUtils { + + private static final ObjectMapper OBJECT_MAPPER; + + static { + OBJECT_MAPPER = ObjectMapperFactory.DEFAULT_MAPPER; + } + + private TestDataUtils() { + } + + public static T inputStreamToObject(@NonNull String resourcePath, + @NonNull Class clazz) throws IOException { + return OBJECT_MAPPER.readValue(fileToString(resourcePath), clazz); + } + + public static List inputStreamsToList(@NonNull Class clazz, + @NonNull String... resourcePath) throws IOException { + return inputStreamsToList(Arrays.asList(resourcePath), clazz); + } + + public static List inputStreamsToList(@NonNull List resourcePath, + @NonNull Class clazz) throws IOException { + return resourcePath.stream() + .map(e -> { + try { + return OBJECT_MAPPER.readValue(fileToString(e), clazz); + } catch (JsonProcessingException jsonProcessingException) { + jsonProcessingException.printStackTrace(); + return null; + } + }) + .collect(Collectors.toList()); + } + + public static List inputStreamToList(@NonNull String resourcePath, + @NonNull Class clazz) throws IOException { + return (List) OBJECT_MAPPER.readValue( + fileToString(resourcePath), + OBJECT_MAPPER.getTypeFactory().constructCollectionType(List.class, clazz)); + } + + public static String fileToString(@NonNull String resourcePath) { + + final ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); + final Scanner s = new Scanner(classLoader.getResourceAsStream(resourcePath)).useDelimiter("\\A"); + return s.hasNext() ? s.next() : ""; + } + +} diff --git a/shopping-crawler/src/test/resources/logback-development.xml b/shopping-crawler/src/test/resources/logback-development.xml new file mode 100644 index 0000000..ab50ad9 --- /dev/null +++ b/shopping-crawler/src/test/resources/logback-development.xml @@ -0,0 +1,12 @@ + + + + + + + + + + + + diff --git a/shopping-crawler/src/test/resources/logback-spring.xml b/shopping-crawler/src/test/resources/logback-spring.xml new file mode 100644 index 0000000..04ccbd8 --- /dev/null +++ b/shopping-crawler/src/test/resources/logback-spring.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/shopping-crawler/src/test/resources/testdata/zboard/file1.html b/shopping-crawler/src/test/resources/testdata/zboard/file1.html new file mode 100644 index 0000000..2de3e57 --- /dev/null +++ b/shopping-crawler/src/test/resources/testdata/zboard/file1.html @@ -0,0 +1,1100 @@ + + + + + + + + + + + + + + + + ๋ฝ๋ฟŒ - ๋ฝ๋ฟŒ๊ฒŒ์‹œํŒ + + + + + + + + + + + + + + + + + + + + +
+
+
+
+ +
+
+
+
+
+
 
+
+
+
+ ๋ฝ๋ฟŒ + ๋ฝ๋ฟŒ +
+ +
+
+
+
+ + +
+ +
+ + +
+
+ +
+ +
+ + +
+ + + + +
+ ๋กœ๊ทธ์ธ + ํšŒ์›๊ฐ€์ž…|์•„์ด๋””๋น„๋ฒˆ์ฐพ๊ธฐ +
+
+
+ +
+ +
+

๋ฝ๋ฟŒ๊ฒŒ์‹œํŒ ์ž…๋‹ˆ๋‹ค.

+ +
+
+ ๊ฐ€์„ฑ๋น„ ์ข‹์€ ์ƒํ’ˆ๊ณผ ์‡ผํ•‘์ •๋ณด๋ฅผ ๊ณต์œ ํ•˜๋Š” ๊ณต๊ฐ„์ž…๋‹ˆ๋‹ค. [๋ฝ๋ฟŒ๊ฒŒ์‹œํŒ ์ด์šฉ๊ทœ์น™] +
+ + + + + + + + + + + + + + + + + + + +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
   ๋ฒˆํ˜ธ     ๊ธ€์“ด์ด  ์ œ๋ชฉ ๋“ฑ๋ก์ผ ์ถ”์ฒœ    ์กฐํšŒ   
๊ณต์ง€๊ด€๋ฆฌ์ž3[ํ•„๋…] ์ œ๋ชฉ์—๋Š” ํŠน์ˆ˜ ๋ฌธ์ž ์‚ฌ์šฉ์„ ์ ˆ๋Œ€ ๊ธˆ์ง€ํ•ฉ๋‹ˆ๋‹ค.- 2010๋…„ 2์›” 24์ผ0 4324677
์•Œ๋ฆผ๊ด€๋ฆฌ์ž์•„์ด์ฝ˜์•Œ๋ฆฌ์ต์Šคํ”„๋ ˆ์Šค ์ƒํ’ˆ์€ [์•Œ๋ฆฌ๋ฝ๋ฟŒ]์— ๊ณต์œ ํ•ด ์ฃผ์„ธ์š”!
์•Œ๋ฆผ๊ด€๋ฆฌ์ž์•„์ด์ฝ˜[ํ‚ค์›Œ๋“œ ์•Œ๋ฆผ] ๊ด€์‹ฌ ํ‚ค์›Œ๋“œ ๋“ฑ๋กํ•˜๊ณ , ์•ฑpush ์•Œ๋ฆผ ๋ฐ›์ž!
537951๋งค๋ ฅ์ ์ธ๋ฝ๋ฟŒ + +
+
+ [์ง€๋งˆ์ผ“]์ฒœ์—ฐ ํ–ฅ์ˆ˜ ๋น„๋ˆ„ ์‚ฌ๋ด‰ํŒŒํ“จ๋ฉ” 135g 3๊ฐœ+์‚ฌ์€ํ’ˆ (9,950์›/๋ฌด๋ฃŒ)1 +
+ [๊ธฐํƒ€] +
1069
537950123aaa + +
+
+ [์ง€๋งˆ์ผ“]์ˆœ๋ฉด ๊ตญ๋‚ด์ƒ์‚ฐ ๋ฌด์ง€ํ‹ฐ3์žฅ (์œ ํด12,410์›/๋ฌด๋ฐฐ) +
+ [์˜๋ฅ˜/์žกํ™”] +
1508
537949์œ ์œ ์ž์ ๋ฃจ + +
+
+ [๋„ค์ด๋ฒ„-๋ฏธ๋ฃจ๋ชฐ] ๋ฒจ๋ฃจ์•„์ฒด ์˜ฌ์ธ์› ํฌ๋งจ 4์ข… (29,900์›/๋ฌด๋ฃŒ)5 +
+ [ํ™”์žฅํ’ˆ] +
1201
537947illilliiill + +
+
+ [ํ‹ฐ๋ชฌ]์—ญ๋Œ€๊ฐ€ ํ•ซ๋”œ! ์Šˆํผ๋Œ€๋”” ์„ธ์ดํ”„ 77ํ‰๋Ÿ‰ ์•„๊ธฐ๋ฌผํ‹ฐ์Šˆ (17,991์›/๋ฌด๋ฐฐ)74  +
+ [๊ธฐํƒ€] +
9 - 0 14917
5379461ํšŒ์ œ๊ณต๋Ÿ‰ + +
+
+ [CGV์”จ๋„ค์ƒต]ํƒ€์นด๋ผํ† ๋ฏธ ๋ฒ„์ฆˆ๋ผ์ดํŠธ์ด์–ด 49,000 / 3,0006 +
+ [๊ธฐํƒ€] +
9522
537945๊ฐ€์„๋ง์ฐจ๋ผ๋–ผ + +
+
+ [์œ„๋ฉ”ํ”„]์ƒค์˜ค๋ฏธ ์Šค๋งˆํŠธ ์„ ํ’๊ธฐ 7์„ธ๋Œ€ 1X ์—…๊ทธ๋ ˆ์ด๋“œ/Type-C ์ถฉ์ „ ํฌํŠธ (43,160์›/๋ฌด๋ฐฐ)16 +
+ [๊ธฐํƒ€] +
1 - 0 17666
537944๊ฑฐ๋ถ๋ฐ˜ + +
+
+ [๋ฌด์‹ ์‚ฌ]FCMM ํ’‹๋ณผ ํ”„๋กœ ์ŠคํŠธ๋ ˆ์น˜ ์œˆ๋“œ๋ธŒ๋ ˆ์ด์ปค (32,364์›/๋ฌด๋ฃŒ)3 +
+ [์˜๋ฅ˜/์žกํ™”] +
13039
537942jimin1120 + +
+
+ ๋ถ์œ ๋Ÿฝ ์Šคํƒ€์ผ ์–‘์† ์Šคํ”„๋ณผ ๋„์ž๊ธฐ ์Šคํ”„๊ทธ๋ฆ‡ ์†Œ์Šค์ ‘์‹œ ์งฑ๊ตฌ๊ทธ๋ฆ‡ ๋ผ๋ฉด๊ทธ๋ฆ‡ ๋งŒ๋‘์ ‘์‹œ ํŠน๊ฐ€์ „(8,400์›~/๋ฌด๋ฃŒ)23 +
+ [๊ธฐํƒ€] +
2 - 0 5767
537941๋„ˆ๊ตฌ๋ฆฌ์‚ฌ๋ž‘ + +
+
+ [๋กฏ๋ฐ์˜จ]์ œ์Šคํ”„๋ฆฌ ๊ณจ๋“œํ‚ค์œ„, ํ† ๋ ˆํƒ€, ๊ฐ€์ž๋ฏธ๋ฐฅ์ƒ, ์ฟก๋ฐ˜ ์ƒˆ์šฐ๋ณถ์Œ๋ฐฅ(๋‹ค์–‘/๋ฌด๋ฃŒ)20 +
+ [์‹ํ’ˆ/๊ฑด๊ฐ•] +
16 - 0 10050
537940๊นกํŒจ๊ฑด๋‹ฌ + +
+
+ [์ž์‚ฌ๋ชฐ]ํ‘ธ๋งˆ ๋ฉ”์‰ฌ ํผ์ŠคํŠธ๋ผ์ดํ”„ ์—๋””์…˜ 5์ข… ์„ธํŠธ (18,220์›/๋ฌด๋ฃŒ)6 +
+ [์˜๋ฅ˜/์žกํ™”] +
4 - 0 7273
537939์‹ฌ์˜์ผ„ + +
+
+ [์œ„๋ฉ”ํ”„]์˜ค๋ฆฌ์˜จ ๋น„์ตธ๋น„ ์ดˆ์ฝ”๋น„์Šคํ‚ท5pX8๊ฐœ ์™ธ ์น˜์ฆˆํ†ก/๊ฐ์žํ†ก/์˜ค์ง•์–ดํ†ก,๋„๋„ํ•œ ๋‚˜์ตธ(13,220์›/๋ฌด๋ฃŒ)10 +
+ [์‹ํ’ˆ/๊ฑด๊ฐ•] +
8 - 0 7724
537938๋ฒŒ๊ฟ€์˜ค์†Œ๋ฆฌ๋Š”๊ฐ•ํ•ด + +
+
+ [๋„ค์ด๋ฒ„]์ถ”์–ต์˜ ์˜ค๋ž€๋‹ค 250g x2๋ด‰(6,200์›/๋ฌด๋ฐฐ)9 +
+ [์‹ํ’ˆ/๊ฑด๊ฐ•] +
5892
537937๋งค์นดaa + +
+
+ [G๋งˆ์ผ“]ํœด๋Œ€ํฐ ์ ค๋ฆฌ์ผ€์ด์Šค1+1 ๊ฐค๋Ÿญ์‹œ,์•„์ดํฐ (3,610์›/๋ฌด๋ฐฐ)5 +
+ [๊ธฐํƒ€] +
5186
537936๋‚˜๋ฌด007 + +
+
+ [CJ์˜จ์Šคํƒ€์ผ]๋™์› ์ง€์ค‘ํ•ด ํ™ฉ๋„ 410g 12์บ”(16,110/๋ฌด๋ฃŒ)44 +
+ [์‹ํ’ˆ/๊ฑด๊ฐ•] +
15 - 0 13237
537935IilliillIi + +
+
+ [๋„ค์ด๋ฒ„]๋งฅ์ฃผํšจ๋ชจ ๋น„์˜คํ‹ด ์•„๋ˆ„์นด์‚ฌ๊ณผ ํ”„๋ฆฌ๋ฏธ์—„ 1๋ฐ•์Šค (1๊ฐœ์›”๋ถ„) (34,900์›/๋ฌด๋ฃŒ)2  +
+ [์‹ํ’ˆ/๊ฑด๊ฐ•] +
3597
537934์ตœ๊ตฐ์–ด๋ญ‰ + +
+
+ [์œ„๋ฉ”ํ”„]๋ธ๋ฆฌ๋งŒ์ฅฌ ๋ถ•์–ด๋นต/ํ•ซ๋„๊ทธ 600g 2๋ด‰ (8,700/๋ฌด๋ฃŒ)15 +
+ [์‹ํ’ˆ/๊ฑด๊ฐ•] +
6 - 0 6264
537933๋ฌด๋…๋ฌด์ƒํ•ด + +
+
+ [์ธํ„ฐํŒŒํฌ]์‚ฐ๋ฆฌ์˜ค ๋ฏธ๋‹ˆ ๋ง๋ž‘์Šคํ‹ฐ์ปค/์ผ€์ดํฌํ•˜์šฐ์Šค ์™ธ ๋‹ค์ˆ˜ (2,970์›๋ถ€ํ„ฐ/3๊ฐœ ์ด์ƒ ์ฃผ๋ฌธ์‹œ ๋ฌด๋ฃŒ) +
+ [์œก์•„] +
1 - 0 6198
537931p1ano + +
+
+ [์œ„๋ฉ”ํ”„]๋น„์˜คํ‹ด ์šธํŠธ๋ผ 4๊ฐœ์›”๋ถ„ (4,900์›/๋ฌด๋ฐฐ)8  +
+ [์‹ํ’ˆ/๊ฑด๊ฐ•] +
3 - 0 6065
537930ํ‚น๊ฐ“์ œ๋„ค๋ŸดMZ + +
+
+ [์œ„๋ฉ”ํ”„]์•„์ŠคํŠธ๋กœ๊ธ€๋ผ์ด๋“œ์ ค 148ml 1+1 (27,470์›/๋ฌด๋ฃŒ๋ฐฐ์†ก)44 +
+ [๊ธฐํƒ€] +
1 - 0 16634
537929๊ฟ‰์ง€ + +
+
+ [์˜ฅ์…˜]๋‹ค๋†๋ฐ”์ด์˜ค ๊ฐ€๋ฌด์น˜์†Œ์ฃผ 43๋„ 375ml(32,900์›/๋ฌด๋ฃŒ)3 +
+ [์‹ํ’ˆ/๊ฑด๊ฐ•] +
1 - 0 5952
์‡ผํ•‘๋ฝ๋ฟŒ ์•„์ด์ฝ˜์•„์Šค๋ผ์Šฌ + +
+
+ [์นด์นด์˜คํ†ก ์Šคํ† ์–ด] ์บ˜๋นˆํด๋ผ์ธ ์ฝ”ํŠผ ๋ฐ˜ํŒ”ํ‹ฐ 15,900์› ์™ธ ํด๋กœ๋ฐ˜ํŒ”ํ‹ฐ, ๋ฐ˜๋ฐ”์ง€, PK์นด๋ผํ‹ฐ, ์›ํ”ผ์Šค, (๋ฌด๋ฐฐ,๋ฌด๊ต,๋ฌด๋ฐ˜) + 256 +
+
+ 		56 - 05325
์‡ผํ•‘๋ฝ๋ฟŒ ์•„์ด์ฝ˜LG์ „์ž์ฟ ํŒก๊ณต์‹๋Œ€ํ–‰... + +
+
+ [์ฟ ํŒก] LG ๊ทธ๋žจ ์ฃผ๋ง ํŠน๊ฐ€ ์ฐฌ์Šค! ์™€์šฐ ์ฆ‰์‹œํ• ์ธ+๋†ํ˜‘์นด๋“œ ์ถ”๊ฐ€ ํ• ์ธ+๋‚ด์ผ ๋ฐฐ์†ก ๋„์ฐฉ + 246 +
+
+ 		57 - 03128
+
+

+ ์‡ผํ•‘ํฌ๋Ÿผ๋ฝ๋ฟŒ์ƒ๋‹ด์‹ค ์•„์ด์ฝ˜๋” ๋งŽ์€ ์‡ผํ•‘ ์ •๋ณด์™€ ๊ตฌ๋งค ์ „ ์ž์œ ๋กœ์šด ์‡ผํ•‘ ์ด์•ผ๊ธฐ๋Š” ์‡ผํ•‘ํฌ๋Ÿผ์—์„œ! +

+
+
ํ•ซ๋”œ ์•„์ด์ฝ˜ + ์ฟ ํŒก + + +
+
+ ๋‚˜์ดํ‚ค ์—๋ธŒ๋ฆฌ๋ฐ์ด ์ฟ ์…˜ํฌ๋ฃจ ํŠธ๋ ˆ์ด๋‹์‚ญ์Šค ์žฅ๋ชฉ์–‘๋ง 1์„ธํŠธ 3์ผค๋ ˆ์ž… (9,490์›) +
+
+ 		7
์‡ผํ•‘ํฌ๋Ÿผ ์•„์ด์ฝ˜ + ll427ll + + +
+
+ [์•Œ๋ฆฌ]ํŽฉ์‹œ์ œ๋กœ ๋ผ์ž„ํ–ฅ 355ml 24์บ”(13,582์›/๋ฌด๋ฃŒ) +
+ [์‹ํ’ˆ/๊ฑด๊ฐ•]
+ 		907
์‡ผํ•‘ํฌ๋Ÿผ ์•„์ด์ฝ˜ + HBAF + + +
+
+  ๊ตญ์‚ฐ ๊ฐˆ์น˜ 1๋งˆ๋ฆฌ 39800์› +
+ [์‹ํ’ˆ/๊ฑด๊ฐ•]
+ 		1 - 010894
์‡ผํ•‘ํฌ๋Ÿผ ์•„์ด์ฝ˜ + ์‹œ๋ป˜๊ฑด๋ง› + + +
+
+  ๋‚จ์ž ์ง‘์—์„œ ์ž…์„ ๋ฐ˜๋ฐ”์ง€ ํ†ตํ’์ž˜๋˜๋Š”๊ฑฐ ์žˆ์„๊นŒ์š”? +
+ [์˜๋ฅ˜/์žกํ™”]
+ 		10935
+
+
+ +
+
+
+
+
+
+ + + + +
+ +
+
+

+ ์‹ค์‹œ๊ฐ„ ์งˆ๋ €์–ด์š” +
    +
  1. 1
  2. 2
+

+ +
    +
  • +
    • +
  • +
    • +
+
+ +
+

+ ์ตœ๊ทผ ๋‚ด๊ฐ€ ๋ณธ ์ƒํ’ˆ +
    +
+

+
    +
  • ์ตœ๊ทผ ๋‚ด๊ฐ€ ๋ณธ ์ƒํ’ˆ์ด ์—†์Šต๋‹ˆ๋‹ค.
+
+
+ + + + + + +
+ +  -์ƒˆ๋กœ๊ณ ์นจ  -๋‹ค์ŒํŽ˜์ด์ง€   +
+ + + + + + + +
+ + 1 2 3 4 5 6 7 8 9 10 ๋‹ค์Œ +
+ + + + + + + + + + + + +
+ + + + + +
+ + + + + + + + + +
+
+ + + + +
+ +
+ + +
+ + + + + + + +
+
+ + + +
+ +
+ + ์‚ฌ์ดํŠธ์†Œ๊ฐœ + ์šด์˜์ฐธ์—ฌ + + + ๊ฒŒ์‹œ์ค‘๋‹จ์š”์ฒญ + + ์ด์šฉ์•ฝ๊ด€ + ๊ฐœ์ธ์ •๋ณด์ฒ˜๋ฆฌ๋ฐฉ์นจ + ์ฒญ์†Œ๋…„๋ณดํ˜ธ์ •์ฑ… + ๋ถˆ๋ฒ•์ดฌ์˜๋ฌผ๋“ฑ ์‹ ๊ณ  + ๊ด‘๊ณ /์ œํœด + ๋ฉ”์ผ๋ฌธ์˜ + + + ๋ฝ๋ฟŒ์ฑ„์šฉ + + ๋‹คํฌ๋ชจ๋“œ +
+ + + + + +
+
+
+

๋ฝ๋ฟŒ๋Š” ์‡ผํ•‘ ์ •๋ณด ์ปค๋ฎค๋‹ˆํ‹ฐ์ž…๋‹ˆ๋‹ค. ์ƒํ’ˆ ์ฃผ๋ฌธ, ๋ฐฐ์†ก ๋ฐ ํ™˜๋ถˆ์˜ ์˜๋ฌด์™€ ์ฑ…์ž„์€ ๊ฐ ํŒ๋งค ์—…์ฒด์— ์žˆ์Šต๋‹ˆ๋‹ค.

+

(์ฃผ)๋ฝ๋ฟŒ์ปค๋ฎค๋‹ˆ์ผ€์ด์…˜ / ๋Œ€ํ‘œ์ด์‚ฌ : ๊ณ ์žฌ์„ฑ / ์‚ฌ์—…์ž๋“ฑ๋ก๋ฒˆํ˜ธ : 117-81-66008 / ํ†ต์‹ ํŒ๋งค์—…์‹ ๊ณ ๋ฒˆํ˜ธ : ์ œ2015-์„œ์šธ์˜๋“ฑํฌ-1441ํ˜ธ

+

์ฃผ์†Œ : ์„œ์šธํŠน๋ณ„์‹œ ์˜๋“ฑํฌ๊ตฌ ๋‹น์‚ฐ๋กœ 41๊ธธ ๋‹น์‚ฐ SK V1์„ผํ„ฐ W๋™ 1411ํ˜ธ + / ํŒฉ์Šค : 02-6280-7029 / ์‚ฌ์—…์žํšŒ์› FAQ

+ +
+ + Copyright 2005-2024 ppomppu. All rights reserved +
+ +
+
+ + + + + + + + + + + + + + + + + diff --git a/support/build.gradle b/support/build.gradle new file mode 100644 index 0000000..53479a9 --- /dev/null +++ b/support/build.gradle @@ -0,0 +1,23 @@ +dependencies { + compileOnly 'org.projectlombok:lombok' + + // https://projectreactor.io/docs/core/release/reference/#debug-activate + annotationProcessor 'org.projectlombok:lombok' + implementation 'org.springframework.boot:spring-boot-starter-web' + +} + +test { + useJUnitPlatform() + testLogging { + events "passed", "skipped", "failed" + } +} + +jar { + enabled = true +} + +bootJar { + enabled = false +} \ No newline at end of file diff --git a/support/next_message_tempalte(block_kit_builder).json b/support/next_message_tempalte(block_kit_builder).json new file mode 100644 index 0000000..eae1772 --- /dev/null +++ b/support/next_message_tempalte(block_kit_builder).json @@ -0,0 +1,31 @@ +{ + "blocks": [ + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "๊ธฐํƒ€)) `[์ฟ ํŒก] ํƒ€ํ‘ธ์ฝ” ์ด์ค‘ ์ง„๊ณต ์Šคํ… ์„ธ๋ผ๋ฏน ์ฝ”ํŒ… ํ…€๋ธ”๋Ÿฌ (ํ™”์ดํŠธ๋งŒ 11,130์›/๋กœ์ผ“์™€์šฐ๋ฌด๋ฃŒ) 4 [๊ธฐํƒ€]`" + }, + "accessory": { + "type": "image", + "image_url": "cdn.ppomppu.co.kr/zboard/data3/2021/1121/m_20211121184835_zfyahnow.png", + "alt_text": "alt text for image" + } + }, + { + "type": "divider" + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "๊ธฐํƒ€)) `[๋กฏ๋ฐ์˜จ] 1๋งค 149์› ์˜ฌ๊ตญ์‚ฐ KF94์ƒˆ๋ถ€๋ฆฌ๋งˆ์Šคํฌ ๋Œ€ํ˜• 200๋งค (29,920์›/๋ฌด๋ฃŒ๋ฐฐ์†ก) 58 [๊ธฐํƒ€]`" + }, + "accessory": { + "type": "image", + "image_url": "https://s3-media3.fl.yelpcdn.com/bphoto/c7ed05m9lC2EmA3Aruue7A/o.jpg", + "alt_text": "alt text for image" + } + } + ] +} \ No newline at end of file diff --git a/support/src/main/java/com/myoa/engineering/crawl/shopping/support/dto/APIResponse.java b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/dto/APIResponse.java new file mode 100644 index 0000000..3c20af2 --- /dev/null +++ b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/dto/APIResponse.java @@ -0,0 +1,63 @@ +package com.myoa.engineering.crawl.shopping.support.dto; + +import java.io.Serializable; +import java.util.Map; +import lombok.Getter; + +/** + * APIResponse + * + * @author Shin Woo-jin (woozu.shin@kakaoent.com) + * @since 2021-09-07 + */ +@Getter +public class APIResponse implements Serializable { + private static final long serialVersionUID = 1523350013713908487L; + + private boolean success; + private T result; + private APIResponseError error; + + public APIResponse(boolean success, T result, APIResponseError error) { + this.success = success; + this.error = error; + this.result = result; + } + + public APIResponse(boolean success, T result) { + this.success = success; + this.result = result; + } + + public APIResponse(boolean success, APIResponseError error) { + this.success = success; + this.error = error; + } + + public APIResponse() {} + + public static APIResponse success(T result) { + return new APIResponse<>(true, result); + } + + public static APIResponse success() { + return new APIResponse<>(true, null); + } + + public static APIResponse fail(T result, String code) { + return new APIResponse(false, result, APIResponseError.of(code)); + } + + public static APIResponse fail(String code) { + return new APIResponse(false, APIResponseError.of(code)); + } + + public static APIResponse fail(String code, String message) { + return new APIResponse(false, APIResponseError.of(code, message)); + } + + public static APIResponse fail(String code, String message, Map reasons) { + return new APIResponse(false, APIResponseError.of(code, message, reasons)); + } + +} diff --git a/support/src/main/java/com/myoa/engineering/crawl/shopping/support/dto/APIResponseError.java b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/dto/APIResponseError.java new file mode 100644 index 0000000..a73502f --- /dev/null +++ b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/dto/APIResponseError.java @@ -0,0 +1,49 @@ +package com.myoa.engineering.crawl.shopping.support.dto; + +import java.io.Serializable; +import java.util.HashMap; +import java.util.Map; +import lombok.Getter; + +/** + * ResponseError + * + * @author Shin Woo-jin (woozu.shin@kakaoent.com) + * @since 2021-09-07 + */ +@Getter +public class APIResponseError implements Serializable { + + private static final long serialVersionUID = 931593091836887301L; + + private String code; + private String message; + private Map reasons; + + public APIResponseError() {} + + public APIResponseError(String code, String message, Map reasons) { + this.code = code; + this.message = message; + this.reasons = reasons; + } + + public static APIResponseError of(String code, String message, Map reasons) { + return new APIResponseError<>(code, message, reasons); + } + + public static APIResponseError of( + String code, String message, String reasonKey, String reasonValue) { + final Map reasons = new HashMap<>(); + reasons.put(reasonKey, reasonValue); + return new APIResponseError(code, message, reasons); + } + + public static APIResponseError of(String code, String message) { + return new APIResponseError<>(code, message, null); + } + + public static APIResponseError of(String code) { + return new APIResponseError<>(code, null, null); + } +} diff --git a/support/src/main/java/com/myoa/engineering/crawl/shopping/support/dto/BlockMessageDTO.java b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/dto/BlockMessageDTO.java new file mode 100644 index 0000000..7784d5a --- /dev/null +++ b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/dto/BlockMessageDTO.java @@ -0,0 +1,67 @@ +package com.myoa.engineering.crawl.shopping.support.dto; + +import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName; +import lombok.Builder; +import lombok.Getter; +import lombok.NoArgsConstructor; + +import java.io.Serializable; +import java.time.Instant; +import java.util.List; + +/** + * SimpleMessageDTO + * + * @author Shin Woo-jin (woo-jin.shin@linecorp.com) + * @since 2021-11-21 + */ +@Getter +@NoArgsConstructor +public class BlockMessageDTO implements Serializable { + + private static final long serialVersionUID = -6992039884035135523L; + + private PpomppuBoardName boardName; + private String title; + private List blocks; + private String url; + private Instant publishedAt; + private Instant requestedAt; + + @Builder + public BlockMessageDTO(PpomppuBoardName boardName, String title, + List blocks, String url, Instant publishedAt, + Instant requestedAt) { + this.boardName = boardName; + this.title = title; + this.blocks = blocks; + this.url = url; + this.publishedAt = publishedAt; + this.requestedAt = requestedAt; + } + + @Getter + @NoArgsConstructor + public static class Block implements Serializable { + private static final long serialVersionUID = 3633781631892663709L; + + private String text; + private String imageUrl; + private String altText; + + public Block(String text, String imageUrl, String altText) { + this.text = text; + this.imageUrl = imageUrl; + this.altText = altText; + } + } + + public static Block createBlock(String text, String imageUrl) { + return new Block(text, imageUrl, ""); + } + + public static Block createBlock(String text, String imageUrl, String altText) { + return new Block(text, imageUrl, altText); + } + +} diff --git a/support/src/main/java/com/myoa/engineering/crawl/shopping/support/dto/SimpleMessageDTO.java b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/dto/SimpleMessageDTO.java new file mode 100644 index 0000000..033e97e --- /dev/null +++ b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/dto/SimpleMessageDTO.java @@ -0,0 +1,37 @@ + package com.myoa.engineering.crawl.shopping.support.dto; + +import java.io.Serializable; +import java.time.Instant; + +import lombok.Builder; +import lombok.Getter; +import lombok.NoArgsConstructor; + +/** + * SimpleMessageDTO + * @author Shin Woo-jin (woo-jin.shin@linecorp.com) + * @since 2021-11-21 + * + */ +@Getter +@NoArgsConstructor +public class SimpleMessageDTO implements Serializable { + + private static final long serialVersionUID = 2203955567672404428L; + + private String title; + private String body; + private String url; + private Instant publishedAt; + private Instant requestedAt; + + @Builder + public SimpleMessageDTO(String title, String body, String url, Instant publishedAt, Instant requestedAt) { + this.title = title; + this.body = body; + this.url = url; + this.publishedAt = publishedAt; + this.requestedAt = requestedAt; + } + +} diff --git a/support/src/main/java/com/myoa/engineering/crawl/shopping/support/dto/constant/CrawlTarget.java b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/dto/constant/CrawlTarget.java new file mode 100644 index 0000000..11ba3e5 --- /dev/null +++ b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/dto/constant/CrawlTarget.java @@ -0,0 +1,13 @@ +package com.myoa.engineering.crawl.shopping.support.dto.constant; + +import lombok.AllArgsConstructor; +import lombok.Getter; + +@Getter +@AllArgsConstructor +public enum CrawlTarget { + PPOMPPU_DOMESTIC, + PPOMPPU_OVERSEA, + FMKOREA, + ; +} diff --git a/support/src/main/java/com/myoa/engineering/crawl/shopping/support/dto/constant/PpomppuBoardName.java b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/dto/constant/PpomppuBoardName.java new file mode 100644 index 0000000..cd6ac23 --- /dev/null +++ b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/dto/constant/PpomppuBoardName.java @@ -0,0 +1,62 @@ +package com.myoa.engineering.crawl.shopping.support.dto.constant; + +import lombok.AllArgsConstructor; +import lombok.Getter; + +import java.util.Arrays; + +/** + * PpomppuBoardName + * + * @author Shin Woo-jin (woo-jin.shin@linecorp.com) + * @since 2021-09-05 + */ +@Getter +@AllArgsConstructor +public enum PpomppuBoardName { + UNKNOWN("", "UNKNOWN", false, false), + PPOMPPU_DOMESTIC_ALL("id=ppomppu", "์ „์ฒด", false, true), + PPOMPPU_DOMESTIC_ETC("id=ppomppu&category=1", "๊ธฐํƒ€", true, true), + PPOMPPU_DOMESTIC_COMPUTER("id=ppomppu&category=4", "์ปดํ“จํ„ฐ", true, true), + PPOMPPU_DOMESTIC_DIGITAL("id=ppomppu&category=5", "๋””์ง€ํ„ธ", true, true), + PPOMPPU_DOMESTIC_FOOD("id=ppomppu&category=6", "์‹ํ’ˆ/๊ฑด๊ฐ•", true, true), + PPOMPPU_DOMESTIC_BOOK("id=ppomppu&category=8", "์„œ์ ", true, true), + PPOMPPU_DOMESTIC_APPLIANCES("id=ppomppu&category=9", "๊ฐ€์ „/๊ฐ€๊ตฌ", true, true), + PPOMPPU_DOMESTIC_PARENTING("id=ppomppu&category=10", "์œก์•„", true, true), + PPOMPPU_DOMESTIC_GIFTCARD("id=ppomppu&category=11", "์ƒํ’ˆ๊ถŒ", true, true), + PPOMPPU_DOMESTIC_CLOTHES("id=ppomppu&category=12", "์˜๋ฅ˜/์žกํ™”", true, true), + PPOMPPU_DOMESTIC_COSMETIC("id=ppomppu&category=13", "ํ™”์žฅํ’ˆ", true, true), + PPOMPPU_DOMESTIC_OUTDOOR("id=ppomppu&category=15", "๋“ฑ์‚ฐ/์บ ํ•‘", true, true), + PPOMPPU_OVERSEA_ALL("id=ppomppu4", "์ „์ฒด", false, false), + PPOMPPU_OVERSEA_ETC("id=ppomppu4&category=1", "๊ธฐํƒ€", true, false), + PPOMPPU_OVERSEA_APPLIANCES("id=ppomppu4&category=7", "๊ฐ€์ „", true, false), + PPOMPPU_OVERSEA_TVAV("id=ppomppu4&category=8", "TV/์˜์ƒ", true, false), + PPOMPPU_OVERSEA_COMPUTER("id=ppomppu4&category=3", "์ปดํ“จํ„ฐ", true, false), + PPOMPPU_OVERSEA_DIGITAL("id=ppomppu4&category=4", "๋””์ง€ํ„ธ", true, false), + PPOMPPU_OVERSEA_MOBILEACCESSORY("id=ppomppu4&category=9", "์•ก์„ธ์„œ๋ฆฌ", false, true), + PPOMPPU_OVERSEA_CLOTHES("id=ppomppu4&category=5", "์˜๋ฅ˜/์žกํ™”", true, false), + PPOMPPU_OVERSEA_WATCH("id=ppomppu4&category=2", "์‹œ๊ณ„", true, false), + PPOMPPU_OVERSEA_SHOES("id=ppomppu4&category=11", "์‹ ๋ฐœ", true, false), + PPOMPPU_OVERSEA_FOOD("id=ppomppu4&category=10", "์‹ํ’ˆ/๊ฑด๊ฐ•", true, false), + PPOMPPU_OVERSEA_PARENTING("id=ppomppu4&category=6", "์œก์•„", true, false), + ; + + private String resourcePath; + private String menuName; + private boolean crawlWithDefaultTimer; + private boolean isDomestic; + + public static final String PPOMPPU_URL = "https://www.ppomppu.co.kr"; + + public static String ofViewPageUrl(String articleUrl) { + return PPOMPPU_URL + "/zboard/" + articleUrl; + } + + public static PpomppuBoardName ofBoardName(String boardName, boolean isDomestic) { + return Arrays.stream(values()) + .filter(e -> e.getMenuName().equals(boardName) && e.isDomestic() == isDomestic) + .findFirst() + .orElse(UNKNOWN); + } + +} diff --git a/support/src/main/java/com/myoa/engineering/crawl/shopping/support/util/DateUtil.java b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/util/DateUtil.java new file mode 100644 index 0000000..b33749a --- /dev/null +++ b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/util/DateUtil.java @@ -0,0 +1,18 @@ +package com.myoa.engineering.crawl.shopping.support.util; + +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; + +/** + * DateUtil + * @author Shin Woo-jin (woo-jin.shin@linecorp.com) + * @since 2022-01-02 + * + */ +public final class DateUtil { + + private DateUtil() { } + + public static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yy.MM.dd HH:mm:ss") + .withZone(ZoneId.of("Asia/Seoul")); +} diff --git a/support/src/main/java/com/myoa/engineering/crawl/shopping/support/util/NumberUtil.java b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/util/NumberUtil.java new file mode 100644 index 0000000..1b36154 --- /dev/null +++ b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/util/NumberUtil.java @@ -0,0 +1,7 @@ +package com.myoa.engineering.crawl.shopping.support.util; + +public final class NumberUtil { + + private NumberUtil() {} + +} diff --git a/support/src/main/java/com/myoa/engineering/crawl/shopping/support/util/ObjectMapperFactory.java b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/util/ObjectMapperFactory.java new file mode 100644 index 0000000..f1404d0 --- /dev/null +++ b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/util/ObjectMapperFactory.java @@ -0,0 +1,102 @@ +package com.myoa.engineering.crawl.shopping.support.util; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.JsonParser.Feature; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.databind.ser.std.StdSerializer; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import java.io.IOException; +import org.springframework.cache.support.NullValue; +import org.springframework.util.StringUtils; + +/** + * ObjectMapperFactory + * + * @author Shin Woo-jin (woozu.shin@kakaoent.com) + * @since 2021-08-31 + */ +public final class ObjectMapperFactory { + + private ObjectMapperFactory() { + } + + private static final ObjectMapper defaultMapper; + private static final ObjectMapper httpMapper; + + static { + defaultMapper = initDefaultMapper(); + httpMapper = initHttpMapper(); + } + + public static ObjectMapper defaultMapper() { + return defaultMapper; + } + + public static ObjectMapper httpMapper() { + return httpMapper; + } + + private static ObjectMapper initDefaultMapper() { + final ObjectMapper objectMapper = new ObjectMapper(); + objectMapper.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES); + objectMapper.disable(SerializationFeature.FAIL_ON_EMPTY_BEANS); + objectMapper.disable(SerializationFeature.FAIL_ON_UNWRAPPED_TYPE_IDENTIFIERS); + objectMapper.enable(DeserializationFeature.READ_UNKNOWN_ENUM_VALUES_AS_NULL); + objectMapper.configure(Feature.ALLOW_UNQUOTED_CONTROL_CHARS, true); + objectMapper.enable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS); + + objectMapper.registerModule(new JavaTimeModule()); + return objectMapper; + } + + private static ObjectMapper initHttpMapper() { + final ObjectMapper objectMapper = initDefaultMapper(); + objectMapper.configure(DeserializationFeature.READ_UNKNOWN_ENUM_VALUES_USING_DEFAULT_VALUE, true); + return objectMapper; + } + + public static String writeAsString(Object o) { + try { + return defaultMapper().writeValueAsString(o); + } catch (JsonProcessingException e) { + e.printStackTrace(); + return null; + } + } + + + /** + * Copy from {@link GenericJackson2JsonRedisSerializer.NullValueSerializer}. + */ + public static class NullValueSerializer extends StdSerializer { + + private static final long serialVersionUID = 6776419544239897328L; + private final String classIdentifier; + + /** + * @param classIdentifier can be {@literal null} and will be defaulted to {@code @class}. + */ + NullValueSerializer(String classIdentifier) { + super(NullValue.class); + this.classIdentifier = StringUtils.hasText(classIdentifier) ? classIdentifier : "@class"; + } + + /* + * (non-Javadoc) + * @see com.fasterxml.jackson.databind.ser.std.StdSerializer#serialize(java.lang.Object, com + * .fasterxml.jackson.core.JsonGenerator, com.fasterxml.jackson.databind.SerializerProvider) + */ + @Override + public void serialize(NullValue value, JsonGenerator jgen, SerializerProvider provider) + throws IOException { + + jgen.writeStartObject(); + jgen.writeStringField(classIdentifier, NullValue.class.getName()); + jgen.writeEndObject(); + } + } +} diff --git a/support/src/main/java/com/myoa/engineering/crawl/shopping/support/util/ObjectUtil.java b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/util/ObjectUtil.java new file mode 100644 index 0000000..49322ac --- /dev/null +++ b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/util/ObjectUtil.java @@ -0,0 +1,15 @@ +package com.myoa.engineering.crawl.shopping.support.util; + +public final class ObjectUtil { + + private ObjectUtil() {} + + public static boolean isEmpty(Object o) { + return o == null; + } + + public static boolean isNotEmpty(Object o) { + return !isEmpty(o); + } + +} diff --git a/support/src/main/java/com/myoa/engineering/crawl/shopping/support/util/WebRequestUtil.java b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/util/WebRequestUtil.java new file mode 100644 index 0000000..00675df --- /dev/null +++ b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/util/WebRequestUtil.java @@ -0,0 +1,15 @@ +package com.myoa.engineering.crawl.shopping.support.util; + +/** + * WebUtil + * + * @author Shin Woo-jin (woozu.shin@kakaoent.com) + * @since 2021-09-08 + */ +public final class WebRequestUtil { + + private WebRequestUtil() {} + + public static final String HEADER_USER_AGENT_KEY = "User-Agent"; + public static final String HEADER_USER_AGENT_VALUE = "Mozilla/5.0 (Linux; Android 8.0.0; SM-G960F Build/R16NW) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.84 Mobile Safari/537.36"; +} diff --git a/support/src/main/java/com/myoa/engineering/crawl/shopping/support/webclient/WebClientBaseScan.java b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/webclient/WebClientBaseScan.java new file mode 100644 index 0000000..0695933 --- /dev/null +++ b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/webclient/WebClientBaseScan.java @@ -0,0 +1,4 @@ +package com.myoa.engineering.crawl.shopping.support.webclient; + +public interface WebClientBaseScan { +} diff --git a/support/src/main/java/com/myoa/engineering/crawl/shopping/support/webclient/factory/WebClientFilterFactory.java b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/webclient/factory/WebClientFilterFactory.java new file mode 100644 index 0000000..cd8c29e --- /dev/null +++ b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/webclient/factory/WebClientFilterFactory.java @@ -0,0 +1,46 @@ +package com.myoa.engineering.crawl.shopping.support.webclient.factory; + +import lombok.extern.slf4j.Slf4j; + +/** + * WebClientFilterFactory + * + * @author Shin Woo-jin (woozu.shin@kakaoent.com) + * @since 2021-09-07 + */ +@Slf4j +public final class WebClientFilterFactory { + + /* + private WebClientFilterFactory() {} + + public static ExchangeFilterFunction logRequest() { + return ExchangeFilterFunction.ofRequestProcessor(WebClientFilterFactory::writeRequest); + } + + public static ExchangeFilterFunction logResponse() { + return ExchangeFilterFunction.ofResponseProcessor(WebClientFilterFactory::writeResponse); + } + + private static Mono writeRequest(ClientRequest clientRequest) { + try { + log.info("[WEBCLIENT REQUEST] uri : {} method : {} headers : {}, body: {}", + clientRequest.url(), clientRequest.method(), clientRequest.headers(), clientRequest.body()); + } catch (Exception e) { + log.error("[WEBCLIENT REQUEST] write request failed", e); + } + return Mono.just(clientRequest); + } + + private static Mono writeResponse(ClientResponse clientResponse) { + try { + log.info("[WEBCLIENT RESPONSE] statusCode : {} headers : {}", + clientResponse.rawStatusCode(), clientResponse.headers().asHttpHeaders()); + } catch (Exception e) { + log.error("[WEBCLIENT RESPONSE] write response failed", e); + } + return Mono.just(clientResponse); + } + */ + +} diff --git a/support/src/main/java/com/myoa/engineering/crawl/shopping/support/webclient/factory/WebFluxExchangeStragiesFactory.java b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/webclient/factory/WebFluxExchangeStragiesFactory.java new file mode 100644 index 0000000..a14ab39 --- /dev/null +++ b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/webclient/factory/WebFluxExchangeStragiesFactory.java @@ -0,0 +1,42 @@ +package com.myoa.engineering.crawl.shopping.support.webclient.factory; + +/** + * WebFluxExchangeStragiesFactory + * + * @author Shin Woo-jin (woozu.shin@kakaoent.com) + * @since 2021-09-08 + */ +public final class WebFluxExchangeStragiesFactory { +/* + private WebFluxExchangeStragiesFactory() {} + + public static ExchangeStrategies ofDefault() { + final ObjectMapper mapper = ObjectMapperFactory.defaultMapper(); + return ExchangeStrategies.builder() + .codecs(configurer -> { + configurer.defaultCodecs().maxInMemorySize(-1); + configurer.defaultCodecs() + .jackson2JsonEncoder(new Jackson2JsonEncoder(mapper, + MimeTypeUtils.APPLICATION_JSON)); + configurer.defaultCodecs() + .jackson2JsonDecoder(new Jackson2JsonDecoder(mapper, + MimeTypeUtils.APPLICATION_JSON)); + }) + .build(); + } + public static ExchangeStrategies ofTextHtml() { + final ObjectMapper mapper = ObjectMapperFactory.defaultMapper(); + return ExchangeStrategies.builder() + .codecs(configurer -> { + configurer.defaultCodecs().maxInMemorySize(-1); + configurer.defaultCodecs() + .jackson2JsonEncoder(new Jackson2JsonEncoder(mapper, + MimeTypeUtils.TEXT_HTML)); + configurer.defaultCodecs() + .jackson2JsonDecoder(new Jackson2JsonDecoder(mapper, + MimeTypeUtils.TEXT_HTML)); + }) + .build(); + } +*/ +} diff --git a/support/src/main/java/com/myoa/engineering/crawl/shopping/support/webclient/properties/WebClientProperties.java b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/webclient/properties/WebClientProperties.java new file mode 100644 index 0000000..6b893f8 --- /dev/null +++ b/support/src/main/java/com/myoa/engineering/crawl/shopping/support/webclient/properties/WebClientProperties.java @@ -0,0 +1,44 @@ +package com.myoa.engineering.crawl.shopping.support.webclient.properties; + +import java.util.ArrayList; +import java.util.List; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.lang.NonNull; +import org.springframework.stereotype.Component; + +import lombok.Data; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +/** + * WebClientPropertiesUnit + * @author Shin Woo-jin (woo-jin.shin@linecorp.com) + * @since 2021-11-18 + * + */ +@NoArgsConstructor +@Setter +@Getter +@Component +@ConfigurationProperties(prefix = "webclient") +public class WebClientProperties { + + private List units = new ArrayList<>(); + + @Data + public static class WebClientPropertiesUnit { + private String unitName; + private String baseUrl; + // TODO headers + } + + public WebClientPropertiesUnit find(@NonNull String unitName) { + return units.stream() + .filter(x -> x.getUnitName().equals(unitName)) + .findFirst() + .orElseThrow( + () -> new IllegalArgumentException("Not found properties unit. unitName : " + unitName)); + } +} diff --git a/support/src/main/resources/application.yml b/support/src/main/resources/application.yml new file mode 100644 index 0000000..e69de29 diff --git a/support/src/main/resources/logback-spring.xml b/support/src/main/resources/logback-spring.xml new file mode 100644 index 0000000..f0fcb6c --- /dev/null +++ b/support/src/main/resources/logback-spring.xml @@ -0,0 +1,11 @@ + + + + + + + + + + + \ No newline at end of file diff --git a/support/src/main/resources/logback/component/logback-file.xml b/support/src/main/resources/logback/component/logback-file.xml new file mode 100644 index 0000000..65e54db --- /dev/null +++ b/support/src/main/resources/logback/component/logback-file.xml @@ -0,0 +1,23 @@ + + + + + ${DIRECTORY}/${LOG_FILE_BASE}_log + + ${DIRECTORY}/${LOG_FILE_BASE}_log.%d{yyyyMMdd}.%i + + 1000MB + + 60 + + + ${FILE_LOG_PATTERN} + ${IMMEDIATE_FLUSH} + + + + 1024 + + + diff --git a/support/src/main/resources/logback/logback-development.xml b/support/src/main/resources/logback/logback-development.xml new file mode 100644 index 0000000..ed84966 --- /dev/null +++ b/support/src/main/resources/logback/logback-development.xml @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + + + + diff --git a/support/src/main/resources/logback/logback-production.xml b/support/src/main/resources/logback/logback-production.xml new file mode 100644 index 0000000..df845f5 --- /dev/null +++ b/support/src/main/resources/logback/logback-production.xml @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + + + +