Remove directory

This commit is contained in:
woozu-shin 2024-05-09 09:34:39 +09:00
parent 99bdae2ed9
commit 4d7bf4fb3c
113 changed files with 5006 additions and 0 deletions

42
.gitignore vendored Normal file
View File

@ -0,0 +1,42 @@
HELP.md
.gradle
build/
!gradle/wrapper/gradle-wrapper.jar
!**/src/main/**/build/
!**/src/test/**/build/
### STS ###
.apt_generated
.classpath
.factorypath
.project
.settings
.springBeans
.sts4-cache
bin/
!**/src/main/**/bin/
!**/src/test/**/bin/
### IntelliJ IDEA ###
.idea
*.iws
*.iml
*.ipr
out/
!**/src/main/**/out/
!**/src/test/**/out/
### NetBeans ###
/nbproject/private/
/nbbuild/
/dist/
/nbdist/
/.nb-gradle/
### VS Code ###
.vscode/
temppassword.yml
data.sql
**/src/main/resources/slack
**/src/main/resources/datasource

9
LICENSE Normal file
View File

@ -0,0 +1,9 @@
MIT License
Copyright (c) <year> <copyright holders>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

46
PpomppuNotifier_DB.sql Normal file
View File

@ -0,0 +1,46 @@
create table "ppomppu_article"
(
"id" bigint generated by default as identity,
"article_id" bigint,
"article_url" varchar(255),
"board_name" integer,
"hit" integer,
"recommended" integer,
"registered_at" timestamp,
"title" varchar(255),
primary key ("id")
)
create table "ppomppu_board_feed_status"
(
"id" bigint generated by default as identity,
"board_name" integer,
"latest_parsed_article_id" bigint,
"updated_at" timestamp,
primary key ("id")
)
create table "published_history"
(
"id" bigint generated by default as identity,
"board_name_list" varchar(255),
"published_at" timestamp,
"user_id" bigint,
primary key ("id")
)
create table "subscribed_board"
(
"id" bigint generated by default as identity,
"board_name" integer,
"user_id" bigint,
primary key ("id")
)
create table "subscribed_user"
(
"id" bigint generated by default as identity,
"registered_at" timestamp,
"user_id" bigint,
primary key ("id")
)

54
PpomppuNotifier_ERD.puml Normal file
View File

@ -0,0 +1,54 @@
@startuml
'https://plantuml.com/sequence-diagram
class SubscribedUser {
- id
+ user_id
+ registered_at
+ created_at
+ modified_at
}
class SubscribedBoard {
- id
- user_id
+ board_name
+ created_at
+ modified_at
}
class PublishedHisotry {
- id
+ user_id
+ board_name_list
+ published_at
+ created_at
+ modified_at
}
class PpomppuArticle {
- id
+ article_id
+ board_name
+ article_url
+ title
+ recommended
+ hit
+ registered_at
+ created_at
+ modified_at
}
class PpomppuBoardFeedStatus {
- id
+ board_name
+ latest_parsed_article_id
+ updated_at
+ created_at
+ modified_at
}
SubscribedUser --o{ SubscribedBoard
SubscribedUser --o{ PublishedHisotry
@enduml

2
README.md Normal file
View File

@ -0,0 +1,2 @@
# PpomppuNotifier

70
build.gradle Normal file
View File

@ -0,0 +1,70 @@
plugins {
id 'java'
id 'idea'
id 'org.springframework.boot' version '2.5.4'
id 'io.spring.dependency-management' version '1.0.11.RELEASE'
}
group = 'com.myoa.engineering.crawl.shopping'
version = '1.1.1'
sourceCompatibility = JavaVersion.VERSION_17
configurations {
compileOnly {
extendsFrom annotationProcessor
}
}
repositories {
mavenCentral()
}
allprojects {
group = 'com.myoa.engineering.crawl.shopping'
version = '1.1.1'
apply plugin: 'java'
apply plugin: 'idea'
apply plugin: 'org.springframework.boot'
apply plugin: 'io.spring.dependency-management'
repositories {
mavenCentral()
maven {
url "https://plugins.gradle.org/m2/"
}
}
ext {
set('springCloudVersion', "2020.0.4")
}
dependencyManagement {
imports {
mavenBom "org.springframework.cloud:spring-cloud-dependencies:${springCloudVersion}"
}
}
/* task initSourceFolders {
sourceSets*.java.srcDirs*.each {
if( !it.exists() ) {
it.mkdirs()
}
}
sourceSets*.resources.srcDirs*.each {
if( !it.exists() ) {
it.mkdirs()
}
}
}*/
}
jar {
enabled = false
}
bootJar {
enabled = false
}

3
copy.bat Normal file
View File

@ -0,0 +1,3 @@
xcopy /y .\processor\build\libs\*.jar .\
xcopy /y .\receiver\build\libs\*.jar .\
xcopy /y .\sender\build\libs\*.jar .\

BIN
gradle/wrapper/gradle-wrapper.jar vendored Normal file

Binary file not shown.

View File

@ -0,0 +1,6 @@
#Sun Apr 28 23:47:38 KST 2024
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-8.5-bin.zip
zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists

185
gradlew vendored Executable file
View File

@ -0,0 +1,185 @@
#!/usr/bin/env sh
#
# Copyright 2015 the original author or authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
##############################################################################
##
## Gradle start up script for UN*X
##
##############################################################################
# Attempt to set APP_HOME
# Resolve links: $0 may be a link
PRG="$0"
# Need this for relative symlinks.
while [ -h "$PRG" ] ; do
ls=`ls -ld "$PRG"`
link=`expr "$ls" : '.*-> \(.*\)$'`
if expr "$link" : '/.*' > /dev/null; then
PRG="$link"
else
PRG=`dirname "$PRG"`"/$link"
fi
done
SAVED="`pwd`"
cd "`dirname \"$PRG\"`/" >/dev/null
APP_HOME="`pwd -P`"
cd "$SAVED" >/dev/null
APP_NAME="Gradle"
APP_BASE_NAME=`basename "$0"`
# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
# Use the maximum available, or set MAX_FD != -1 to use that value.
MAX_FD="maximum"
warn () {
echo "$*"
}
die () {
echo
echo "$*"
echo
exit 1
}
# OS specific support (must be 'true' or 'false').
cygwin=false
msys=false
darwin=false
nonstop=false
case "`uname`" in
CYGWIN* )
cygwin=true
;;
Darwin* )
darwin=true
;;
MSYS* | MINGW* )
msys=true
;;
NONSTOP* )
nonstop=true
;;
esac
CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
# Determine the Java command to use to start the JVM.
if [ -n "$JAVA_HOME" ] ; then
if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
# IBM's JDK on AIX uses strange locations for the executables
JAVACMD="$JAVA_HOME/jre/sh/java"
else
JAVACMD="$JAVA_HOME/bin/java"
fi
if [ ! -x "$JAVACMD" ] ; then
die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
Please set the JAVA_HOME variable in your environment to match the
location of your Java installation."
fi
else
JAVACMD="java"
which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
Please set the JAVA_HOME variable in your environment to match the
location of your Java installation."
fi
# Increase the maximum file descriptors if we can.
if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then
MAX_FD_LIMIT=`ulimit -H -n`
if [ $? -eq 0 ] ; then
if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then
MAX_FD="$MAX_FD_LIMIT"
fi
ulimit -n $MAX_FD
if [ $? -ne 0 ] ; then
warn "Could not set maximum file descriptor limit: $MAX_FD"
fi
else
warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT"
fi
fi
# For Darwin, add options to specify how the application appears in the dock
if $darwin; then
GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\""
fi
# For Cygwin or MSYS, switch paths to Windows format before running java
if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then
APP_HOME=`cygpath --path --mixed "$APP_HOME"`
CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
JAVACMD=`cygpath --unix "$JAVACMD"`
# We build the pattern for arguments to be converted via cygpath
ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`
SEP=""
for dir in $ROOTDIRSRAW ; do
ROOTDIRS="$ROOTDIRS$SEP$dir"
SEP="|"
done
OURCYGPATTERN="(^($ROOTDIRS))"
# Add a user-defined pattern to the cygpath arguments
if [ "$GRADLE_CYGPATTERN" != "" ] ; then
OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)"
fi
# Now convert the arguments - kludge to limit ourselves to /bin/sh
i=0
for arg in "$@" ; do
CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -`
CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option
if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition
eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"`
else
eval `echo args$i`="\"$arg\""
fi
i=`expr $i + 1`
done
case $i in
0) set -- ;;
1) set -- "$args0" ;;
2) set -- "$args0" "$args1" ;;
3) set -- "$args0" "$args1" "$args2" ;;
4) set -- "$args0" "$args1" "$args2" "$args3" ;;
5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;;
6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;;
7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;;
8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;;
9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;;
esac
fi
# Escape application args
save () {
for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done
echo " "
}
APP_ARGS=`save "$@"`
# Collect all arguments for the java command, following the shell quoting and substitution rules
eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS"
exec "$JAVACMD" "$@"

89
gradlew.bat vendored Normal file
View File

@ -0,0 +1,89 @@
@rem
@rem Copyright 2015 the original author or authors.
@rem
@rem Licensed under the Apache License, Version 2.0 (the "License");
@rem you may not use this file except in compliance with the License.
@rem You may obtain a copy of the License at
@rem
@rem https://www.apache.org/licenses/LICENSE-2.0
@rem
@rem Unless required by applicable law or agreed to in writing, software
@rem distributed under the License is distributed on an "AS IS" BASIS,
@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@rem See the License for the specific language governing permissions and
@rem limitations under the License.
@rem
@if "%DEBUG%" == "" @echo off
@rem ##########################################################################
@rem
@rem Gradle startup script for Windows
@rem
@rem ##########################################################################
@rem Set local scope for the variables with windows NT shell
if "%OS%"=="Windows_NT" setlocal
set DIRNAME=%~dp0
if "%DIRNAME%" == "" set DIRNAME=.
set APP_BASE_NAME=%~n0
set APP_HOME=%DIRNAME%
@rem Resolve any "." and ".." in APP_HOME to make it shorter.
for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi
@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m"
@rem Find java.exe
if defined JAVA_HOME goto findJavaFromJavaHome
set JAVA_EXE=java.exe
%JAVA_EXE% -version >NUL 2>&1
if "%ERRORLEVEL%" == "0" goto execute
echo.
echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
echo.
echo Please set the JAVA_HOME variable in your environment to match the
echo location of your Java installation.
goto fail
:findJavaFromJavaHome
set JAVA_HOME=%JAVA_HOME:"=%
set JAVA_EXE=%JAVA_HOME%/bin/java.exe
if exist "%JAVA_EXE%" goto execute
echo.
echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
echo.
echo Please set the JAVA_HOME variable in your environment to match the
echo location of your Java installation.
goto fail
:execute
@rem Setup the command line
set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
@rem Execute Gradle
"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %*
:end
@rem End local scope for the variables with windows NT shell
if "%ERRORLEVEL%"=="0" goto mainEnd
:fail
rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
rem the _cmd.exe /c_ return code!
if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
exit /b 1
:mainEnd
if "%OS%"=="Windows_NT" endlocal
:omega

14
settings.gradle Normal file
View File

@ -0,0 +1,14 @@
rootProject.name = 'PpomppuNotifier'
//include 'reader'
//project(':reader').projectDir = "$rootDir/reader" as File
//include 'processor'
//project(':processor').projectDir = "$rootDir/processor" as File
//include 'writer'
//project(':writer').projectDir = "$rootDir/writer" as File
include 'shopping-crawler'
project(':shopping-crawler').projectDir = "$rootDir/shopping-crawler" as File
include 'support'
project(':support').projectDir = "$rootDir/support" as File

Binary file not shown.

View File

@ -0,0 +1,41 @@
dependencies {
developmentOnly 'org.springframework.boot:spring-boot-devtools'
runtimeOnly 'com.h2database:h2'
runtimeOnly 'com.mysql:mysql-connector-j:8.4.0'
compileOnly 'org.projectlombok:lombok'
implementation project(':support')
// https://projectreactor.io/docs/core/release/reference/#debug-activate
implementation("org.springframework.boot:spring-boot-starter-web") {
exclude group: "org.springframework.boot", module: "spring-boot-starter-tomcat"
}
implementation("org.springframework.boot:spring-boot-starter-undertow") {
exclude group: "io.undertow", module: "undertow-websockets-jsr"
}
implementation 'org.springframework.boot:spring-boot-starter-data-jpa'
implementation 'org.springframework.boot:spring-boot-configuration-processor'
implementation 'org.springframework.cloud:spring-cloud-starter-config'
implementation 'org.springframework.boot:spring-boot-starter-actuator'
implementation 'org.jsoup:jsoup:1.17.2'
implementation 'com.h2database:h2:2.2.224'
implementation "org.springframework.cloud:spring-cloud-starter-openfeign"
implementation "io.github.openfeign:feign-hc5"
implementation 'org.ahocorasick:ahocorasick:0.6.3'
annotationProcessor 'org.springframework.boot:spring-boot-configuration-processor'
annotationProcessor 'org.projectlombok:lombok'
testImplementation 'org.springframework.boot:spring-boot-starter-test'
testImplementation('org.assertj:assertj-core:3.25.3')
testImplementation("org.jeasy:easy-random-core:5.0.0")
testCompileOnly 'org.projectlombok:lombok'
testAnnotationProcessor('org.projectlombok:lombok')
}
test {
useJUnitPlatform()
testLogging {
events "passed", "skipped", "failed"
}
}

View File

@ -0,0 +1,19 @@
package com.myoa.engineering.crawl.shopping;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.cloud.openfeign.EnableFeignClients;
import org.springframework.scheduling.annotation.EnableScheduling;
@EnableScheduling
@EnableFeignClients
@EnableConfigurationProperties
@SpringBootApplication
public class ShoppingCrawlerApplication {
public static void main(String[] args) {
SpringApplication.run(ShoppingCrawlerApplication.class, args);
}
}

View File

@ -0,0 +1,30 @@
package com.myoa.engineering.crawl.shopping.configuration;
import feign.Logger;
import feign.RequestInterceptor;
import feign.codec.ErrorDecoder;
import lombok.extern.slf4j.Slf4j;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.http.HttpHeaders;
import org.springframework.http.MediaType;
@Slf4j
@Configuration
public class FeignDefaultConfig {
@Bean
Logger.Level feignLoggerLevel() {
return Logger.Level.FULL;
}
public static final String MIME_TYPE =
MediaType.APPLICATION_JSON_VALUE + ";charset=utf-8";
@Bean
public RequestInterceptor requestInterceptor() {
return requestTemplate -> requestTemplate.header(HttpHeaders.CONTENT_TYPE, MIME_TYPE);
}
}

View File

@ -0,0 +1,36 @@
package com.myoa.engineering.crawl.shopping.configuration.datasource;
import lombok.extern.slf4j.Slf4j;
import org.h2.tools.Server;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Profile;
import org.springframework.context.event.ContextClosedEvent;
import org.springframework.context.event.ContextRefreshedEvent;
import org.springframework.context.event.EventListener;
import java.sql.SQLException;
@Slf4j
@Profile({"datasource-local", "datasource-development"})
@Configuration
public class H2ConsoleConfiguration {
private Server webServer;
@Value("${spring.h2.console.port}")
private String port;
@EventListener(ContextRefreshedEvent.class)
public void start() throws SQLException {
log.info("starting h2 console");
this.webServer = Server.createWebServer("-webPort", port, "-tcpAllowOthers").start();
}
@EventListener(ContextClosedEvent.class)
public void stop() {
log.info("stopping h2 console");
this.webServer.stop(); ;
}
}

View File

@ -0,0 +1,126 @@
package com.myoa.engineering.crawl.shopping.configuration.datasource;
import com.myoa.engineering.crawl.shopping.configuration.datasource.properties.DatasourceProperties;
import com.myoa.engineering.crawl.shopping.configuration.datasource.properties.HibernateProperties;
import com.myoa.engineering.crawl.shopping.configuration.datasource.properties.HikariProperties;
import com.myoa.engineering.crawl.shopping.domain.entity.BaseScanDomain;
import com.myoa.engineering.crawl.shopping.infra.repository.BaseScanRepository;
import com.zaxxer.hikari.HikariConfig;
import com.zaxxer.hikari.HikariDataSource;
import lombok.NonNull;
import org.hibernate.cfg.AvailableSettings;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.boot.orm.jpa.EntityManagerFactoryBuilder;
import org.springframework.boot.orm.jpa.hibernate.SpringImplicitNamingStrategy;
import org.springframework.boot.orm.jpa.hibernate.SpringPhysicalNamingStrategy;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.data.jpa.repository.config.EnableJpaAuditing;
import org.springframework.data.jpa.repository.config.EnableJpaRepositories;
import org.springframework.orm.jpa.JpaTransactionManager;
import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean;
import org.springframework.transaction.PlatformTransactionManager;
import javax.persistence.EntityManagerFactory;
import javax.sql.DataSource;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
@Configuration
@EnableJpaAuditing
@EnableJpaRepositories(basePackageClasses = BaseScanRepository.class,
entityManagerFactoryRef = "shoppingCrawlerEntityManagerFactory",
transactionManagerRef = "shoppingCrawlerTransactionManager"
)
public class ShoppingCrawlerDatasourceConfiguration {
private static final String DATA_SOURCE_UNIT_NAME = "crawler-shopping";
private final DatasourceProperties dataSourceProeprties;
private final HikariProperties hikariProperties;
private final HibernateProperties hibernateProperties;
public ShoppingCrawlerDatasourceConfiguration(DatasourceProperties dataSourceProeprties,
HikariProperties hikariProperties,
HibernateProperties hibernateProperties) {
this.dataSourceProeprties = dataSourceProeprties;
this.hikariProperties = hikariProperties;
this.hibernateProperties = hibernateProperties;
}
@Bean(name = "shoppingCrawlerDataSource")
public DataSource dataSource() {
DatasourceProperties.DataSourcePropertiesUnit dataSourcePropertiesUnit = dataSourceProeprties.find(DATA_SOURCE_UNIT_NAME);
final HikariConfig hikariConfig = new HikariConfig();
hikariConfig.setJdbcUrl(dataSourcePropertiesUnit.toCompletedJdbcUrl());
hikariConfig.setDriverClassName(dataSourcePropertiesUnit.getDriverClassName());
hikariConfig.setUsername(dataSourcePropertiesUnit.getUsername());
hikariConfig.setPassword(dataSourcePropertiesUnit.getPassword());
hikariConfig.setAutoCommit(hikariProperties.getAutoCommit());
hikariConfig.setMaximumPoolSize(hikariProperties.getMaximumPoolSize());
hikariConfig.setMinimumIdle(hikariProperties.getMinimumIdle());
if (hikariProperties.getMaximumPoolSize() > hikariProperties.getMinimumIdle()) {
hikariConfig.setIdleTimeout(hikariProperties.getIdleTimeout());
}
hikariConfig.setValidationTimeout(hikariProperties.getValidationTimeout());
hikariConfig.setConnectionTimeout(hikariProperties.getConnectionTimeout());
hikariConfig.setMaxLifetime(hikariProperties.getMaxLifetime());
final DataSource dataSource = new HikariDataSource(hikariConfig);
return dataSource;
}
@Bean("shoppingCrawlerEntityManagerFactory")
public LocalContainerEntityManagerFactoryBean entityManagerFactory(
EntityManagerFactoryBuilder builder,
@Qualifier("shoppingCrawlerDataSource") DataSource dataSource) {
return builder.dataSource(dataSource)
.packages(BaseScanDomain.class)
.properties(getPropsMap(DATA_SOURCE_UNIT_NAME))
.build();
}
@Bean("shoppingCrawlerTransactionManager")
public PlatformTransactionManager transactionManager(
@Qualifier("shoppingCrawlerEntityManagerFactory") EntityManagerFactory entityManagerFactory) {
return new JpaTransactionManager(entityManagerFactory);
}
public static Properties getProps(@NonNull HibernateProperties.HibernatePropertiesUnit hibernateProperties) {
Properties properties = new Properties();
properties.put(AvailableSettings.DIALECT, hibernateProperties.getDialect());
properties.put(AvailableSettings.FORMAT_SQL, hibernateProperties.getFormatSql());
properties.put(AvailableSettings.SHOW_SQL, hibernateProperties.getShowSql());
properties.put(AvailableSettings.HBM2DDL_AUTO, hibernateProperties.getHbm2ddlAuto());
properties.put(AvailableSettings.CONNECTION_PROVIDER_DISABLES_AUTOCOMMIT, hibernateProperties.getDisableAutoCommit());
properties.put(AvailableSettings.IMPLICIT_NAMING_STRATEGY, SpringImplicitNamingStrategy.class.getName());
properties.put(AvailableSettings.PHYSICAL_NAMING_STRATEGY, SpringPhysicalNamingStrategy.class.getName());
properties.put(AvailableSettings.GENERATE_STATISTICS, "false");
// properties.put(AvailableSettings.GLOBALLY_QUOTED_IDENTIFIERS, "true");
// properties.put(AvailableSettings.GLOBALLY_QUOTED_IDENTIFIERS_SKIP_COLUMN_DEFINITIONS, "true");
properties.put(AvailableSettings.STATEMENT_BATCH_SIZE, "20");
properties.put(AvailableSettings.ORDER_INSERTS, "true");
properties.put(AvailableSettings.ORDER_UPDATES, "true");
properties.put(AvailableSettings.BATCH_VERSIONED_DATA, "true");
// properties.put(AvailableSettings.JPA_ID_GENERATOR_GLOBAL_SCOPE_COMPLIANCE, "false");
return properties;
}
public Map<String, String> getPropsMap(@NonNull String unitName) {
return convertPropertiestoMaps(getProps(hibernateProperties.find(unitName)));
}
public Map<String, String> convertPropertiestoMaps(Properties properties) {
Map<String, String> propertiesMap = new HashMap<>();
for (Enumeration<?> e = properties.propertyNames(); e.hasMoreElements(); ) {
String key = (String) e.nextElement();
propertiesMap.put(key, properties.getProperty(key));
}
return propertiesMap;
}
}

View File

@ -0,0 +1,45 @@
package com.myoa.engineering.crawl.shopping.configuration.datasource.properties;
import com.myoa.engineering.crawl.shopping.support.util.ObjectUtil;
import lombok.Data;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;
import java.util.List;
@Component
@Data
@ConfigurationProperties(prefix = "datasource")
public class DatasourceProperties {
private List<DataSourcePropertiesUnit> units;
@Data
public static class DataSourcePropertiesUnit {
private String unitName;
private String schemaName;
private String connectionParameters;
private String dbConnectionUrl;
private Boolean isSimpleConnectionUrl;
private String username;
private String password;
private String driverClassName;
public String toCompletedJdbcUrl() {
if (ObjectUtil.isEmpty(isSimpleConnectionUrl) || isSimpleConnectionUrl == false) {
return String.format("%s/%s?%s", dbConnectionUrl, schemaName, connectionParameters);
}
return dbConnectionUrl;
}
}
public DataSourcePropertiesUnit find(String unitName) {
return units.stream()
.filter(e -> e.getUnitName().equals(unitName))
.findFirst()
.orElseThrow(
() -> new IllegalArgumentException(this.getClass().getName() + ": unitName Not found. " + unitName));
}
}

View File

@ -0,0 +1,39 @@
package com.myoa.engineering.crawl.shopping.configuration.datasource.properties;
import lombok.Getter;
import lombok.Setter;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;
import java.util.List;
@Component
@Setter
@Getter
@ConfigurationProperties(prefix = "hibernate")
public class HibernateProperties {
private List<HibernatePropertiesUnit> units;
@Getter
@Setter
public static class HibernatePropertiesUnit {
private String unitName;
private String dialect;
private String formatSql;
private String showSql;
private String hbm2ddlAuto;
private String disableAutoCommit;
}
public HibernatePropertiesUnit find(String unitName) {
return units.stream()
.filter(x -> x.getUnitName().equals(unitName))
.findFirst()
.orElseThrow(
() -> new IllegalArgumentException(this.getClass().getName() + ": unitName Not found. " + unitName));
}
}

View File

@ -0,0 +1,22 @@
package com.myoa.engineering.crawl.shopping.configuration.datasource.properties;
import lombok.Getter;
import lombok.Setter;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;
@Component
@Setter
@Getter
@ConfigurationProperties(prefix = "spring.datasource.hikari")
public class HikariProperties {
private Integer minimumIdle;
private Integer maximumPoolSize;
private Integer idleTimeout;
private Integer validationTimeout;
private Integer connectionTimeout;
private Integer maxLifetime;
private Boolean autoCommit;
}

View File

@ -0,0 +1,32 @@
package com.myoa.engineering.crawl.shopping.configuration.slack.properties;
import java.util.List;
import lombok.Data;
import lombok.Getter;
import lombok.Setter;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;
@Data
@Component
@ConfigurationProperties(prefix = "slack.bot")
public class SlackSecretProperties {
private List<SlackSecretPropertiesUnit> units;
@Data
public static class SlackSecretPropertiesUnit {
private String botUnitName;
private String username;
private String iconEmoji;
private String channel;
private String token;
}
public SlackSecretPropertiesUnit find(String botUnitName) {
return units.stream()
.filter(e -> e.getBotUnitName().equals(botUnitName))
.findFirst()
.orElseThrow(() -> new IllegalArgumentException("not found bot unit name : " + botUnitName));
}
}

View File

@ -0,0 +1,22 @@
package com.myoa.engineering.crawl.shopping.controller;
import com.myoa.engineering.crawl.shopping.crawlhandler.PpomppuCrawlDomesticHandler;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
@RestController
@RequestMapping("/api/v1/exploit")
public class TestAPIController {
private final PpomppuCrawlDomesticHandler ppomppuCrawlDomesticHandler;
public TestAPIController(PpomppuCrawlDomesticHandler ppomppuCrawlDomesticHandler) {
this.ppomppuCrawlDomesticHandler = ppomppuCrawlDomesticHandler;
}
@GetMapping("/triggers")
public void triggerExploit() {
ppomppuCrawlDomesticHandler.handle();
}
}

View File

@ -0,0 +1,10 @@
package com.myoa.engineering.crawl.shopping.crawlhandler;
import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget;
public interface CrawlHandler {
CrawlTarget getCrawlTarget();
void handle();
}

View File

@ -0,0 +1,18 @@
package com.myoa.engineering.crawl.shopping.crawlhandler;
import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
@Slf4j
@Component
public class FmkoreaCrawlHandler implements CrawlHandler {
@Override
public CrawlTarget getCrawlTarget() {
return CrawlTarget.FMKOREA;
}
@Override
public void handle() {
}
}

View File

@ -0,0 +1,58 @@
package com.myoa.engineering.crawl.shopping.crawlhandler;
import com.myoa.engineering.crawl.shopping.crawlhandler.parser.PpomppuArticleParserV2;
import com.myoa.engineering.crawl.shopping.domain.entity.v2.Article;
import com.myoa.engineering.crawl.shopping.infra.client.ppomppu.PpomppuBoardClientV2;
import com.myoa.engineering.crawl.shopping.service.ArticleCommandService;
import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget;
import org.springframework.stereotype.Component;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Stream;
@Component
public class PpomppuCrawlDomesticHandler implements CrawlHandler {
private final PpomppuBoardClientV2 ppomppuBoardClient;
private final PpomppuArticleParserV2 ppomppuArticleParserV2;
private final ArticleCommandService articleCommandService;
public PpomppuCrawlDomesticHandler(PpomppuBoardClientV2 ppomppuBoardClient,
PpomppuArticleParserV2 ppomppuArticleParserV2,
ArticleCommandService articleCommandService) {
this.ppomppuBoardClient = ppomppuBoardClient;
this.ppomppuArticleParserV2 = ppomppuArticleParserV2;
this.articleCommandService = articleCommandService;
}
@Override
public CrawlTarget getCrawlTarget() {
return CrawlTarget.PPOMPPU_DOMESTIC;
}
@Override
public void handle() {
String boardHtmlPage1 = ppomppuBoardClient.getBoardHtml("/zboard/zboard.php", generateRequestParams(1));
List<Article> parsedPage1 = ppomppuArticleParserV2.parse(boardHtmlPage1);
String boardHtmlPage2 = ppomppuBoardClient.getBoardHtml("/zboard/zboard.php", generateRequestParams(2));
List<Article> parsedPage2 = ppomppuArticleParserV2.parse(boardHtmlPage2);
List<Article> merged = Stream.of(parsedPage1, parsedPage2)
.flatMap(List::stream)
.map(e -> e.updateCrawlTarget(getCrawlTarget()))
.toList();
articleCommandService.upsert(merged);
}
private Map<String, String> generateRequestParams(int pageId) {
Map<String, String> params = new HashMap<>();
params.put("id", "ppomppu");
params.put("page", String.valueOf(pageId));
return params;
}
}

View File

@ -0,0 +1,115 @@
package com.myoa.engineering.crawl.shopping.crawlhandler.parser;
import com.myoa.engineering.crawl.shopping.domain.entity.v2.Article;
import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName;
import com.myoa.engineering.crawl.shopping.util.DateTimeUtils;
import com.myoa.engineering.crawl.shopping.util.NumberUtils;
import io.micrometer.core.instrument.util.StringUtils;
import lombok.extern.slf4j.Slf4j;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.stereotype.Component;
import java.time.ZoneId;
import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@Slf4j
@Component
public final class PpomppuArticleParserV2 {
private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yy.MM.dd HH:mm:ss")
.withZone(ZoneId.of("Asia/Seoul"));
private PpomppuArticleParserV2() {
}
public List<Article> parse(String html) {
Elements trElements = converHtmlToTrElements(html);
return trElements.stream()
.filter(this::isRealArticle)
.map(this::parse)
.toList();
}
private Elements converHtmlToTrElements(String data) {
Document document = Jsoup.parse(data);
Elements trList = document.getElementById("revolution_main_table").getElementsByTag("tr");
return trList;
}
private boolean isRealArticle(Element tr) {
Elements tdList = tr.getElementsByTag("td");
if (tdList.size() != 6) {
return false;
}
if (!hasOnlyNumeric(tdList.get(0))) {
return false;
}
return true;
}
private Pattern pattern_numeric = Pattern.compile("\\d+");
private boolean hasOnlyNumeric(Element td) {
return pattern_numeric.matcher(td.text()).matches();
}
private Article parse(Element tr) {
Elements tdList = tr.getElementsByTag("td");
Long articleId = Long.parseLong(tdList.get(0).text());
String title = tdList.get(2).text();
String articleUrl = parseArticleUrl(tdList.get(2).getElementsByTag("a").attr("href"));
String boardName = parseBoardName(title);
Integer recommended = parseRecommended(tdList.get(4));
Integer hit = NumberUtils.parseInt(tdList.get(5).text(), 0);
ZonedDateTime registeredAt = DateTimeUtils.parse(tdList.get(3).text());
return Article.builder()
.articleId(articleId)
.title(title)
.boardName(boardName)
.articleUrl(articleUrl)
.recommended(recommended)
.hit(hit)
.registeredAt(registeredAt)
.build();
}
public Integer parseRecommended(Element td) {
final String voteString = td.text();
if (StringUtils.isEmpty(voteString)) {
return null;
}
final int voteUp = Integer.parseInt(td.text().split(" - ")[0]);
final int voteDown = Integer.parseInt(td.text().split(" - ")[1]);
int recommended = voteUp - voteDown;
return recommended;
}
public static String parseArticleUrl(String data) {
return PpomppuBoardName.ofViewPageUrl(data);
}
Pattern patternBoardName = Pattern.compile("\\[(.+?)\\]");
public String parseBoardName(String fullTitle) {
Matcher matcher = patternBoardName.matcher(fullTitle);
String lastMatched = null;
while (matcher.find()) {
lastMatched = matcher.group(1);
}
return lastMatched;
}
}

View File

@ -0,0 +1,28 @@
package com.myoa.engineering.crawl.shopping.domain.entity;
import lombok.Getter;
import org.springframework.data.annotation.CreatedDate;
import org.springframework.data.annotation.LastModifiedDate;
import org.springframework.data.jpa.domain.support.AuditingEntityListener;
import javax.persistence.Column;
import javax.persistence.EntityListeners;
import javax.persistence.MappedSuperclass;
import java.io.Serializable;
import java.time.Instant;
@Getter
@MappedSuperclass
@EntityListeners(AuditingEntityListener.class)
public abstract class Auditable implements Serializable {
private static final long serialVersionUID = -7105030870015828551L;
@Column
@CreatedDate
private Instant createdAt;
@Column
@LastModifiedDate
private Instant modifiedAt;
}

View File

@ -0,0 +1,10 @@
package com.myoa.engineering.crawl.shopping.domain.entity;
/**
* BaseScanDomain
*
* @author Shin Woo-jin (woozu.shin@kakaoent.com)
* @since 2021-09-08
*/
public interface BaseScanDomain {
}

View File

@ -0,0 +1,66 @@
package com.myoa.engineering.crawl.shopping.domain.entity.v1;
import com.myoa.engineering.crawl.shopping.domain.entity.Auditable;
import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName;
import lombok.Builder;
import lombok.Getter;
import lombok.NoArgsConstructor;
import javax.persistence.*;
import java.time.Instant;
@Getter
@NoArgsConstructor
@Entity
@Table(name = "ppomppu_article")
public class PpomppuArticle extends Auditable {
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
private Long id;
@Column
private Long articleId;
@Column
@Enumerated(EnumType.STRING)
private PpomppuBoardName boardName;
@Column
private String articleUrl;
@Column
private String thumbnailUrl;
@Column
private String title;
@Column
private Integer hit;
@Column
private Integer recommended;
@Column
private Instant registeredAt;
@Builder
public PpomppuArticle(Long id, Long articleId, PpomppuBoardName boardName, String articleUrl,
String thumbnailUrl, String title, Integer recommended, Integer hit,
Instant registeredAt) {
this.id = id;
this.articleId = articleId;
this.boardName = boardName;
this.articleUrl = articleUrl;
this.thumbnailUrl = thumbnailUrl;
this.title = title;
this.recommended = recommended;
this.hit = hit;
this.registeredAt = registeredAt;
}
public PpomppuArticle updateBoardName(PpomppuBoardName boardName) {
this.boardName = boardName;
return this;
}
}

View File

@ -0,0 +1,52 @@
package com.myoa.engineering.crawl.shopping.domain.entity.v1;
import com.myoa.engineering.crawl.shopping.domain.entity.Auditable;
import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName;
import lombok.Builder;
import lombok.Getter;
import lombok.NoArgsConstructor;
import javax.persistence.*;
import java.time.Instant;
@Getter
@NoArgsConstructor
@Entity
@Table(name = "ppomppu_board_feed_status")
public class PpomppuBoardFeedStatus extends Auditable {
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
private Long id;
@Column
private Long latestParsedArticleId;
@Column
@Enumerated(EnumType.STRING)
private PpomppuBoardName boardName;
@Column
private Instant updatedAt;
public static PpomppuBoardFeedStatus of(PpomppuBoardName boardName, Long latestArticleId) {
return PpomppuBoardFeedStatus.builder()
.boardName(boardName)
.latestParsedArticleId(latestArticleId)
.updatedAt(Instant.now())
.build();
}
public void updateArticleId(Long latestArticleId) {
this.updatedAt = Instant.now();
this.latestParsedArticleId = latestArticleId;
}
@Builder
public PpomppuBoardFeedStatus(Long id, Long latestParsedArticleId, PpomppuBoardName boardName, Instant updatedAt) {
this.id = id;
this.latestParsedArticleId = latestParsedArticleId;
this.boardName = boardName;
this.updatedAt = updatedAt;
}
}

View File

@ -0,0 +1,29 @@
package com.myoa.engineering.crawl.shopping.domain.entity.v1;
import com.myoa.engineering.crawl.shopping.domain.entity.Auditable;
import lombok.Getter;
import lombok.NoArgsConstructor;
import javax.persistence.*;
import java.time.Instant;
@Getter
@NoArgsConstructor
@Entity
@Table(name = "published_history")
public class PublishedHistory extends Auditable {
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
private Long id;
@Column
private Long userId;
@Column
private String boardNameList;
@Column
private Instant publishedAt;
}

View File

@ -0,0 +1,27 @@
package com.myoa.engineering.crawl.shopping.domain.entity.v1;
import com.myoa.engineering.crawl.shopping.domain.entity.Auditable;
import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName;
import lombok.Getter;
import lombok.NoArgsConstructor;
import javax.persistence.*;
@Getter
@NoArgsConstructor
@Entity
@Table(name = "subscribed_board")
public class SubscribedBoard extends Auditable {
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
private Long id;
@Column
private Long userId;
@Column
@Enumerated(EnumType.STRING)
private PpomppuBoardName boardName;
}

View File

@ -0,0 +1,26 @@
package com.myoa.engineering.crawl.shopping.domain.entity.v1;
import com.myoa.engineering.crawl.shopping.domain.entity.Auditable;
import lombok.Getter;
import lombok.NoArgsConstructor;
import javax.persistence.*;
import java.time.Instant;
@Getter
@NoArgsConstructor
@Entity
@Table(name = "subscribed_user")
public class SubscribedUser extends Auditable {
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
private Long id;
@Column
private Long userId;
@Column
private Instant registeredAt;
}

View File

@ -0,0 +1,29 @@
package com.myoa.engineering.crawl.shopping.domain.entity.v2;
import com.myoa.engineering.crawl.shopping.domain.entity.Auditable;
import javax.persistence.*;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Getter;
import lombok.NoArgsConstructor;
@Getter
@Builder
@NoArgsConstructor
@AllArgsConstructor
@Entity
@Table
public class AppUser extends Auditable {
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
private Long id;
@Column
private String name;
@Column
private String slackId;
@Column
private Boolean enabled;
}

View File

@ -0,0 +1,63 @@
package com.myoa.engineering.crawl.shopping.domain.entity.v2;
import com.myoa.engineering.crawl.shopping.domain.entity.Auditable;
import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget;
import javax.persistence.*;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Getter;
import lombok.NoArgsConstructor;
import java.time.ZonedDateTime;
@Getter
@Builder
@NoArgsConstructor
@AllArgsConstructor
@Entity
@Table
public class Article extends Auditable {
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
private Long id;
@Column
private Long articleId;
@Column
@Enumerated(EnumType.STRING)
private CrawlTarget crawlTarget;
@Column
private String boardName;
@Column
private String articleUrl;
@Column
private String title;
@Column
private Integer hit;
@Column
private Integer recommended;
@Column
private ZonedDateTime registeredAt;
public Article update(Article article) {
this.boardName = article.getBoardName();
this.articleUrl = article.getArticleUrl();
this.title = article.getTitle();
this.hit = article.getHit();
this.recommended = article.getRecommended();
return this;
}
public Article updateCrawlTarget(CrawlTarget crawlTarget) {
this.crawlTarget = crawlTarget;
return this;
}
}

View File

@ -0,0 +1,33 @@
package com.myoa.engineering.crawl.shopping.domain.entity.v2;
import com.myoa.engineering.crawl.shopping.domain.entity.Auditable;
import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget;
import javax.persistence.*;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Getter;
import lombok.NoArgsConstructor;
@Getter
@Builder
@NoArgsConstructor
@AllArgsConstructor
@Entity
@Table
public class SubscribedKeyword extends Auditable {
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
private Long id;
@Column
private String keyword;
@Column
@Enumerated(EnumType.STRING)
private CrawlTarget crawlTarget;
@Column
private String userId;
}

View File

@ -0,0 +1,24 @@
package com.myoa.engineering.crawl.shopping.domain.model;
import com.myoa.engineering.crawl.shopping.domain.model.v2.ArticleModel;
import lombok.*;
import java.util.List;
@ToString
@Getter
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class UserNotifyModel {
private String userId;
private List<ArticleModel> articles;
public static UserNotifyModel of(String userId, List<ArticleModel> articles) {
return UserNotifyModel.builder()
.userId(userId)
.articles(articles)
.build();
}
}

View File

@ -0,0 +1,26 @@
package com.myoa.engineering.crawl.shopping.domain.model.v2;
import com.myoa.engineering.crawl.shopping.domain.entity.v2.AppUser;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Getter;
@Getter
@Builder
@AllArgsConstructor
public class AppUserModel {
private Long id;
private String name;
private String slackId;
private Boolean enabled;
public static AppUserModel from(AppUser entity) {
return AppUserModel.builder()
.id(entity.getId())
.name(entity.getName())
.slackId(entity.getSlackId())
.enabled(entity.getEnabled())
.build();
}
}

View File

@ -0,0 +1,25 @@
package com.myoa.engineering.crawl.shopping.domain.model.v2;
import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget;
import lombok.*;
import java.time.ZonedDateTime;
@ToString
@Getter
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class ArticleModel {
private Long id;
private Long articleId;
private CrawlTarget crawlTarget;
private String boardName;
private String articleUrl;
private String title;
private Integer hit;
private Integer recommended;
private ZonedDateTime registeredAt;
}

View File

@ -0,0 +1,27 @@
package com.myoa.engineering.crawl.shopping.domain.model.v2;
import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget;
import com.myoa.engineering.crawl.shopping.util.AhoCorasickUtils;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Getter;
import org.ahocorasick.trie.Trie;
import java.util.List;
@Getter
@Builder
@AllArgsConstructor
public class SubscribedKeywordAggregatedModel {
private final Trie ahoCorasickTrie;
private final String userId;
private final CrawlTarget crawlTarget;
public static SubscribedKeywordAggregatedModel of(String userId, CrawlTarget crawlTarget, List<String> keywords) {
return SubscribedKeywordAggregatedModel.builder()
.userId(userId)
.crawlTarget(crawlTarget)
.ahoCorasickTrie(AhoCorasickUtils.generateTrie(keywords))
.build();
}
}

View File

@ -0,0 +1,46 @@
package com.myoa.engineering.crawl.shopping.dto;
import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName;
import lombok.Builder;
import lombok.Getter;
import lombok.NoArgsConstructor;
import java.io.Serializable;
import java.time.Instant;
/**
* FeedParsedResult
*
* @author Shin Woo-jin (woozu.shin@kakaoent.com)
* @since 2021-09-08
*/
@Getter
@NoArgsConstructor
public class FeedParsedResult implements Serializable {
private static final long serialVersionUID = -3771310078623481348L;
private PpomppuBoardName boardName;
private Instant requestedAt;
private Instant processedAt;
@Builder
public FeedParsedResult(PpomppuBoardName boardName, Instant requestedAt, Instant processedAt) {
this.boardName = boardName;
this.requestedAt = requestedAt;
this.processedAt = processedAt;
}
public static FeedParsedResult of(PpomppuBoardName boardName) {
return FeedParsedResult.builder()
.boardName(boardName)
.requestedAt(Instant.now())
.build();
}
public FeedParsedResult done() {
this.processedAt = Instant.now();
return this;
}
}

View File

@ -0,0 +1,65 @@
package com.myoa.engineering.crawl.shopping.dto;
import com.myoa.engineering.crawl.shopping.domain.entity.v1.PpomppuArticle;
import com.myoa.engineering.crawl.shopping.support.util.DateUtil;
import lombok.Builder;
import lombok.Getter;
import lombok.NoArgsConstructor;
import java.time.Instant;
/**
* PpomppuArticleParseDTO
*
* @author Shin Woo-jin (woozu.shin@kakaoent.com)
* @since 2021-09-08
*/
@Getter
@NoArgsConstructor
public class PpomppuArticleParseDTO {
private String id;
private String articleId;
private String boardName;
private String articleUrl;
private String thumbnailUrl;
private String title;
private String hit;
private Integer recommended;
private String registeredAt;
@Builder
public PpomppuArticleParseDTO(String id, String articleId, String boardName, String articleUrl,
String thumbnailUrl, String title, String hit, Integer recommended,
String registeredAt) {
this.id = id;
this.articleId = articleId;
this.boardName = boardName;
this.articleUrl = articleUrl;
this.thumbnailUrl = thumbnailUrl;
this.title = title;
this.hit = hit;
this.recommended = recommended;
this.registeredAt = registeredAt;
}
public boolean isInValidated() {
return articleId == null || articleId.isEmpty()
|| hit == null || hit.isEmpty();
}
public PpomppuArticle convert() {
if (isInValidated()) {
throw new IllegalArgumentException("PpomppuArticleParseDTO was invalidated");
}
return PpomppuArticle.builder()
.articleId(Long.parseLong(articleId))
.title(title)
.articleUrl(articleUrl)
.thumbnailUrl(thumbnailUrl)
.recommended(recommended)
.hit(Integer.parseInt(hit))
.registeredAt(DateUtil.DATE_TIME_FORMATTER.parse(registeredAt, Instant::from))
.build();
}
}

View File

@ -0,0 +1,71 @@
package com.myoa.engineering.crawl.shopping.dto;
import com.myoa.engineering.crawl.shopping.domain.entity.v1.PpomppuArticle;
import com.myoa.engineering.crawl.shopping.support.dto.BlockMessageDTO;
import com.myoa.engineering.crawl.shopping.support.dto.SimpleMessageDTO;
import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName;
import com.myoa.engineering.crawl.shopping.support.util.DateUtil;
import java.time.Instant;
import java.util.List;
import java.util.function.Function;
import java.util.stream.Collectors;
/**
* PpomppuArticleTransformer
*
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-11-21
*/
public final class PpomppuArticleTransformer {
private PpomppuArticleTransformer() {
}
private static final String MESSAGE_FORMAT_V1 = "%s)) <%s|LINK> `%s` ";
private static final String MESSAGE_FORMAT_V2 = "%s *<%s|LINK>*\n%s";
private static final String TITLE_FORMAT_V1 = "_*:hearts: %s | %s*_";
public static final Function<PpomppuArticle, SimpleMessageDTO> TRANSFORM_TO_MESSAGE_DTO = article ->
SimpleMessageDTO.builder()
.requestedAt(Instant.now())
.publishedAt(article.getRegisteredAt())
.title(String.format(MESSAGE_FORMAT_V1,
article.getBoardName().getMenuName(), article.getArticleUrl(),
article.getTitle()))
.body(article.getArticleUrl())
.build();
// https://stackoverflow.com/questions/24882927/using-streams-to-convert-a-list-of-objects-into-a-string-obtained-from-the-tostr
public static SimpleMessageDTO transformToSimpleMessage(List<PpomppuArticle> articles) {
Instant requestedAt = Instant.now();
String body = articles.stream()
.map(PpomppuArticleTransformer::convertToInlineMessage)
.collect(Collectors.joining("\n\n"));
return SimpleMessageDTO.builder()
.requestedAt(requestedAt)
.title(DateUtil.DATE_TIME_FORMATTER.format(requestedAt))
.body(body)
.build();
}
public static BlockMessageDTO transformToBlockMessage(PpomppuBoardName boardName, List<PpomppuArticle> articles) {
Instant requestedAt = Instant.now();
List<BlockMessageDTO.Block> body = articles.stream()
.map(e -> BlockMessageDTO.createBlock(convertToInlineMessage(e),
e.getThumbnailUrl()))
.collect(Collectors.toList());
return BlockMessageDTO.builder()
.requestedAt(requestedAt)
.title(String.format(TITLE_FORMAT_V1,
boardName.getMenuName(),
DateUtil.DATE_TIME_FORMATTER.format(requestedAt)))
.blocks(body)
.build();
}
public static String convertToInlineMessage(PpomppuArticle article) {
return String.format(MESSAGE_FORMAT_V2,
article.getBoardName().getMenuName(), article.getArticleUrl(), article.getTitle());
}
}

View File

@ -0,0 +1,19 @@
package com.myoa.engineering.crawl.shopping.dto.constant;
import lombok.AllArgsConstructor;
import lombok.Getter;
/**
* WebClientPropertiesUnitName
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-11-18
*
*/
@Getter
@AllArgsConstructor
public enum WebClientPropertiesUnitName {
PPOMPPU_NOTIFIER_SENDER_API("ppn-sender-api"),
;
private String unitName;
}

View File

@ -0,0 +1,12 @@
package com.myoa.engineering.crawl.shopping.dto.slack;
import java.io.Serializable;
/**
* MessageDTO
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-11-14
*
*/
public interface MessageDTO extends Serializable {
}

View File

@ -0,0 +1,47 @@
package com.myoa.engineering.crawl.shopping.dto.slack;
import com.fasterxml.jackson.annotation.JsonInclude;
import lombok.Builder;
import lombok.Getter;
import lombok.NoArgsConstructor;
/**
* MessageBlock
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-11-30
*
*/
@Getter
@NoArgsConstructor
@JsonInclude(JsonInclude.Include.NON_NULL)
public class SlackBaseMessageBlock implements SlackMessageBlock {
private static final long serialVersionUID = 1597984001727808419L;
private SlackMessageBlockType type;
private String text;
@Builder
private SlackBaseMessageBlock(SlackMessageBlockType type, String text) {
this.type = type;
this.text = text;
}
public static SlackBaseMessageBlock ofMarkDown(String message) {
return SlackBaseMessageBlock.builder()
.type(SlackMessageBlockType.MARKDOWN)
.text(message)
.build();
}
public static SlackBaseMessageBlock ofDivider() {
return SlackBaseMessageBlock.builder()
.type(SlackMessageBlockType.DIVIDER)
.build();
}
@Override
public String getType() {
return type.getType();
}
}

View File

@ -0,0 +1,49 @@
package com.myoa.engineering.crawl.shopping.dto.slack;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.Builder;
import lombok.Getter;
import lombok.NoArgsConstructor;
/**
* SlackImageMessageBlock
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-11-30
*
*/
@Getter
@NoArgsConstructor
@JsonInclude(JsonInclude.Include.NON_NULL)
public class SlackImageMessageBlock implements SlackMessageBlock {
private static final long serialVersionUID = 1597984001727808419L;
private SlackMessageBlockType type;
@JsonProperty(value = "image_url", required = true)
private String imageUrl;
@JsonProperty(value = "alt_text", required = true)
private String altText;
@Builder
private SlackImageMessageBlock(SlackMessageBlockType type, String imageUrl, String altText) {
this.type = type;
this.imageUrl = imageUrl;
this.altText = altText;
}
public static SlackImageMessageBlock of(String imageUrl, String altText) {
return SlackImageMessageBlock.builder()
.type(SlackMessageBlockType.IMAGE)
.imageUrl(imageUrl)
.altText(altText)
.build();
}
@Override
public String getType() {
return type.getType();
}
}

View File

@ -0,0 +1,15 @@
package com.myoa.engineering.crawl.shopping.dto.slack;
import java.io.Serializable;
/**
* SlackMessageBlock
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-12-01
*
*/
public interface SlackMessageBlock extends Serializable {
String getType();
}

View File

@ -0,0 +1,22 @@
package com.myoa.engineering.crawl.shopping.dto.slack;
import lombok.AllArgsConstructor;
import lombok.Getter;
/**
* BlockType
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-11-30
*
*/
@Getter
@AllArgsConstructor
public enum SlackMessageBlockType {
SECTION("section"),
MARKDOWN("mrkdwn"),
DIVIDER("divider"),
IMAGE("image"),
;
private String type;
}

View File

@ -0,0 +1,61 @@
package com.myoa.engineering.crawl.shopping.dto.slack;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.myoa.engineering.crawl.shopping.support.dto.BlockMessageDTO;
import lombok.Builder;
import lombok.Getter;
import lombok.NoArgsConstructor;
import java.util.List;
/**
* SlackMessageDTO
*
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-11-14
*/
@Getter
@NoArgsConstructor
@JsonInclude(JsonInclude.Include.NON_NULL)
public class SlackMessageDTO implements MessageDTO {
private final static long serialVersionUID = 4737608709660494713L;
private String text;
private String channel;
private String username;
private List<SlackMessageBlock> blocks;
@JsonProperty("icon_emoji")
private String iconEmoji;
@Builder
public SlackMessageDTO(String text, String channel, String username,
List<SlackMessageBlock> blocks, String iconEmoji) {
this.text = text;
this.channel = channel;
this.username = username;
this.blocks = blocks;
this.iconEmoji = iconEmoji;
}
public void applyText(String text) {
this.text = text;
}
public void addSectionBlock(BlockMessageDTO.Block block) {
SlackSectionMessageBlock slackSectionMessageBlock = SlackSectionMessageBlock.ofMarkDown(block.getText());
slackSectionMessageBlock.applyImageaccessory(block.getImageUrl(), block.getAltText());
addBlock(slackSectionMessageBlock);
}
public void addSectionBlock(String rawBlockMessage) {
addBlock(SlackSectionMessageBlock.ofMarkDown(rawBlockMessage));
}
public void addBlock(SlackMessageBlock block) {
blocks.add(block);
}
}

View File

@ -0,0 +1,48 @@
package com.myoa.engineering.crawl.shopping.dto.slack;
import com.fasterxml.jackson.annotation.JsonInclude;
import lombok.Builder;
import lombok.Getter;
import lombok.NoArgsConstructor;
/**
* SectionBlock
*
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-11-30
*/
@Getter
@NoArgsConstructor
@JsonInclude(JsonInclude.Include.NON_NULL)
public class SlackSectionMessageBlock implements SlackMessageBlock {
private static final long serialVersionUID = -7600944576753160168L;
private SlackMessageBlockType type;
private SlackBaseMessageBlock text;
private SlackImageMessageBlock accessory;
@Builder
private SlackSectionMessageBlock(SlackMessageBlockType type, SlackBaseMessageBlock text,
SlackImageMessageBlock accessory) {
this.type = type;
this.text = text;
this.accessory = accessory;
}
public static SlackSectionMessageBlock ofMarkDown(String message) {
return SlackSectionMessageBlock.builder()
.type(SlackMessageBlockType.SECTION)
.text(SlackBaseMessageBlock.ofMarkDown(message))
.build();
}
public SlackSectionMessageBlock applyImageaccessory(String imageUrl, String altText) {
this.accessory = SlackImageMessageBlock.of(imageUrl, altText);
return this;
}
@Override
public String getType() {
return type.getType();
}
}

View File

@ -0,0 +1,13 @@
package com.myoa.engineering.crawl.shopping.event;
import com.myoa.engineering.crawl.shopping.domain.model.v2.ArticleModel;
import org.springframework.context.ApplicationEvent;
import java.util.List;
public class ArticleUpsertEvent extends ApplicationEvent {
public ArticleUpsertEvent(List<ArticleModel> source) {
super(source);
}
}

View File

@ -0,0 +1,15 @@
package com.myoa.engineering.crawl.shopping.event;
import com.myoa.engineering.crawl.shopping.domain.model.v2.ArticleModel;
import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget;
import lombok.AllArgsConstructor;
import lombok.Getter;
import java.util.List;
@Getter
@AllArgsConstructor
public class ArticleUpsertEventPayload {
private final List<ArticleModel> articles;
private final CrawlTarget crawlTarget;
}

View File

@ -0,0 +1,86 @@
package com.myoa.engineering.crawl.shopping.event.handler;
import com.myoa.engineering.crawl.shopping.domain.model.UserNotifyModel;
import com.myoa.engineering.crawl.shopping.domain.model.v2.AppUserModel;
import com.myoa.engineering.crawl.shopping.domain.model.v2.ArticleModel;
import com.myoa.engineering.crawl.shopping.domain.model.v2.SubscribedKeywordAggregatedModel;
import com.myoa.engineering.crawl.shopping.event.ArticleUpsertEvent;
import com.myoa.engineering.crawl.shopping.service.AppUserQueryService;
import com.myoa.engineering.crawl.shopping.service.SubscribedKeywordCacheService;
import com.myoa.engineering.crawl.shopping.service.UserNotifyService;
import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget;
import org.springframework.context.event.EventListener;
import org.springframework.stereotype.Component;
import java.util.List;
import java.util.Map;
import java.util.function.Function;
import java.util.stream.Collectors;
@Component
public class ArticleUpsertEventListener {
private final SubscribedKeywordCacheService subscribedKeywordCacheService;
private final AppUserQueryService appUserQueryService;
private final UserNotifyService userNotifyService;
public ArticleUpsertEventListener(SubscribedKeywordCacheService subscribedKeywordCacheService,
AppUserQueryService appUserQueryService, UserNotifyService userNotifyService) {
this.subscribedKeywordCacheService = subscribedKeywordCacheService;
this.appUserQueryService = appUserQueryService;
this.userNotifyService = userNotifyService;
}
@EventListener
public void handleArticleUpsertEvent(ArticleUpsertEvent event) {
Map<CrawlTarget, List<ArticleModel>> articleMap =
((List<ArticleModel>) event.getSource()).stream()
.collect(Collectors.groupingBy(ArticleModel::getCrawlTarget));
List<AppUserModel> appUsers = appUserQueryService.findAll();
appUsers.stream()
.filter(AppUserModel::getEnabled)
.map(user -> {
List<ArticleModel> filteredArticles = handleAhoCorasick(articleMap)
.apply(subscribedKeywordCacheService.getSubscribedKeywordsCached(user.getName()));
return UserNotifyModel.of(user.getName(), filteredArticles);
})
.forEach(this::notifyMessage);
}
private Function<Map<CrawlTarget, SubscribedKeywordAggregatedModel>, List<ArticleModel>> handleAhoCorasick(
Map<CrawlTarget, List<ArticleModel>> articleMap) {
return userTrieModel -> {
return userTrieModel
.entrySet()
.stream().filter(e -> articleMap.containsKey(e.getKey()))
.map((entry) -> filterAhocorasick(articleMap.get(entry.getKey()), entry.getValue()))
.flatMap(List::stream)
.toList();
// return UserNotifyModel.of(userTrieModel.values().stream().findFirst().get().getUserId(),
// filteredArticle);
};
}
private List<ArticleModel> filterAhocorasick(List<ArticleModel> articles,
SubscribedKeywordAggregatedModel trieModel) {
return articles.stream()
.filter(article -> !trieModel.getAhoCorasickTrie()
.parseText(article.getTitle())
.isEmpty())
.toList();
//ArticleUpsertEventListener::printArticle
}
private void notifyMessage(UserNotifyModel article) {
System.out.println("article = " + article);
if (article.getArticles().isEmpty()){
return;
}
userNotifyService.notify("안녕 " + article.getUserId() + "\n" + article.getArticles());
}
}

View File

@ -0,0 +1,12 @@
package com.myoa.engineering.crawl.shopping.infra.client.fmkorea;
import org.springframework.cloud.openfeign.FeignClient;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
@FeignClient(value = "fmkorea-board-client", url = "https://fmkorea.com")
public interface FmkoreaBoardClient {
@GetMapping("{boardLink}")
String getBoardHtml(@PathVariable("boardLink") String boardLink);
}

View File

@ -0,0 +1,38 @@
package com.myoa.engineering.crawl.shopping.infra.client.ppomppu;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
/**
* PpomppuBoardFeedRetriever
*
* @author Shin Woo-jin (woozu.shin@kakaoent.com)
* @since 2021-09-08
*/
@Slf4j
@Component
public class PpomppuBoardClient {
/*
private final WebClient webClient;
public PpomppuBoardClient(WebClient.Builder webClientBuilder) {
this.webClient = webClientBuilder.baseUrl(PpomppuBoardName.PPOMPPU_URL)
.exchangeStrategies(WebFluxExchangeStragiesFactory.ofTextHtml())
.filter(WebClientFilterFactory.logRequest())
.filter(WebClientFilterFactory.logResponse())
.build();
}
public Mono<String> getHtml(String uri) {
return webClient.get()
.uri(uri)
.exchangeToMono(e -> e.bodyToMono(String.class))
.publishOn(Schedulers.boundedElastic())
.onErrorResume(WebClientRequestException.class, t -> {
log.info("Exception occured, ignoring. : {}", t.getClass().getSimpleName());
return Mono.empty();
});
// .doOnNext(e -> log.info("[getHtml] {}", e));
}
*/
}

View File

@ -0,0 +1,17 @@
package com.myoa.engineering.crawl.shopping.infra.client.ppomppu;
import org.springframework.cloud.openfeign.FeignClient;
import org.springframework.cloud.openfeign.SpringQueryMap;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestParam;
import java.util.Map;
@FeignClient(value = "ppomppu-board-client", url = "https://www.ppomppu.co.kr")
public interface PpomppuBoardClientV2 {
@GetMapping("{boardLink}")
String getBoardHtml(@PathVariable("boardLink") String boardLink,
@SpringQueryMap Map<String, String> params);
}

View File

@ -0,0 +1,17 @@
package com.myoa.engineering.crawl.shopping.infra.client.slack;
import com.myoa.engineering.crawl.shopping.configuration.FeignDefaultConfig;
import com.myoa.engineering.crawl.shopping.dto.slack.SlackMessageDTO;
import org.springframework.cloud.openfeign.FeignClient;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestHeader;
@FeignClient(value = "slack-api-client", url = "https://slack.com/api",
configuration = FeignDefaultConfig.class)
public interface SlackAPIClient {
@PostMapping("/chat.postMessage")
String sendMessage(@RequestBody SlackMessageDTO message,
@RequestHeader("Authorization") String token);
}

View File

@ -0,0 +1,4 @@
package com.myoa.engineering.crawl.shopping.infra.repository;
public interface BaseScanRepository {
}

View File

@ -0,0 +1,10 @@
package com.myoa.engineering.crawl.shopping.infra.repository.v1;
import com.myoa.engineering.crawl.shopping.domain.entity.v1.PpomppuArticle;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.stereotype.Repository;
@Repository
public interface PpomppuArticleRepository extends JpaRepository<PpomppuArticle, Long> {
}

View File

@ -0,0 +1,15 @@
package com.myoa.engineering.crawl.shopping.infra.repository.v1;
import com.myoa.engineering.crawl.shopping.domain.entity.v1.PpomppuBoardFeedStatus;
import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.stereotype.Repository;
import java.util.Optional;
@Repository
public interface PpomppuBoardFeedStatusRepository extends JpaRepository<PpomppuBoardFeedStatus, Long> {
Optional<PpomppuBoardFeedStatus> findByBoardName(PpomppuBoardName boardName);
}

View File

@ -0,0 +1,9 @@
package com.myoa.engineering.crawl.shopping.infra.repository.v2;
import com.myoa.engineering.crawl.shopping.domain.entity.v2.AppUser;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.stereotype.Repository;
@Repository
public interface AppUserRepository extends JpaRepository<AppUser, Long> {
}

View File

@ -0,0 +1,16 @@
package com.myoa.engineering.crawl.shopping.infra.repository.v2;
import com.myoa.engineering.crawl.shopping.domain.entity.v2.Article;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.stereotype.Repository;
import java.util.Collection;
import java.util.List;
import java.util.Optional;
@Repository
public interface ArticleRepository extends JpaRepository<Article, Long> {
Optional<Article> findByArticleId(Long articleId);
List<Article> findByArticleIdIn(Collection<Long> articleId);
}

View File

@ -0,0 +1,23 @@
package com.myoa.engineering.crawl.shopping.infra.repository.v2;
import com.myoa.engineering.crawl.shopping.domain.entity.v2.SubscribedKeyword;
import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.stereotype.Repository;
import java.util.List;
@Repository
public interface SubscribedKeywordRepository extends JpaRepository<SubscribedKeyword, Long> {
List<SubscribedKeyword> findByCrawlTarget(CrawlTarget crawlTarget);
/* @Query("SELECT new com.myoa.engineering.crawl.shopping.domain.model.v2.SubscribedKeywordUserAggregatedModel(" +
" s.userId, s.keyword, s.crawlTarget) " +
" FROM SubscribedKeyword s GROUP BY s.userId ")
List<SubscribedKeywordUserAggregatedModel> findGroupByUserId(String userId);*/
List<SubscribedKeyword> findByUserIdAndCrawlTarget(String userId, CrawlTarget crawlTarget);
List<SubscribedKeyword> findByUserId(String userId);
}

View File

@ -0,0 +1,27 @@
package com.myoa.engineering.crawl.shopping.scheduler;
import com.myoa.engineering.crawl.shopping.crawlhandler.CrawlHandler;
import lombok.extern.slf4j.Slf4j;
import org.springframework.scheduling.annotation.EnableScheduling;
import org.springframework.stereotype.Component;
import java.util.List;
@Slf4j
@Component
@EnableScheduling
public class ParseEventEmitter {
private final List<CrawlHandler> crawlHandlers;
public ParseEventEmitter(List<CrawlHandler> crawlHandlers) {
this.crawlHandlers = crawlHandlers;
}
// @Scheduled(cron = "0 0/5 * * * ?")
public void emit() {
log.info("[emitDomesticBoard] trigger fired!");
crawlHandlers.forEach(CrawlHandler::handle);
}
}

View File

@ -0,0 +1,25 @@
package com.myoa.engineering.crawl.shopping.service;
import com.myoa.engineering.crawl.shopping.domain.model.v2.AppUserModel;
import com.myoa.engineering.crawl.shopping.infra.repository.v2.AppUserRepository;
import org.springframework.stereotype.Service;
import java.util.List;
@Service
public class AppUserQueryService {
private final AppUserRepository appUserRepository;
public AppUserQueryService(AppUserRepository appUserRepository) {
this.appUserRepository = appUserRepository;
}
public List<AppUserModel> findAll() {
return appUserRepository.findAll()
.stream()
.map(AppUserModel::from)
.toList();
}
}

View File

@ -0,0 +1,81 @@
package com.myoa.engineering.crawl.shopping.service;
import com.myoa.engineering.crawl.shopping.domain.entity.v2.Article;
import com.myoa.engineering.crawl.shopping.domain.model.v2.ArticleModel;
import com.myoa.engineering.crawl.shopping.event.ArticleUpsertEvent;
import com.myoa.engineering.crawl.shopping.infra.repository.v2.ArticleRepository;
import javax.transaction.Transactional;
import org.springframework.context.ApplicationEventPublisher;
import org.springframework.stereotype.Service;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.function.Function;
import java.util.stream.Collectors;
@Service
public class ArticleCommandService {
private final ArticleRepository articleRepository;
private final ApplicationEventPublisher applicationEventPublisher;
public ArticleCommandService(ArticleRepository articleRepository,
ApplicationEventPublisher applicationEventPublisher) {
this.articleRepository = articleRepository;
this.applicationEventPublisher = applicationEventPublisher;
}
@Transactional
public void upsert(List<Article> articles) {
Map<Long, Article> articleMap = articles.stream()
.collect(Collectors.toMap(Article::getArticleId, e -> e));
List<Article> saved = articleRepository.findByArticleIdIn(articleMap.keySet());
List<Article> updated = saved.stream()
.filter(e -> articleMap.containsKey(e.getArticleId()))
.map(e -> e.update(articleMap.get(e.getArticleId())))
.peek(e -> articleMap.remove(e.getArticleId()))
.sorted(Comparator.comparing(Article::getArticleId))
.toList();
List<Article> newArticles = articleMap.values()
.stream()
.sorted(Comparator.comparing(Article::getArticleId))
.toList();
articleRepository.saveAll(updated);
articleRepository.saveAll(newArticles);
publishEvent(newArticles);
}
private void publishEvent(List<Article> articles) {
List<ArticleModel> articleModels =
articles.stream()
.map(transformer)
.toList();
applicationEventPublisher.publishEvent(new ArticleUpsertEvent(articleModels));
}
@Transactional
public void upsert(Article article) {
Article saved = articleRepository.findByArticleId(article.getArticleId())
.orElse(article);
articleRepository.save(saved);
}
public static Function<Article, ArticleModel> transformer =
article -> ArticleModel.builder()
.id(article.getId())
.articleId(article.getArticleId())
.crawlTarget(article.getCrawlTarget())
.boardName(article.getBoardName())
.articleUrl(article.getArticleUrl())
.title(article.getTitle())
.hit(article.getHit())
.recommended(article.getRecommended())
.registeredAt(article.getRegisteredAt())
.build();
}

View File

@ -0,0 +1,67 @@
package com.myoa.engineering.crawl.shopping.service;
import com.myoa.engineering.crawl.shopping.domain.entity.v1.PpomppuArticle;
import com.myoa.engineering.crawl.shopping.domain.entity.v1.PpomppuBoardFeedStatus;
import com.myoa.engineering.crawl.shopping.infra.repository.v1.PpomppuArticleRepository;
import com.myoa.engineering.crawl.shopping.infra.repository.v1.PpomppuBoardFeedStatusRepository;
import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import java.util.List;
import java.util.Optional;
import java.util.stream.Collectors;
@Slf4j
@Service
public class PpomppuArticleService {
private final PpomppuArticleRepository ppomppuArticleRepository;
private final PpomppuBoardFeedStatusRepository ppomppuBoardFeedStatusRepository;
public PpomppuArticleService(PpomppuArticleRepository ppomppuArticleRepository,
PpomppuBoardFeedStatusRepository ppomppuBoardFeedStatusRepository) {
this.ppomppuArticleRepository = ppomppuArticleRepository;
this.ppomppuBoardFeedStatusRepository = ppomppuBoardFeedStatusRepository;
}
@Transactional(readOnly = true)
public List<PpomppuArticle> filterOnlyNewArticles(PpomppuBoardName boardName, List<PpomppuArticle> articles) {
Optional<PpomppuBoardFeedStatus> boardFeedStatus = ppomppuBoardFeedStatusRepository.findByBoardName(boardName);
Long latestArticleId = boardFeedStatus.map(PpomppuBoardFeedStatus::getLatestParsedArticleId)
.orElse(0L);
log.info("latestArticleId : {}", latestArticleId);
return articles.stream()
.filter(e -> e.getArticleId().compareTo(latestArticleId) > 0)
.collect(Collectors.toList());
}
@Transactional
public List<PpomppuArticle> save(PpomppuBoardName boardName, List<PpomppuArticle> articles) {
Long latestArticleId = articles.stream()
.map(PpomppuArticle::getArticleId)
.max(Long::compareTo)
.orElse(0L);
// save PpomppuBoardFeedStatus
Optional<PpomppuBoardFeedStatus> boardFeedStatus = ppomppuBoardFeedStatusRepository.findByBoardName(boardName);
log.info("[save] boardName: {}, isPresent?: {}, latestArticleId: {}",
boardName, boardFeedStatus.isPresent(), latestArticleId);
log.info("[save] articles count: {}, article ids: {}",
articles.size(), articles.stream().map(PpomppuArticle::getArticleId).toArray());
boardFeedStatus.ifPresentOrElse(e -> {
if (latestArticleId.longValue() > 0L) {
e.updateArticleId(latestArticleId);
ppomppuBoardFeedStatusRepository.save(e);
}
},
() -> ppomppuBoardFeedStatusRepository.save(PpomppuBoardFeedStatus.of(boardName,
latestArticleId)));
// save real articles.
return ppomppuArticleRepository.saveAll(articles);
}
}

View File

@ -0,0 +1,41 @@
package com.myoa.engineering.crawl.shopping.service;
import com.myoa.engineering.crawl.shopping.domain.entity.v2.SubscribedKeyword;
import com.myoa.engineering.crawl.shopping.domain.model.v2.SubscribedKeywordAggregatedModel;
import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget;
import org.springframework.cache.annotation.Cacheable;
import org.springframework.stereotype.Component;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
@Component
public class SubscribedKeywordCacheService {
private final SubscribedKeywordQueryService subscribedKeywordQueryService;
public SubscribedKeywordCacheService(SubscribedKeywordQueryService subscribedKeywordQueryService) {
this.subscribedKeywordQueryService = subscribedKeywordQueryService;
}
@Cacheable(cacheNames = "subscribe.keywords", key = "#userId + '_' + #crawlTarget.name()")
public SubscribedKeywordAggregatedModel getSubscribedKeywordsCached(String userId, CrawlTarget crawlTarget) {
System.out.println("getSubscribedKeywordsCached");
List<String> keywords = subscribedKeywordQueryService.findByUserWithTarget(userId, crawlTarget)
.stream().map(SubscribedKeyword::getKeyword).toList();
return SubscribedKeywordAggregatedModel.of(userId, crawlTarget, keywords);
}
@Cacheable(cacheNames = "subscribe.keywords", key = "#userId")
public Map<CrawlTarget, SubscribedKeywordAggregatedModel> getSubscribedKeywordsCached(String userId) {
System.out.println("getSubscribedKeywordsCached");
return subscribedKeywordQueryService.findByUser(userId)
.stream()
.collect(Collectors.groupingBy(SubscribedKeyword::getCrawlTarget,
Collectors.mapping(SubscribedKeyword::getKeyword, Collectors.toList())))
.entrySet().stream()
.collect(Collectors.toMap(Map.Entry::getKey, e -> SubscribedKeywordAggregatedModel.of(userId, e.getKey(), e.getValue())));
}
}

View File

@ -0,0 +1,30 @@
package com.myoa.engineering.crawl.shopping.service;
import com.myoa.engineering.crawl.shopping.domain.entity.v2.SubscribedKeyword;
import com.myoa.engineering.crawl.shopping.infra.repository.v2.SubscribedKeywordRepository;
import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget;
import org.springframework.stereotype.Service;
import java.util.List;
@Service
public class SubscribedKeywordQueryService {
private final SubscribedKeywordRepository subscribedKeywordRepository;
public SubscribedKeywordQueryService(SubscribedKeywordRepository subscribedKeywordRepository) {
this.subscribedKeywordRepository = subscribedKeywordRepository;
}
public List<SubscribedKeyword> findAll() {
return subscribedKeywordRepository.findAll();
}
public List<SubscribedKeyword> findByUserWithTarget(String userId, CrawlTarget crawlTarget) {
return subscribedKeywordRepository.findByUserIdAndCrawlTarget(userId, crawlTarget);
}
public List<SubscribedKeyword> findByUser(String userId) {
return subscribedKeywordRepository.findByUserId(userId);
}
}

View File

@ -0,0 +1,34 @@
package com.myoa.engineering.crawl.shopping.service;
import com.myoa.engineering.crawl.shopping.configuration.slack.properties.SlackSecretProperties;
import com.myoa.engineering.crawl.shopping.dto.slack.SlackMessageDTO;
import com.myoa.engineering.crawl.shopping.infra.client.slack.SlackAPIClient;
import org.springframework.stereotype.Service;
@Service
public class UserNotifyService {
private static final String SLACK_PROPERTIES_UNIT_NAME = "shopping-crawler";
// private static final String NOTIFY_CHANNEL_ID = "notify_shopping";
// private static final String NOTIFY_ICON_EMOJI = ":monge_big:";
// private static final String NOTIFY_BOT_NAME = "몽이 탈호구봇";
private final SlackAPIClient slackAPIClient;
private final SlackSecretProperties.SlackSecretPropertiesUnit slackSecretProperties;
public UserNotifyService(SlackAPIClient slackAPIClient,
SlackSecretProperties slackSecretProperties) {
this.slackAPIClient = slackAPIClient;
this.slackSecretProperties = slackSecretProperties.find(SLACK_PROPERTIES_UNIT_NAME);
}
public void notify(String message) {
SlackMessageDTO slackMessageDTO = SlackMessageDTO.builder()
.channel(slackSecretProperties.getChannel())
.text(message)
.iconEmoji(slackSecretProperties.getIconEmoji())
.username(slackSecretProperties.getUsername())
.build();
slackAPIClient.sendMessage(slackMessageDTO, slackSecretProperties.getToken());
}
}

View File

@ -0,0 +1,18 @@
package com.myoa.engineering.crawl.shopping.util;
import org.ahocorasick.trie.Trie;
import java.util.List;
public final class AhoCorasickUtils {
private AhoCorasickUtils() {
}
public static Trie generateTrie(List<String> keywords) {
return Trie.builder()
.addKeywords(keywords)
.ignoreCase()
.build();
}
}

View File

@ -0,0 +1,27 @@
package com.myoa.engineering.crawl.shopping.util;
import java.time.*;
import java.time.format.DateTimeFormatter;
public final class DateTimeUtils {
private static final DateTimeFormatter FORMATTER_HHMMss = DateTimeFormatter.ofPattern("HH:mm:ss");
private static final ZoneId ZONE_ASIA_SEOUL = ZoneId.of("Asia/Seoul");
private DateTimeUtils() {
}
public static ZonedDateTime parse(String HHMMss) {
try {
LocalTime time = LocalTime.parse(HHMMss, FORMATTER_HHMMss);
LocalDateTime dateTime = LocalDateTime.of(LocalDate.now(), time);
if (dateTime.isAfter(LocalDateTime.now())) {
dateTime = dateTime.minusDays(1);
}
return dateTime.atZone(ZONE_ASIA_SEOUL);
} catch (Exception e) {
return null;
}
}
}

View File

@ -0,0 +1,32 @@
package com.myoa.engineering.crawl.shopping.util;
public final class NumberUtils {
private NumberUtils() {
}
public static Integer parseInt(String value) {
return parseInt(value, null);
}
public static Integer parseInt(String value, Integer defaultValue) {
try {
return Integer.parseInt(value);
} catch (NumberFormatException e) {
return defaultValue;
}
}
public static Long parseLong(String value) {
return parseLong(value, null);
}
public static Long parseLong(String value, Long defaultValue) {
try {
return Long.parseLong(value);
} catch (NumberFormatException e) {
return defaultValue;
}
}
}

View File

@ -0,0 +1,62 @@
package com.myoa.engineering.crawl.shopping.util;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.core.json.JsonReadFeature;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
import com.fasterxml.jackson.databind.json.JsonMapper;
import com.fasterxml.jackson.datatype.jdk8.Jdk8Module;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import com.fasterxml.jackson.module.paramnames.ParameterNamesModule;
public final class ObjectMapperFactory {
public static final ObjectMapper DEFAULT_MAPPER;
public static final ObjectMapper REDIS_MAPPER;
@Deprecated
public static final ObjectMapper LOGGING_MAPPER;
static {
DEFAULT_MAPPER = initDefaultMapper();
REDIS_MAPPER = initRedisMapper();
LOGGING_MAPPER = JsonMapper.builder()
.serializationInclusion(JsonInclude.Include.NON_EMPTY)
.build();
}
private ObjectMapperFactory() {
}
private static ObjectMapper initDefaultMapper() {
return JsonMapper.builder()
.configure(JsonReadFeature.ALLOW_UNQUOTED_FIELD_NAMES.mappedFeature(), true)
.configure(JsonReadFeature.ALLOW_UNESCAPED_CONTROL_CHARS.mappedFeature(), true)
.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES)
.disable(SerializationFeature.FAIL_ON_EMPTY_BEANS)
.disable(SerializationFeature.FAIL_ON_UNWRAPPED_TYPE_IDENTIFIERS)
.enable(DeserializationFeature.READ_UNKNOWN_ENUM_VALUES_AS_NULL)
.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS)
.serializationInclusion(JsonInclude.Include.NON_NULL)
.build()
.registerModule(new ParameterNamesModule())
.registerModule(new Jdk8Module())
.registerModule(new JavaTimeModule());
}
private static ObjectMapper initRedisMapper() {
return JsonMapper.builder()
.configure(JsonReadFeature.ALLOW_UNQUOTED_FIELD_NAMES.mappedFeature(), true)
.configure(JsonReadFeature.ALLOW_UNESCAPED_CONTROL_CHARS.mappedFeature(), true)
.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES)
.disable(SerializationFeature.FAIL_ON_EMPTY_BEANS)
.disable(SerializationFeature.FAIL_ON_UNWRAPPED_TYPE_IDENTIFIERS)
.enable(DeserializationFeature.READ_UNKNOWN_ENUM_VALUES_AS_NULL)
.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS)
.serializationInclusion(JsonInclude.Include.NON_NULL)
.build()
.registerModule(new ParameterNamesModule())
.registerModule(new Jdk8Module())
.registerModule(new JavaTimeModule());
}
}

View File

@ -0,0 +1,121 @@
/*
* Copyright (c) 2019 LINE Corporation. All rights reserved.
* LINE Corporation PROPRIETARY/CONFIDENTIAL. Use is subject to license terms.
*/
package com.myoa.engineering.crawl.shopping.util;
import java.util.Arrays;
import java.util.Collection;
/**
* NumberUtils
*
* @author Shin Woo-jin (lp12254@linecorp.com)
* @since 2019-10-28
*/
public final class ObjectUtils {
private ObjectUtils() {
}
/**
* Check if given object is null.
* <code>
* e == object == &gt; false e == null == &gt; true
* </code>
*
* @param e Target object
* @param <E> Unfixed specific type. If you want restrict specific interface, Copy and extend qualifier.
* @return Is null given object?
*/
public static <E> boolean isNullObject(final E e) {
return e == null;
}
/**
* Check if given object is not null.
* <code>
* e == object == &gt; false e == null == &gt; true
* </code>
*
* @param e Target object
* @param <E> Unfixed specific type. If you want restrict specific interface, Copy and extend qualifier.
* @return Is not null given object?
*/
public static <E> boolean isNotEmpty(final E e) {
return !isNullObject(e);
}
/**
* Check if there are any null object in given objects.
* <code>
* args == object = &gt; false args == object, object = &gt; false args == null, null, object = &gt; true args
* == null = &gt; true args == null, null = &gt; true
* </code>
*
* @param args Want to check objects that have null.
* @return Is there objects array has null?
*/
public static boolean hasNullObject(Object... args) {
return Arrays.stream(args).anyMatch(ObjectUtils::isNullObject);
}
/**
* Check given objects are not empty.
* <code>
* args == object = &gt; true args == object, object = &gt; true args == null, null, object = &gt; false args
* == null = &gt; false args == null, null = &gt; false
* </code>
*
* @param args Want to check objects that have null.
* @return Is there objects array has null?
*/
public static boolean hasAllObject(Object... args) {
return Arrays.stream(args).noneMatch(ObjectUtils::isNullObject);
}
/**
* Check if there are all null object in given objects.
* <code>
* args == object = &gt; false args == object, object = &gt; false args == null, null, object = &gt; false args
* == null = &gt; true args == null, null = &gt; true
* </code>
*
* @param args Want to check objects that have null.
* @return Is there null all of given objects?
*/
public static boolean hasAllNullObjects(final Object... args) {
return Arrays.stream(args).allMatch(ObjectUtils::isNullObject);
}
/**
* Check if given collection object is null or empty collecton.
* <code>
* e == null = &gt; true e == emptyCollection = &gt; true e == hasElement = &gt; false
* </code>
*
* @param e e is must be Collection object
* @param <E> E is must be extended Collection Class
* @return boolean. given collection is null or empty?
*/
public static <E extends Collection<?>> boolean isNullOrEmptyCollection(final E e) {
return e == null || e.isEmpty();
}
/**
* Get collection's size. Even it pointed null
*
* @param e e is must be Collection object
* @param <E> E is must be extended Collection Class
* @return integer value. given collection's size.
*/
public static <E extends Collection<?>> int getCollectionSize(final E e) {
if (isNullOrEmptyCollection(e)) {
return 0;
}
return e.size();
}
}

View File

@ -0,0 +1,12 @@
spring:
config:
activate:
on-profile: development
import:
- "configserver:http://192.168.0.100:11080"
server:
port: 20081
# import: optional:configserver:http://localhost:11080 # can be start up even config server was not found.

View File

@ -0,0 +1,18 @@
spring:
config:
activate:
on-profile: local
import:
- classpath:/datasource/local.yml
- classpath:/slack/local.yml
server:
port: 20080
# import: optional:configserver:http://localhost:11080 # can be start up even config server was not found.
feign:
client:
config:
default:
loggerLevel: FULL

View File

@ -0,0 +1,6 @@
spring:
config:
activate:
on-profile: production
import:
- "configserver:http://ppn-config-server:20080"

View File

@ -0,0 +1,28 @@
spring:
application:
name: crawler-shopping
main:
allow-bean-definition-overriding: true
profiles:
active: ${SPRING_ACTIVE_PROFILE:local}
group:
local: "local,datasource-local,webclient-local"
development: "development,datasource-development,webclient-development"
production: "production,datasource-production,webclient-production"
freemarker:
enabled: false
cloud:
config:
enabled: false
server:
port: 20080
error:
whitelabel:
enabled: false
management:
endpoints:
web:
exposure:
include: refresh,health

View File

@ -0,0 +1,15 @@
<?xml version="1.0" encoding="UTF-8"?>
<configuration>
<springProperty name="DEFAULT_LEVEL_CONFIG" source="log.defaultLevel" />
<springProfile name="local">
<include resource="logback/logback-development.xml" />
<logger name="org.apache.kafka" level="INFO" />
</springProfile>
<springProfile name="development">
<include resource="logback/logback-development.xml" />
<logger name="org.apache.kafka" level="INFO" />
</springProfile>
<springProfile name="production">
<include resource="logback/logback-production.xml" />
</springProfile>
</configuration>

View File

@ -0,0 +1,23 @@
<included>
<property name="FILE_LOG_PATTERN"
value="%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{45}:%L - %msg%n" />
<property name="LOG_FILE_BASE" value="lcp-benefit-benefit-api" />
<appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${DIRECTORY}/${LOG_FILE_BASE}_log</file>
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<fileNamePattern>${DIRECTORY}/${LOG_FILE_BASE}_log.%d{yyyyMMdd}.%i</fileNamePattern>
<timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
<maxFileSize>1000MB</maxFileSize>
</timeBasedFileNamingAndTriggeringPolicy>
<maxHistory>60</maxHistory>
</rollingPolicy>
<encoder>
<pattern>${FILE_LOG_PATTERN}</pattern>
<immediateFlush>${IMMEDIATE_FLUSH}</immediateFlush>
</encoder>
</appender>
<appender name="ASYNC_FILE" class="ch.qos.logback.classic.AsyncAppender">
<queueSize>1024</queueSize>
<appender-ref ref="FILE" />
</appender>
</included>

View File

@ -0,0 +1,19 @@
<?xml version="1.0" encoding="UTF-8"?>
<included>
<!-- =========== property BETA ========= -->
<property name="DEFAULT_LEVEL" value="${DEFAULT_LEVEL_CONFIG:-INFO}"/>
<!--file-->
<property name="DIRECTORY" value="/home1/www/logs/supervisor"/>
<property name="IMMEDIATE_FLUSH" value="true"/>
<!--nelo2-->
<property name="NELO2_LEVEL" value="WARN"/>
<!-- =========== include appender =========== -->
<include resource="org/springframework/boot/logging/logback/defaults.xml"/>
<include resource="org/springframework/boot/logging/logback/console-appender.xml"/>
<include resource="logback/component/logback-nelo2.xml"/>
<include resource="logback/component/logback-datachain.xml"/>
<!-- =========== root logger ============== -->
<root level="${DEFAULT_LEVEL}">
<appender-ref ref="CONSOLE"/>
</root>
</included>

View File

@ -0,0 +1,19 @@
<?xml version="1.0" encoding="UTF-8"?>
<included>
<!-- =========== property RELEASE ========= -->
<property name="DEFAULT_LEVEL" value="${DEFAULT_LEVEL_CONFIG:-INFO}"/>
<!--file-->
<property name="DIRECTORY" value="/home1/www/logs/supervisor"/>
<property name="IMMEDIATE_FLUSH" value="true"/>
<!--nelo2-->
<property name="NELO2_LEVEL" value="WARN"/>
<!-- =========== include appender =========== -->
<include resource="org/springframework/boot/logging/logback/defaults.xml"/>
<include resource="org/springframework/boot/logging/logback/console-appender.xml"/>
<include resource="logback/component/logback-nelo2.xml"/>
<include resource="logback/component/logback-datachain.xml"/>
<!-- =========== root logger ============== -->
<root level="${DEFAULT_LEVEL}">
<appender-ref ref="CONSOLE"/>
</root>
</included>

View File

@ -0,0 +1,6 @@
package com.myoa.engineering.crawl.shopping.crawlhandler.parser;
class PpomppuArticleParserV2Test {
}

View File

@ -0,0 +1,81 @@
package com.myoa.engineering.crawl.shopping.event.handler;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.ahocorasick.trie.Emit;
import org.ahocorasick.trie.Trie;
import org.jeasy.random.EasyRandom;
import org.jeasy.random.EasyRandomParameters;
import org.junit.jupiter.api.Test;
import java.util.Collection;
import java.util.List;
class ArticleUpsertEventListenerTest {
@Test
public void test1() {
List<String> baseData = List.of(
"[공홈]베베숲 시그니처 위드 블루 20팩 (25,990원/무료)2 [기타]",
"[현대H몰]10주년 스페셜 에디션 봉고데기 40mm 세트 (67,640원/무료)3 [가전/가구]",
"[SSG]필립스 면도기 칫솔 기획전( 437,000원~/무료)1 [기타]",
"(티몬)제스프리 골드키위 중대과 1.8kg내외 (카페,토페 14,823원/무배)3 [식품/건강]",
"[공식몰]연세생활건강 당뇨영양식 24팩 (33,500원/유배)1 [식품/건강]",
"[인터파크]크리스탈라이트 아이스티 에이드 온더고 60개입 멀티팩(17,800원/무료)6 [기타]",
"[인팍쇼핑]샤카웨어 액티브 맥스 헤비웨이트 10종 택 1 (7,350원/무료)7 [의류/잡화]"
);
Trie trie = Trie.builder()
.ignoreCase()
.addKeyword("블루")
.addKeyword("봉고")
.build();
List<Collection<Emit>> list = baseData.stream()
.map(trie::parseText)
.toList();
System.out.println(list);
}
@Test
void test2() throws InterruptedException {
EasyRandomParameters paramsKeyword = new EasyRandomParameters()
.stringLengthRange(2, 10);
EasyRandom GEN_KEYWORD = new EasyRandom(paramsKeyword);
EasyRandomParameters paramsSentence = new EasyRandomParameters()
.stringLengthRange(100, 1000);
EasyRandom GEN_SENTENCE = new EasyRandom(paramsSentence);
Trie.TrieBuilder trieBuilder = Trie.builder().ignoreOverlaps();
GEN_KEYWORD.objects(TestA.class, 1000)
.map(TestA::getFieldA)
.forEach(trieBuilder::addKeyword);
Trie trie = trieBuilder.build();
List<Collection<Emit>> listOfEmits = GEN_SENTENCE.objects(TestA.class, 2000)
.map(TestA::getFieldA)
.map(trie::parseText)
.toList();
for(Collection<Emit> emits : listOfEmits) {
System.out.println(emits);
System.out.println("--------------------------");
}
}
@Data
@AllArgsConstructor
@NoArgsConstructor
public class TestA {
private String fieldA;
}
}

View File

@ -0,0 +1,62 @@
package com.myoa.engineering.crawl.shopping.util;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.springframework.lang.NonNull;
import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import java.util.Scanner;
import java.util.stream.Collectors;
public final class TestDataUtils {
private static final ObjectMapper OBJECT_MAPPER;
static {
OBJECT_MAPPER = ObjectMapperFactory.DEFAULT_MAPPER;
}
private TestDataUtils() {
}
public static <T> T inputStreamToObject(@NonNull String resourcePath,
@NonNull Class<T> clazz) throws IOException {
return OBJECT_MAPPER.readValue(fileToString(resourcePath), clazz);
}
public static <T> List<T> inputStreamsToList(@NonNull Class<T> clazz,
@NonNull String... resourcePath) throws IOException {
return inputStreamsToList(Arrays.asList(resourcePath), clazz);
}
public static <T> List<T> inputStreamsToList(@NonNull List<String> resourcePath,
@NonNull Class<T> clazz) throws IOException {
return resourcePath.stream()
.map(e -> {
try {
return OBJECT_MAPPER.readValue(fileToString(e), clazz);
} catch (JsonProcessingException jsonProcessingException) {
jsonProcessingException.printStackTrace();
return null;
}
})
.collect(Collectors.toList());
}
public static <T> List<T> inputStreamToList(@NonNull String resourcePath,
@NonNull Class<T> clazz) throws IOException {
return (List<T>) OBJECT_MAPPER.readValue(
fileToString(resourcePath),
OBJECT_MAPPER.getTypeFactory().constructCollectionType(List.class, clazz));
}
public static String fileToString(@NonNull String resourcePath) {
final ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
final Scanner s = new Scanner(classLoader.getResourceAsStream(resourcePath)).useDelimiter("\\A");
return s.hasNext() ? s.next() : "";
}
}

View File

@ -0,0 +1,12 @@
<?xml version="1.0" encoding="UTF-8"?>
<included>
<!-- =========== property BETA ========= -->
<property name="DEFAULT_LEVEL" value="${DEFAULT_LEVEL_CONFIG:-INFO}"/>
<!-- =========== include appender =========== -->
<include resource="org/springframework/boot/logging/logback/defaults.xml"/>
<include resource="org/springframework/boot/logging/logback/console-appender.xml"/>
<!-- =========== root logger ============== -->
<root level="${DEFAULT_LEVEL}">
<appender-ref ref="CONSOLE"/>
</root>
</included>

View File

@ -0,0 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<configuratiown>
<springProperty name="DEFAULT_LEVEL_CONFIG" source="log.defaultLevel"/>
<include resource="logback-development.xml"/>
</configuratiown>

File diff suppressed because it is too large Load Diff

23
support/build.gradle Normal file
View File

@ -0,0 +1,23 @@
dependencies {
compileOnly 'org.projectlombok:lombok'
// https://projectreactor.io/docs/core/release/reference/#debug-activate
annotationProcessor 'org.projectlombok:lombok'
implementation 'org.springframework.boot:spring-boot-starter-web'
}
test {
useJUnitPlatform()
testLogging {
events "passed", "skipped", "failed"
}
}
jar {
enabled = true
}
bootJar {
enabled = false
}

View File

@ -0,0 +1,31 @@
{
"blocks": [
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": "기타)) `[쿠팡] 타푸코 이중 진공 스텐 세라믹 코팅 텀블러 (화이트만 11,130원/로켓와우무료) 4 [기타]`<https://www.ppomppu.co.kr/zboardview.php?id=ppomppu&page=1&divpage=69&category=1&&no=404126|LINK>"
},
"accessory": {
"type": "image",
"image_url": "cdn.ppomppu.co.kr/zboard/data3/2021/1121/m_20211121184835_zfyahnow.png",
"alt_text": "alt text for image"
}
},
{
"type": "divider"
},
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": "기타)) `[롯데온] 1매 149원 올국산 KF94새부리마스크 대형 200매 (29,920원/무료배송) 58 [기타]`<https://www.ppomppu.co.kr/zboardview.php?id=ppomppu&page=1&divpage=69&category=1&&no=404113|LINK>"
},
"accessory": {
"type": "image",
"image_url": "https://s3-media3.fl.yelpcdn.com/bphoto/c7ed05m9lC2EmA3Aruue7A/o.jpg",
"alt_text": "alt text for image"
}
}
]
}

View File

@ -0,0 +1,63 @@
package com.myoa.engineering.crawl.shopping.support.dto;
import java.io.Serializable;
import java.util.Map;
import lombok.Getter;
/**
* APIResponse
*
* @author Shin Woo-jin (woozu.shin@kakaoent.com)
* @since 2021-09-07
*/
@Getter
public class APIResponse<T> implements Serializable {
private static final long serialVersionUID = 1523350013713908487L;
private boolean success;
private T result;
private APIResponseError error;
public APIResponse(boolean success, T result, APIResponseError error) {
this.success = success;
this.error = error;
this.result = result;
}
public APIResponse(boolean success, T result) {
this.success = success;
this.result = result;
}
public APIResponse(boolean success, APIResponseError error) {
this.success = success;
this.error = error;
}
public APIResponse() {}
public static <T> APIResponse<T> success(T result) {
return new APIResponse<>(true, result);
}
public static APIResponse<Void> success() {
return new APIResponse<>(true, null);
}
public static <T> APIResponse<T> fail(T result, String code) {
return new APIResponse<T>(false, result, APIResponseError.of(code));
}
public static APIResponse<Void> fail(String code) {
return new APIResponse<Void>(false, APIResponseError.of(code));
}
public static APIResponse<Void> fail(String code, String message) {
return new APIResponse<Void>(false, APIResponseError.of(code, message));
}
public static <K, V> APIResponse<Void> fail(String code, String message, Map<K, V> reasons) {
return new APIResponse<Void>(false, APIResponseError.of(code, message, reasons));
}
}

View File

@ -0,0 +1,49 @@
package com.myoa.engineering.crawl.shopping.support.dto;
import java.io.Serializable;
import java.util.HashMap;
import java.util.Map;
import lombok.Getter;
/**
* ResponseError
*
* @author Shin Woo-jin (woozu.shin@kakaoent.com)
* @since 2021-09-07
*/
@Getter
public class APIResponseError<K, V> implements Serializable {
private static final long serialVersionUID = 931593091836887301L;
private String code;
private String message;
private Map<K, V> reasons;
public APIResponseError() {}
public APIResponseError(String code, String message, Map<K, V> reasons) {
this.code = code;
this.message = message;
this.reasons = reasons;
}
public static <K, V> APIResponseError<K, V> of(String code, String message, Map<K, V> reasons) {
return new APIResponseError<>(code, message, reasons);
}
public static APIResponseError<String, String> of(
String code, String message, String reasonKey, String reasonValue) {
final Map<String, String> reasons = new HashMap<>();
reasons.put(reasonKey, reasonValue);
return new APIResponseError<String, String>(code, message, reasons);
}
public static APIResponseError<Void, Void> of(String code, String message) {
return new APIResponseError<>(code, message, null);
}
public static APIResponseError<Void, Void> of(String code) {
return new APIResponseError<>(code, null, null);
}
}

View File

@ -0,0 +1,67 @@
package com.myoa.engineering.crawl.shopping.support.dto;
import com.myoa.engineering.crawl.shopping.support.dto.constant.PpomppuBoardName;
import lombok.Builder;
import lombok.Getter;
import lombok.NoArgsConstructor;
import java.io.Serializable;
import java.time.Instant;
import java.util.List;
/**
* SimpleMessageDTO
*
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-11-21
*/
@Getter
@NoArgsConstructor
public class BlockMessageDTO implements Serializable {
private static final long serialVersionUID = -6992039884035135523L;
private PpomppuBoardName boardName;
private String title;
private List<Block> blocks;
private String url;
private Instant publishedAt;
private Instant requestedAt;
@Builder
public BlockMessageDTO(PpomppuBoardName boardName, String title,
List<Block> blocks, String url, Instant publishedAt,
Instant requestedAt) {
this.boardName = boardName;
this.title = title;
this.blocks = blocks;
this.url = url;
this.publishedAt = publishedAt;
this.requestedAt = requestedAt;
}
@Getter
@NoArgsConstructor
public static class Block implements Serializable {
private static final long serialVersionUID = 3633781631892663709L;
private String text;
private String imageUrl;
private String altText;
public Block(String text, String imageUrl, String altText) {
this.text = text;
this.imageUrl = imageUrl;
this.altText = altText;
}
}
public static Block createBlock(String text, String imageUrl) {
return new Block(text, imageUrl, "");
}
public static Block createBlock(String text, String imageUrl, String altText) {
return new Block(text, imageUrl, altText);
}
}

View File

@ -0,0 +1,37 @@
package com.myoa.engineering.crawl.shopping.support.dto;
import java.io.Serializable;
import java.time.Instant;
import lombok.Builder;
import lombok.Getter;
import lombok.NoArgsConstructor;
/**
* SimpleMessageDTO
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-11-21
*
*/
@Getter
@NoArgsConstructor
public class SimpleMessageDTO implements Serializable {
private static final long serialVersionUID = 2203955567672404428L;
private String title;
private String body;
private String url;
private Instant publishedAt;
private Instant requestedAt;
@Builder
public SimpleMessageDTO(String title, String body, String url, Instant publishedAt, Instant requestedAt) {
this.title = title;
this.body = body;
this.url = url;
this.publishedAt = publishedAt;
this.requestedAt = requestedAt;
}
}

View File

@ -0,0 +1,13 @@
package com.myoa.engineering.crawl.shopping.support.dto.constant;
import lombok.AllArgsConstructor;
import lombok.Getter;
@Getter
@AllArgsConstructor
public enum CrawlTarget {
PPOMPPU_DOMESTIC,
PPOMPPU_OVERSEA,
FMKOREA,
;
}

View File

@ -0,0 +1,62 @@
package com.myoa.engineering.crawl.shopping.support.dto.constant;
import lombok.AllArgsConstructor;
import lombok.Getter;
import java.util.Arrays;
/**
* PpomppuBoardName
*
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2021-09-05
*/
@Getter
@AllArgsConstructor
public enum PpomppuBoardName {
UNKNOWN("", "UNKNOWN", false, false),
PPOMPPU_DOMESTIC_ALL("id=ppomppu", "전체", false, true),
PPOMPPU_DOMESTIC_ETC("id=ppomppu&category=1", "기타", true, true),
PPOMPPU_DOMESTIC_COMPUTER("id=ppomppu&category=4", "컴퓨터", true, true),
PPOMPPU_DOMESTIC_DIGITAL("id=ppomppu&category=5", "디지털", true, true),
PPOMPPU_DOMESTIC_FOOD("id=ppomppu&category=6", "식품/건강", true, true),
PPOMPPU_DOMESTIC_BOOK("id=ppomppu&category=8", "서적", true, true),
PPOMPPU_DOMESTIC_APPLIANCES("id=ppomppu&category=9", "가전/가구", true, true),
PPOMPPU_DOMESTIC_PARENTING("id=ppomppu&category=10", "육아", true, true),
PPOMPPU_DOMESTIC_GIFTCARD("id=ppomppu&category=11", "상품권", true, true),
PPOMPPU_DOMESTIC_CLOTHES("id=ppomppu&category=12", "의류/잡화", true, true),
PPOMPPU_DOMESTIC_COSMETIC("id=ppomppu&category=13", "화장품", true, true),
PPOMPPU_DOMESTIC_OUTDOOR("id=ppomppu&category=15", "등산/캠핑", true, true),
PPOMPPU_OVERSEA_ALL("id=ppomppu4", "전체", false, false),
PPOMPPU_OVERSEA_ETC("id=ppomppu4&category=1", "기타", true, false),
PPOMPPU_OVERSEA_APPLIANCES("id=ppomppu4&category=7", "가전", true, false),
PPOMPPU_OVERSEA_TVAV("id=ppomppu4&category=8", "TV/영상", true, false),
PPOMPPU_OVERSEA_COMPUTER("id=ppomppu4&category=3", "컴퓨터", true, false),
PPOMPPU_OVERSEA_DIGITAL("id=ppomppu4&category=4", "디지털", true, false),
PPOMPPU_OVERSEA_MOBILEACCESSORY("id=ppomppu4&category=9", "액세서리", false, true),
PPOMPPU_OVERSEA_CLOTHES("id=ppomppu4&category=5", "의류/잡화", true, false),
PPOMPPU_OVERSEA_WATCH("id=ppomppu4&category=2", "시계", true, false),
PPOMPPU_OVERSEA_SHOES("id=ppomppu4&category=11", "신발", true, false),
PPOMPPU_OVERSEA_FOOD("id=ppomppu4&category=10", "식품/건강", true, false),
PPOMPPU_OVERSEA_PARENTING("id=ppomppu4&category=6", "육아", true, false),
;
private String resourcePath;
private String menuName;
private boolean crawlWithDefaultTimer;
private boolean isDomestic;
public static final String PPOMPPU_URL = "https://www.ppomppu.co.kr";
public static String ofViewPageUrl(String articleUrl) {
return PPOMPPU_URL + "/zboard/" + articleUrl;
}
public static PpomppuBoardName ofBoardName(String boardName, boolean isDomestic) {
return Arrays.stream(values())
.filter(e -> e.getMenuName().equals(boardName) && e.isDomestic() == isDomestic)
.findFirst()
.orElse(UNKNOWN);
}
}

View File

@ -0,0 +1,18 @@
package com.myoa.engineering.crawl.shopping.support.util;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
/**
* DateUtil
* @author Shin Woo-jin (woo-jin.shin@linecorp.com)
* @since 2022-01-02
*
*/
public final class DateUtil {
private DateUtil() { }
public static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yy.MM.dd HH:mm:ss")
.withZone(ZoneId.of("Asia/Seoul"));
}

Some files were not shown because too many files have changed in this diff Show More