Implement ProcessorAPIWebClient

.gitignore

@@ -35,3 +35,5 @@ out/
 
 ### VS Code ###
 .vscode/
+
+temppassword.yml

gradlew.bat

@@ -1,89 +1,89 @@
-@rem
+@rem
-@rem Copyright 2015 the original author or authors.
+@rem Copyright 2015 the original author or authors.
-@rem
+@rem
-@rem Licensed under the Apache License, Version 2.0 (the "License");
+@rem Licensed under the Apache License, Version 2.0 (the "License");
-@rem you may not use this file except in compliance with the License.
+@rem you may not use this file except in compliance with the License.
-@rem You may obtain a copy of the License at
+@rem You may obtain a copy of the License at
-@rem
+@rem
-@rem      https://www.apache.org/licenses/LICENSE-2.0
+@rem      https://www.apache.org/licenses/LICENSE-2.0
-@rem
+@rem
-@rem Unless required by applicable law or agreed to in writing, software
+@rem Unless required by applicable law or agreed to in writing, software
-@rem distributed under the License is distributed on an "AS IS" BASIS,
+@rem distributed under the License is distributed on an "AS IS" BASIS,
-@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-@rem See the License for the specific language governing permissions and
+@rem See the License for the specific language governing permissions and
-@rem limitations under the License.
+@rem limitations under the License.
-@rem
+@rem
-
+
-@if "%DEBUG%" == "" @echo off
+@if "%DEBUG%" == "" @echo off
-@rem ##########################################################################
+@rem ##########################################################################
-@rem
+@rem
-@rem  Gradle startup script for Windows
+@rem  Gradle startup script for Windows
-@rem
+@rem
-@rem ##########################################################################
+@rem ##########################################################################
-
+
-@rem Set local scope for the variables with windows NT shell
+@rem Set local scope for the variables with windows NT shell
-if "%OS%"=="Windows_NT" setlocal
+if "%OS%"=="Windows_NT" setlocal
-
+
-set DIRNAME=%~dp0
+set DIRNAME=%~dp0
-if "%DIRNAME%" == "" set DIRNAME=.
+if "%DIRNAME%" == "" set DIRNAME=.
-set APP_BASE_NAME=%~n0
+set APP_BASE_NAME=%~n0
-set APP_HOME=%DIRNAME%
+set APP_HOME=%DIRNAME%
-
+
-@rem Resolve any "." and ".." in APP_HOME to make it shorter.
+@rem Resolve any "." and ".." in APP_HOME to make it shorter.
-for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi
+for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi
-
+
-@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
-set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m"
+set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m"
-
+
-@rem Find java.exe
+@rem Find java.exe
-if defined JAVA_HOME goto findJavaFromJavaHome
+if defined JAVA_HOME goto findJavaFromJavaHome
-
+
-set JAVA_EXE=java.exe
+set JAVA_EXE=java.exe
-%JAVA_EXE% -version >NUL 2>&1
+%JAVA_EXE% -version >NUL 2>&1
-if "%ERRORLEVEL%" == "0" goto execute
+if "%ERRORLEVEL%" == "0" goto execute
-
+
-echo.
+echo.
-echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
-echo.
+echo.
-echo Please set the JAVA_HOME variable in your environment to match the
+echo Please set the JAVA_HOME variable in your environment to match the
-echo location of your Java installation.
+echo location of your Java installation.
-
+
-goto fail
+goto fail
-
+
-:findJavaFromJavaHome
+:findJavaFromJavaHome
-set JAVA_HOME=%JAVA_HOME:"=%
+set JAVA_HOME=%JAVA_HOME:"=%
-set JAVA_EXE=%JAVA_HOME%/bin/java.exe
+set JAVA_EXE=%JAVA_HOME%/bin/java.exe
-
+
-if exist "%JAVA_EXE%" goto execute
+if exist "%JAVA_EXE%" goto execute
-
+
-echo.
+echo.
-echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
+echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
-echo.
+echo.
-echo Please set the JAVA_HOME variable in your environment to match the
+echo Please set the JAVA_HOME variable in your environment to match the
-echo location of your Java installation.
+echo location of your Java installation.
-
+
-goto fail
+goto fail
-
+
-:execute
+:execute
-@rem Setup the command line
+@rem Setup the command line
-
+
-set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
+set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
-
+
-
+
-@rem Execute Gradle
+@rem Execute Gradle
-"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %*
+"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %*
-
+
-:end
+:end
-@rem End local scope for the variables with windows NT shell
+@rem End local scope for the variables with windows NT shell
-if "%ERRORLEVEL%"=="0" goto mainEnd
+if "%ERRORLEVEL%"=="0" goto mainEnd
-
+
-:fail
+:fail
-rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
+rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
-rem the _cmd.exe /c_ return code!
+rem the _cmd.exe /c_ return code!
-if  not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
+if  not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
-exit /b 1
+exit /b 1
-
+
-:mainEnd
+:mainEnd
-if "%OS%"=="Windows_NT" endlocal
+if "%OS%"=="Windows_NT" endlocal
-
+
-:omega
+:omega

processor/build.gradle

@@ -0,0 +1,23 @@
+dependencies {
+    developmentOnly 'org.springframework.boot:spring-boot-devtools'
+    runtimeOnly 'com.h2database:h2'
+    runtimeOnly 'mysql:mysql-connector-java'
+    compileOnly 'org.projectlombok:lombok'
+
+    implementation project(':support')
+    // https://projectreactor.io/docs/core/release/reference/#debug-activate
+    implementation 'org.springframework.boot:spring-boot-starter-webflux'
+    implementation 'org.telegram:telegrambots:5.3.0'
+
+    annotationProcessor 'org.springframework.boot:spring-boot-configuration-processor'
+    annotationProcessor 'org.projectlombok:lombok'
+
+    testImplementation 'org.springframework.boot:spring-boot-starter-test'
+}
+
+test {
+    useJUnitPlatform()
+    testLogging {
+        events "passed", "skipped", "failed"
+    }
+}

processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/ProcessorApplication.java

@@ -1,10 +1,19 @@
 package com.myoa.engineering.crawl.ppomppu.processor;
 
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+
 /**
  * ProcessorApplication
  * @author Shin Woo-jin (woo-jin.shin@linecorp.com)
  * @since 2021-08-20
  *
  */
+@SpringBootApplication
 public class ProcessorApplication {
+
+    public static void main(String[] args) {
+        SpringApplication.run(ProcessorApplication.class, args);
+    }
+
 }

processor/src/main/java/com/myoa/engineering/crawl/ppomppu/processor/controller/CrawlAPIController.java

@@ -0,0 +1,30 @@
+package com.myoa.engineering.crawl.ppomppu.processor.controller;
+
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+
+import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
+
+import lombok.extern.slf4j.Slf4j;
+import reactor.core.publisher.Mono;
+
+/**
+ * CrawlAPIController
+ * @author Shin Woo-jin (woo-jin.shin@linecorp.com)
+ * @since 2021-09-05
+ *
+ */
+@Slf4j
+@RestController
+@RequestMapping("/api/v1/crawl")
+public class CrawlAPIController {
+
+    @PostMapping("/boards/{boardName}")
+    public Mono<String> crawlBoard(@PathVariable("boardName") PpomppuBoardName boardName) {
+        log.info("got request... {}", boardName);
+        return Mono.just(boardName.getBoardPath());
+    }
+
+}

processor/src/main/resources/application-development.yml

@@ -0,0 +1,6 @@
+spring:
+  config:
+    activate:
+      on-profile: development
+    import:
+      - classpath:/development/webclient.yml

processor/src/main/resources/application.yml

@@ -0,0 +1,14 @@
+spring:
+  main:
+    allow-bean-definition-overriding: true
+  profiles:
+    active: development
+  freemarker:
+    enabled: false
+
+
+server:
+  port: 20081
+  error:
+    whitelabel:
+      enabled: false

receiver/build.gradle

@@ -7,6 +7,7 @@ dependencies {
     implementation project(':support')
     // https://projectreactor.io/docs/core/release/reference/#debug-activate
     implementation 'org.springframework.boot:spring-boot-starter-webflux'
+    implementation 'org.springframework.boot:spring-boot-configuration-processor'
     implementation 'org.telegram:telegrambots:5.3.0'
 
     annotationProcessor 'org.springframework.boot:spring-boot-configuration-processor'

receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/ReceiverApplication.java

@@ -2,6 +2,9 @@ package com.myoa.engineering.crawl.ppomppu.receiver;
 
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.boot.context.properties.EnableConfigurationProperties;
+
+import com.myoa.engineering.crawl.ppomppu.receiver.configuration.properties.TelegramBotProperties;
 
 /**
  * ReceiverApplication
@@ -10,6 +13,7 @@ import org.springframework.boot.autoconfigure.SpringBootApplication;
  *
  */
 @SpringBootApplication
+@EnableConfigurationProperties({ TelegramBotProperties.class })
 public class ReceiverApplication {
 
     public static void main(String[] args) {

receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/configuration/TelegramBotConfiguration.java

@@ -8,8 +8,9 @@ import org.telegram.telegrambots.meta.TelegramBotsApi;
 import org.telegram.telegrambots.meta.exceptions.TelegramApiException;
 import org.telegram.telegrambots.updatesreceivers.DefaultBotSession;
 
-import com.myoa.engineering.crawl.ppomppu.receiver.handler.message.MessageHandler;
+import com.myoa.engineering.crawl.ppomppu.receiver.configuration.properties.TelegramBotProperties;
 import com.myoa.engineering.crawl.ppomppu.receiver.dispatch.MessageDispatcher;
+import com.myoa.engineering.crawl.ppomppu.receiver.handler.message.MessageHandler;
 
 /**
  * TelegramBotConfiguration
@@ -20,9 +21,6 @@ import com.myoa.engineering.crawl.ppomppu.receiver.dispatch.MessageDispatcher;
 @Configuration
 public class TelegramBotConfiguration {
 
-    private static final String BOT_TOKEN = ""; // TODO extract to property
-    private static final String BOT_NAME = ""; // TODO extract to property
-
     @Bean
     public TelegramBotsApi telegramBotsApi(MessageDispatcher messageDispatcher) throws TelegramApiException {
         TelegramBotsApi api = new TelegramBotsApi(DefaultBotSession.class);
@@ -31,7 +29,8 @@ public class TelegramBotConfiguration {
     }
 
     @Bean
-    public MessageDispatcher messageDispatcher(List<MessageHandler> messageHandlers) {
+    public MessageDispatcher messageDispatcher(List<MessageHandler> messageHandlers,
-        return new MessageDispatcher(messageHandlers, BOT_TOKEN, BOT_NAME);
+                                               TelegramBotProperties botProperties) {
+        return new MessageDispatcher(messageHandlers, botProperties.getName(), botProperties.getToken());
     }
 }

receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/configuration/properties/TelegramBotProperties.java

@@ -0,0 +1,25 @@
+package com.myoa.engineering.crawl.ppomppu.receiver.configuration.properties;
+
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.boot.context.properties.ConstructorBinding;
+
+import lombok.Getter;
+
+/**
+ * TelegramBotProperties
+ * @author Shin Woo-jin (woo-jin.shin@linecorp.com)
+ * @since 2021-09-05
+ *
+ */
+@Getter
+@ConstructorBinding
+@ConfigurationProperties(prefix = "telegram.bot")
+public class TelegramBotProperties {
+    private final String name;
+    private final String token;
+
+    public TelegramBotProperties(final String name, final String token) {
+        this.name = name;
+        this.token = token;
+    }
+}

receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/controller/v1/EventAPIController.java

@@ -0,0 +1,33 @@
+package com.myoa.engineering.crawl.ppomppu.receiver.controller.v1;
+
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+
+import com.myoa.engineering.crawl.ppomppu.receiver.service.ProcessorAPIService;
+import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
+
+import reactor.core.publisher.Mono;
+
+/**
+ * EventAPIController
+ * @author Shin Woo-jin (woo-jin.shin@linecorp.com)
+ * @since 2021-09-05
+ *
+ */
+@RestController
+@RequestMapping("/api/v1/")
+public class EventAPIController {
+
+    private final ProcessorAPIService processorAPIService;
+
+    public EventAPIController(ProcessorAPIService processorAPIService) {
+        this.processorAPIService = processorAPIService;
+    }
+
+    @PostMapping("/exploit/parsePpomppuRSS/{boardName}")
+    public Mono<String> parsePpomppuRSS(@PathVariable("boardName") PpomppuBoardName boardName) {
+        return processorAPIService.emitParseEvent(boardName);
+    }
+}

receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/dispatch/MessageDispatcher.java

@@ -14,13 +14,13 @@ import lombok.extern.slf4j.Slf4j;
 public class MessageDispatcher extends TelegramLongPollingBot {
 
     private final List<MessageHandler> messageHandlers;
-    private final String botToken;
     private final String botName;
+    private final String botToken;
 
-    public MessageDispatcher(List<MessageHandler> messageHandlers, String botToken, String botName) {
+    public MessageDispatcher(List<MessageHandler> messageHandlers, String botName, String botToken) {
         this.messageHandlers = messageHandlers;
-        this.botToken = botToken;
         this.botName = botName;
+        this.botToken = botToken;
     }
 
     @Override

receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/handler/message/text/CommonTextHandler.java

@@ -25,7 +25,6 @@ public class CommonTextHandler implements TextMessageHandler {
     @Override
     public void handle(Message message) {
         log.info("CommonTextHandler : {}", message.getText());
-
     }
 
 }

receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/infrastructure/client/ProcessorAPIWebClient.java

@@ -1,22 +1,32 @@
 package com.myoa.engineering.crawl.ppomppu.receiver.infrastructure.client;
 
 import org.springframework.beans.factory.annotation.Value;
+import org.springframework.core.ParameterizedTypeReference;
 import org.springframework.stereotype.Component;
 import org.springframework.web.reactive.function.client.WebClient;
 
-import java.util.List;
+import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
+
+import reactor.core.publisher.Mono;
+import reactor.core.scheduler.Schedulers;
 
 @Component
 public class ProcessorAPIWebClient {
 
-
+    @Value("${webclient.base-url}")
-    @Value("${webclient.some}")
     private String baseUrl;
 
     private final WebClient webClient;
 
     public ProcessorAPIWebClient(WebClient.Builder webClientBuilder) {
-        this.webClient = webClientBuilder.baseUrl("soundhoundfound-processor:20080")
+        this.webClient = webClientBuilder.baseUrl("http://localhost:20081")
-                .build();
+                                         .build();
+    }
+
+    public Mono<String> emitParseEvent(PpomppuBoardName boardName) {
+        return webClient.post()
+                        .uri("/api/v1/crawl/boards/{boardName}", boardName)
+                        .exchangeToMono(e -> e.bodyToMono(new ParameterizedTypeReference<String>() {}))
+                        .publishOn(Schedulers.boundedElastic());
     }
 }

receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/service/ProcessorAPIService.java

@@ -0,0 +1,30 @@
+package com.myoa.engineering.crawl.ppomppu.receiver.service;
+
+import org.springframework.stereotype.Service;
+
+import com.myoa.engineering.crawl.ppomppu.receiver.infrastructure.client.ProcessorAPIWebClient;
+import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
+
+import lombok.extern.slf4j.Slf4j;
+import reactor.core.publisher.Mono;
+
+/**
+ * ProcessorAPIService
+ * @author Shin Woo-jin (woo-jin.shin@linecorp.com)
+ * @since 2021-09-05
+ *
+ */
+@Slf4j
+@Service
+public class ProcessorAPIService {
+
+    private final ProcessorAPIWebClient processorAPIWebClient;
+
+    public ProcessorAPIService(ProcessorAPIWebClient processorAPIWebClient) {
+        this.processorAPIWebClient = processorAPIWebClient;
+    }
+
+    public Mono<String> emitParseEvent(PpomppuBoardName boardName) {
+        return processorAPIWebClient.emitParseEvent(boardName);
+    }
+}

receiver/src/main/java/com/myoa/engineering/crawl/ppomppu/receiver/shceduler/ParseEventEmitter.java

@@ -0,0 +1,40 @@
+package com.myoa.engineering.crawl.ppomppu.receiver.shceduler;
+
+import org.springframework.scheduling.annotation.EnableScheduling;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Component;
+
+import com.myoa.engineering.crawl.ppomppu.receiver.service.ProcessorAPIService;
+import com.myoa.engineering.crawl.ppomppu.support.dto.code.PpomppuBoardName;
+
+import lombok.extern.slf4j.Slf4j;
+
+/**
+ * ParseEventEmitter
+ * @author Shin Woo-jin (woo-jin.shin@linecorp.com)
+ * @since 2021-09-05
+ *
+ */
+@Slf4j
+@Component
+@EnableScheduling
+public class ParseEventEmitter {
+
+    private final ProcessorAPIService processorAPIService;
+
+    public ParseEventEmitter(ProcessorAPIService processorAPIService) {
+        this.processorAPIService = processorAPIService;
+    }
+
+    @Scheduled(fixedRate = 60 * 1000L)
+    public void emitDomesticBoard() {
+        log.info("[emitDomesticBoard] trigger fired!");
+        processorAPIService.emitParseEvent(PpomppuBoardName.PPOMPPU_DOMESTIC).block();
+    }
+
+    @Scheduled(fixedRate = 300 * 1000L)
+    public void emitOverseaBoard() {
+        log.info("[emitOverseaBoard] trigger fired!");
+        processorAPIService.emitParseEvent(PpomppuBoardName.PPOMPPU_OVERSEA).block();
+    }
+}

receiver/src/main/resources/application-development.yml

@@ -3,4 +3,5 @@ spring:
     activate:
       on-profile: development
     import:
-      - classpath:/development/webclient.yml
+      - classpath:/webclient-development.yml
+      - classpath:/temppassword.yml

receiver/src/main/resources/application.yml

@@ -6,5 +6,5 @@ spring:
   freemarker:
     enabled: false
 
-webclient:
+server:
-  some: not-test
+  port: 20080

receiver/src/main/resources/webclient-development.yml

@@ -0,0 +1,5 @@
+webclient:
+  base-url: http://localhost:20081
+  units:
+    - unit-name: processor-api
+      base-url: http://localhost:20081

receiver/src/main/resources/webclient-production.yml

@@ -0,0 +1,5 @@
+webclient:
+  base-url: http://ppomppu_notifier_processor:20080
+  units:
+    - unit-name: processor-api
+      base-url: http://ppomppu_notifier_processor:20080

support/build.gradle

@@ -0,0 +1,17 @@
+dependencies {
+    runtimeOnly 'mysql:mysql-connector-java'
+    compileOnly 'org.projectlombok:lombok'
+
+    // https://projectreactor.io/docs/core/release/reference/#debug-activate
+
+    annotationProcessor 'org.springframework.boot:spring-boot-configuration-processor'
+    annotationProcessor 'org.projectlombok:lombok'
+
+}
+
+test {
+    useJUnitPlatform()
+    testLogging {
+        events "passed", "skipped", "failed"
+    }
+}

support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/dto/code/PpomppuBoardName.java

@@ -0,0 +1,24 @@
+package com.myoa.engineering.crawl.ppomppu.support.dto.code;
+
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+
+/**
+ * PpomppuBoardName
+ * @author Shin Woo-jin (woo-jin.shin@linecorp.com)
+ * @since 2021-09-05
+ *
+ */
+@Getter
+@NoArgsConstructor
+public enum PpomppuBoardName {
+    PPOMPPU_DOMESTIC("ppomppu"),
+    PPOMPPU_OVERSEA("ppomppu4"),
+    ;
+
+    private String boardPath;
+
+    PpomppuBoardName(String boardPath) {
+        this.boardPath = boardPath;
+    }
+}

support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/util/NumberUtil.java

@@ -1,4 +1,4 @@
-package com.myoa.engineering.music.soundhoundfound.support.util;
+package com.myoa.engineering.crawl.ppomppu.support.util;
 
 public final class NumberUtil {
 

support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/util/ObjectUtil.java

@@ -1,4 +1,4 @@
-package com.myoa.engineering.music.soundhoundfound.support.util;
+package com.myoa.engineering.crawl.ppomppu.support.util;
 
 public final class ObjectUtil {
 

support/src/main/java/com/myoa/engineering/crawl/ppomppu/support/webclient/WebClientBaseScan.java

@@ -0,0 +1,4 @@
+package com.myoa.engineering.crawl.ppomppu.support.webclient;
+
+public interface WebClientBaseScan {
+}

support/src/main/java/com/myoa/engineering/music/soundhoundfound/support/webclient/WebClientBaseScan.java

@@ -1,4 +0,0 @@
-package com.myoa.engineering.music.soundhoundfound.support.webclient;
-
-public interface WebClientBaseScan {
-}