From 0365f64984d323e32bac4697afaf78469539d685 Mon Sep 17 00:00:00 2001 From: woozu-shin Date: Fri, 17 May 2024 11:30:22 +0900 Subject: [PATCH] [NO-ISSUE] Add fake headers --- .../crawlhandler/FmkoreaCrawlHandler.java | 9 --------- .../client/fmkorea/FmkoreaBoardClientV2.java | 15 ++++++++++++++- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/crawlhandler/FmkoreaCrawlHandler.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/crawlhandler/FmkoreaCrawlHandler.java index 522d7f1..0158b3b 100644 --- a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/crawlhandler/FmkoreaCrawlHandler.java +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/crawlhandler/FmkoreaCrawlHandler.java @@ -8,9 +8,7 @@ import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.stream.Stream; @Slf4j @@ -53,11 +51,4 @@ public class FmkoreaCrawlHandler implements CrawlHandler { articleCommandService.upsert(merged); } - private Map generateRequestParams(int pageId, String cookie) { - Map params = new HashMap<>(); - params.put("mid", "hotdeal"); - params.put("page", String.valueOf(pageId)); - params.put("Cookie", cookie); - return params; - } } diff --git a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/fmkorea/FmkoreaBoardClientV2.java b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/fmkorea/FmkoreaBoardClientV2.java index c84cf00..43f06fa 100644 --- a/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/fmkorea/FmkoreaBoardClientV2.java +++ b/shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/fmkorea/FmkoreaBoardClientV2.java @@ -1,7 +1,7 @@ package com.myoa.engineering.crawl.shopping.infra.client.fmkorea; import lombok.extern.slf4j.Slf4j; -import org.springframework.http.HttpStatus; +import org.springframework.http.HttpHeaders; import org.springframework.stereotype.Component; import org.springframework.web.reactive.function.client.WebClient; @@ -18,6 +18,19 @@ public class FmkoreaBoardClientV2 { .codecs(configurer -> configurer.defaultCodecs() .maxInMemorySize(2 * 1024 * 1024)) .baseUrl("https://www.fmkorea.com") + .defaultHeader(HttpHeaders.USER_AGENT, "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3") + .defaultHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7\n") + .defaultHeader(HttpHeaders.REFERER, "https://www.fmkorea.com") + .defaultHeader(HttpHeaders.ACCEPT_ENCODING, "gzip, deflate, br") + .defaultHeader(HttpHeaders.ACCEPT_LANGUAGE, "ko-KR,ko;q=0.9,en-US;q=0.8,en;q=0.7") + .defaultHeader(HttpHeaders.CACHE_CONTROL, "max-age=0") + .defaultHeader("Sec-Ch-Ua", "\"Whale\";v=\"3\", \"Not-A.Brand\";v=\"8\", \"Chromium\";v=\"122\"") + .defaultHeader("Sec-Ch-Ua-Mobile", "?0") + .defaultHeader("Sec-Ch-Ua-Platform", "\"macOS\"") + .defaultHeader("Sec-Fetch-Dest", "document") + .defaultHeader("Sec-Fetch-Mode", "navigate") + .defaultHeader("Sec-Fetch-Site", "same-origin") + .defaultHeader("Upgrade-Insecure-Requests", "1") .build(); }