MyoaEngineering
/
ShoppingCrawler
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

[NO-ISSUE] Add fake headers

This commit is contained in:
woozu-shin 2024-05-17 11:30:22 +09:00
parent 47799bb94d
commit 0365f64984
2 changed files with 14 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/crawlhandler/FmkoreaCrawlHandler.java
View File

@ -8,9 +8,7 @@ import com.myoa.engineering.crawl.shopping.support.dto.constant.CrawlTarget;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map;
import java.util.stream.Stream; import java.util.stream.Stream;
@Slf4j @Slf4j
@ -53,11 +51,4 @@ public class FmkoreaCrawlHandler implements CrawlHandler {
articleCommandService.upsert(merged); articleCommandService.upsert(merged);
} }
private Map<String, String> generateRequestParams(int pageId, String cookie) {
Map<String, String> params = new HashMap<>();
params.put("mid", "hotdeal");
params.put("page", String.valueOf(pageId));
params.put("Cookie", cookie);
return params;
}
} }

15
shopping-crawler/src/main/java/com/myoa/engineering/crawl/shopping/infra/client/fmkorea/FmkoreaBoardClientV2.java
View File

@ -1,7 +1,7 @@
package com.myoa.engineering.crawl.shopping.infra.client.fmkorea; package com.myoa.engineering.crawl.shopping.infra.client.fmkorea;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.http.HttpStatus; import org.springframework.http.HttpHeaders;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import org.springframework.web.reactive.function.client.WebClient; import org.springframework.web.reactive.function.client.WebClient;
@ -18,6 +18,19 @@ public class FmkoreaBoardClientV2 {
.codecs(configurer -> configurer.defaultCodecs() .codecs(configurer -> configurer.defaultCodecs()
.maxInMemorySize(2 * 1024 * 1024)) .maxInMemorySize(2 * 1024 * 1024))
.baseUrl("https://www.fmkorea.com") .baseUrl("https://www.fmkorea.com")
.defaultHeader(HttpHeaders.USER_AGENT, "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3")
.defaultHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7\n")
.defaultHeader(HttpHeaders.REFERER, "https://www.fmkorea.com")
.defaultHeader(HttpHeaders.ACCEPT_ENCODING, "gzip, deflate, br")
.defaultHeader(HttpHeaders.ACCEPT_LANGUAGE, "ko-KR,ko;q=0.9,en-US;q=0.8,en;q=0.7")
.defaultHeader(HttpHeaders.CACHE_CONTROL, "max-age=0")
.defaultHeader("Sec-Ch-Ua", "\"Whale\";v=\"3\", \"Not-A.Brand\";v=\"8\", \"Chromium\";v=\"122\"")
.defaultHeader("Sec-Ch-Ua-Mobile", "?0")
.defaultHeader("Sec-Ch-Ua-Platform", "\"macOS\"")
.defaultHeader("Sec-Fetch-Dest", "document")
.defaultHeader("Sec-Fetch-Mode", "navigate")
.defaultHeader("Sec-Fetch-Site", "same-origin")
.defaultHeader("Upgrade-Insecure-Requests", "1")
.build(); .build();
} }