v0.3.4: update PpomppuScraper.kt
This commit is contained in:
1 parent
0597a002cd
commit
cf2d7c615a
1 file changed
+12
-12
@@ -3,6 +3,7 @@ package com.hotdeal.alarm.data.remote.scraper
|
|||||||
import android.util.Log
|
import android.util.Log
|
||||||
import com.hotdeal.alarm.domain.model.HotDeal
|
import com.hotdeal.alarm.domain.model.HotDeal
|
||||||
import kotlinx.coroutines.Dispatchers
|
import kotlinx.coroutines.Dispatchers
|
||||||
|
import kotlinx.coroutines.async
|
||||||
import kotlinx.coroutines.delay
|
import kotlinx.coroutines.delay
|
||||||
import kotlinx.coroutines.withContext
|
import kotlinx.coroutines.withContext
|
||||||
import okhttp3.OkHttpClient
|
import okhttp3.OkHttpClient
|
||||||
@@ -13,7 +14,7 @@ import kotlin.random.Random
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* 뽐뿌 스크래퍼
|
* 뽐뿌 스크래퍼
|
||||||
* - hotlist_flag=999 페이지에서 인기글 ID를 수집하여 HOT 뱃지 표시
|
* - hotlist_flag=999 페이지와 일반 게시판을 병렬 스크래핑하여 HOT 뱃지 신속 감지
|
||||||
* - 기존 hotpop_bg_color CSS 클래스도 폴백으로 감지
|
* - 기존 hotpop_bg_color CSS 클래스도 폴백으로 감지
|
||||||
*/
|
*/
|
||||||
class PpomppuScraper(client: OkHttpClient) : BaseScraper(client) {
|
class PpomppuScraper(client: OkHttpClient) : BaseScraper(client) {
|
||||||
@@ -27,15 +28,13 @@ class PpomppuScraper(client: OkHttpClient) : BaseScraper(client) {
|
|||||||
|
|
||||||
override suspend fun scrape(board: String): Result<List<HotDeal>> = withContext(Dispatchers.IO) {
|
override suspend fun scrape(board: String): Result<List<HotDeal>> = withContext(Dispatchers.IO) {
|
||||||
try {
|
try {
|
||||||
// 1단계: 인기글(hotlist) 페이지에서 인기 게시물 ID 목록 수집
|
|
||||||
val hotPostIds = fetchHotPostIds(board)
|
|
||||||
Log.d("Ppomppu", "인기글 ID ${hotPostIds.size}개 수집됨: $hotPostIds")
|
|
||||||
|
|
||||||
// 2단계: 일반 게시판 페이지 스크래핑
|
|
||||||
val url = getBoardUrl(board)
|
val url = getBoardUrl(board)
|
||||||
Log.d("Ppomppu", "스크래핑 시작: $url")
|
Log.d("Ppomppu", "스크래핑 시작: $url")
|
||||||
|
|
||||||
|
// 1. 인기글 목록과 일반 게시판을 병렬(async)로 동시에 요청하여 딜레이 최소화
|
||||||
|
val hotPostIdsDeferred = async { fetchHotPostIds(board) }
|
||||||
|
|
||||||
val delayTime = Random.nextLong(2000, 4000)
|
val delayTime = Random.nextLong(500, 1500)
|
||||||
delay(delayTime)
|
delay(delayTime)
|
||||||
|
|
||||||
val userAgent = getRandomUserAgent()
|
val userAgent = getRandomUserAgent()
|
||||||
@@ -45,9 +44,12 @@ class PpomppuScraper(client: OkHttpClient) : BaseScraper(client) {
|
|||||||
.header("Accept-Language", "ko-KR,ko;q=0.9,en-US;q=0.8,en;q=0.7")
|
.header("Accept-Language", "ko-KR,ko;q=0.9,en-US;q=0.8,en;q=0.7")
|
||||||
.header("Accept-Charset", "utf-8,ISO-8859-1")
|
.header("Accept-Charset", "utf-8,ISO-8859-1")
|
||||||
.header("Referer", "https://www.ppomppu.co.kr/")
|
.header("Referer", "https://www.ppomppu.co.kr/")
|
||||||
.timeout(30000)
|
.timeout(12000)
|
||||||
.followRedirects(true)
|
.followRedirects(true)
|
||||||
.get()
|
.get()
|
||||||
|
|
||||||
|
val hotPostIds = hotPostIdsDeferred.await()
|
||||||
|
Log.d("Ppomppu", "인기글 ID ${hotPostIds.size}개 수집됨: $hotPostIds")
|
||||||
|
|
||||||
val deals = mutableListOf<HotDeal>()
|
val deals = mutableListOf<HotDeal>()
|
||||||
val rowElements: Elements = doc.select("tr.baseList")
|
val rowElements: Elements = doc.select("tr.baseList")
|
||||||
@@ -55,7 +57,7 @@ class PpomppuScraper(client: OkHttpClient) : BaseScraper(client) {
|
|||||||
|
|
||||||
var count = 0
|
var count = 0
|
||||||
rowElements.forEach { row ->
|
rowElements.forEach { row ->
|
||||||
if (count >= 20) return@forEach
|
if (count >= 25) return@forEach
|
||||||
|
|
||||||
try {
|
try {
|
||||||
val titleElement = row.selectFirst("a.baseList-title") ?: return@forEach
|
val titleElement = row.selectFirst("a.baseList-title") ?: return@forEach
|
||||||
@@ -108,14 +110,12 @@ class PpomppuScraper(client: OkHttpClient) : BaseScraper(client) {
|
|||||||
val hotUrl = "${baseUrl}zboard.php?id=$board&hotlist_flag=999"
|
val hotUrl = "${baseUrl}zboard.php?id=$board&hotlist_flag=999"
|
||||||
Log.d("Ppomppu", "인기글 페이지 조회: $hotUrl")
|
Log.d("Ppomppu", "인기글 페이지 조회: $hotUrl")
|
||||||
|
|
||||||
delay(Random.nextLong(1000, 2000))
|
|
||||||
|
|
||||||
val doc = Jsoup.connect(hotUrl)
|
val doc = Jsoup.connect(hotUrl)
|
||||||
.userAgent(getRandomUserAgent())
|
.userAgent(getRandomUserAgent())
|
||||||
.header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
|
.header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
|
||||||
.header("Accept-Language", "ko-KR,ko;q=0.9,en-US;q=0.8,en;q=0.7")
|
.header("Accept-Language", "ko-KR,ko;q=0.9,en-US;q=0.8,en;q=0.7")
|
||||||
.header("Referer", "https://www.ppomppu.co.kr/")
|
.header("Referer", "https://www.ppomppu.co.kr/")
|
||||||
.timeout(15000)
|
.timeout(8000)
|
||||||
.followRedirects(true)
|
.followRedirects(true)
|
||||||
.get()
|
.get()
|
||||||
|
|
||||||
|
|||||||
Reference in new issue
Block a user