From cf2d7c615a47444dc8c9c95f0d9487b3eca87248 Mon Sep 17 00:00:00 2001 From: sanjeok77 <1+sanjeok77@noreply.localhost> Date: Tue, 25 Aug 2026 01:58:42 +0000 Subject: [PATCH] v0.3.4: update PpomppuScraper.kt --- .../data/remote/scraper/PpomppuScraper.kt | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/app/src/main/java/com/hotdeal/alarm/data/remote/scraper/PpomppuScraper.kt b/app/src/main/java/com/hotdeal/alarm/data/remote/scraper/PpomppuScraper.kt index d9fd0f6..2546620 100644 --- a/app/src/main/java/com/hotdeal/alarm/data/remote/scraper/PpomppuScraper.kt +++ b/app/src/main/java/com/hotdeal/alarm/data/remote/scraper/PpomppuScraper.kt @@ -3,6 +3,7 @@ package com.hotdeal.alarm.data.remote.scraper import android.util.Log import com.hotdeal.alarm.domain.model.HotDeal import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.async import kotlinx.coroutines.delay import kotlinx.coroutines.withContext import okhttp3.OkHttpClient @@ -13,7 +14,7 @@ import kotlin.random.Random /** * 뽐뿌 스크래퍼 - * - hotlist_flag=999 페이지에서 인기글 ID를 수집하여 HOT 뱃지 표시 + * - hotlist_flag=999 페이지와 일반 게시판을 병렬 스크래핑하여 HOT 뱃지 신속 감지 * - 기존 hotpop_bg_color CSS 클래스도 폴백으로 감지 */ class PpomppuScraper(client: OkHttpClient) : BaseScraper(client) { @@ -27,15 +28,13 @@ class PpomppuScraper(client: OkHttpClient) : BaseScraper(client) { override suspend fun scrape(board: String): Result> = withContext(Dispatchers.IO) { try { - // 1단계: 인기글(hotlist) 페이지에서 인기 게시물 ID 목록 수집 - val hotPostIds = fetchHotPostIds(board) - Log.d("Ppomppu", "인기글 ID ${hotPostIds.size}개 수집됨: $hotPostIds") - - // 2단계: 일반 게시판 페이지 스크래핑 val url = getBoardUrl(board) Log.d("Ppomppu", "스크래핑 시작: $url") + + // 1. 인기글 목록과 일반 게시판을 병렬(async)로 동시에 요청하여 딜레이 최소화 + val hotPostIdsDeferred = async { fetchHotPostIds(board) } - val delayTime = Random.nextLong(2000, 4000) + val delayTime = Random.nextLong(500, 1500) delay(delayTime) val userAgent = getRandomUserAgent() @@ -45,9 +44,12 @@ class PpomppuScraper(client: OkHttpClient) : BaseScraper(client) { .header("Accept-Language", "ko-KR,ko;q=0.9,en-US;q=0.8,en;q=0.7") .header("Accept-Charset", "utf-8,ISO-8859-1") .header("Referer", "https://www.ppomppu.co.kr/") - .timeout(30000) + .timeout(12000) .followRedirects(true) .get() + + val hotPostIds = hotPostIdsDeferred.await() + Log.d("Ppomppu", "인기글 ID ${hotPostIds.size}개 수집됨: $hotPostIds") val deals = mutableListOf() val rowElements: Elements = doc.select("tr.baseList") @@ -55,7 +57,7 @@ class PpomppuScraper(client: OkHttpClient) : BaseScraper(client) { var count = 0 rowElements.forEach { row -> - if (count >= 20) return@forEach + if (count >= 25) return@forEach try { val titleElement = row.selectFirst("a.baseList-title") ?: return@forEach @@ -108,14 +110,12 @@ class PpomppuScraper(client: OkHttpClient) : BaseScraper(client) { val hotUrl = "${baseUrl}zboard.php?id=$board&hotlist_flag=999" Log.d("Ppomppu", "인기글 페이지 조회: $hotUrl") - delay(Random.nextLong(1000, 2000)) - val doc = Jsoup.connect(hotUrl) .userAgent(getRandomUserAgent()) .header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8") .header("Accept-Language", "ko-KR,ko;q=0.9,en-US;q=0.8,en;q=0.7") .header("Referer", "https://www.ppomppu.co.kr/") - .timeout(15000) + .timeout(8000) .followRedirects(true) .get()