Initial commit: HotDeal Alarm Android App

Features:
- Multi-site hot deal scraping (Ppomppu, Clien, Ruriweb, Coolenjoy)
- Site filter with color-coded badges
- Board display names (e.g., ppomppu8 -> 알리뽐뿌)
- Anti-bot protection with request delays and User-Agent rotation
- Keyword matching and notifications
- Material Design 3 UI
This commit is contained in:
sanjeok77
2026-03-04 01:29:34 +09:00
commit 25a349f273
84 changed files with 6455 additions and 0 deletions
@@ -0,0 +1,56 @@
package com.hotdeal.alarm.data.local.db
import androidx.room.Database
import androidx.room.RoomDatabase
import androidx.room.TypeConverters
import androidx.room.migration.Migration
import androidx.sqlite.db.SupportSQLiteDatabase
import com.hotdeal.alarm.data.local.db.dao.HotDealDao
import com.hotdeal.alarm.data.local.db.dao.KeywordDao
import com.hotdeal.alarm.data.local.db.dao.SiteConfigDao
import com.hotdeal.alarm.data.local.db.entity.HotDealEntity
import com.hotdeal.alarm.data.local.db.entity.KeywordEntity
import com.hotdeal.alarm.data.local.db.entity.SiteConfigEntity
/**
* 핫딜 알람 데이터베이스
*/
@Database(
entities = [
HotDealEntity::class,
SiteConfigEntity::class,
KeywordEntity::class
],
version = 3,
exportSchema = false
)
@TypeConverters(Converters::class)
abstract class AppDatabase : RoomDatabase() {
abstract fun hotDealDao(): HotDealDao
abstract fun siteConfigDao(): SiteConfigDao
abstract fun keywordDao(): KeywordDao
companion object {
const val DATABASE_NAME = "hotdeal_alarm_db"
/**
* Migration 1 -> 2: 키워드 매칭 모드 추가
*/
val MIGRATION_1_2 = object : Migration(1, 2) {
override fun migrate(db: SupportSQLiteDatabase) {
db.execSQL("ALTER TABLE keywords ADD COLUMN matchMode TEXT NOT NULL DEFAULT 'CONTAINS'")
}
}
/**
* Migration 2 -> 3: 인덱스 추가
*/
val MIGRATION_2_3 = object : Migration(2, 3) {
override fun migrate(db: SupportSQLiteDatabase) {
db.execSQL("CREATE INDEX IF NOT EXISTS index_hot_deals_siteName_boardName ON hot_deals(siteName, boardName)")
db.execSQL("CREATE INDEX IF NOT EXISTS index_hot_deals_createdAt ON hot_deals(createdAt)")
}
}
}
}
@@ -0,0 +1,19 @@
package com.hotdeal.alarm.data.local.db
import androidx.room.TypeConverter
/**
* Room Type Converters
*/
class Converters {
@TypeConverter
fun fromStringList(value: String?): List<String> {
return value?.split(",")?.map { it.trim() } ?: emptyList()
}
@TypeConverter
fun toStringList(list: List<String>?): String? {
return list?.joinToString(",")
}
}
@@ -0,0 +1,23 @@
package com.hotdeal.alarm.data.local.db.dao
import androidx.room.Dao
import androidx.room.Insert
import androidx.room.OnConflictStrategy
import androidx.room.Query
import com.hotdeal.alarm.data.local.db.entity.DealEntity
import kotlinx.coroutines.flow.Flow
@Dao
interface DealDao {
@Query("SELECT * FROM deals ORDER BY timestamp DESC")
fun getAllDeals(): Flow<List<DealEntity>>
@Insert(onConflict = OnConflictStrategy.REPLACE)
suspend fun insertDeals(deals: List<DealEntity>)
@Query("DELETE FROM deals WHERE timestamp < :threshold")
suspend fun deleteOldDeals(threshold: Long)
@Query("UPDATE deals SET isRead = 1 WHERE id = :id")
suspend fun markAsRead(id: String)
}
@@ -0,0 +1,102 @@
package com.hotdeal.alarm.data.local.db.dao
import androidx.room.*
import com.hotdeal.alarm.data.local.db.entity.HotDealEntity
import kotlinx.coroutines.flow.Flow
/**
* 핫딜 DAO
*/
@Dao
interface HotDealDao {
/**
* 모든 핫딜 조회 (Flow)
*/
@Query("SELECT * FROM hot_deals ORDER BY createdAt DESC")
fun observeAllDeals(): Flow<List<HotDealEntity>>
/**
* 특정 사이트의 핫딜 조회
*/
@Query("SELECT * FROM hot_deals WHERE siteName = :siteName ORDER BY createdAt DESC")
fun observeDealsBySite(siteName: String): Flow<List<HotDealEntity>>
/**
* 특정 사이트/게시판의 핫딜 조회
*/
@Query("SELECT * FROM hot_deals WHERE siteName = :siteName AND boardName = :boardName ORDER BY createdAt DESC")
fun observeDealsBySiteAndBoard(siteName: String, boardName: String): Flow<List<HotDealEntity>>
/**
* 최근 핫딜 조회
*/
@Query("SELECT * FROM hot_deals WHERE createdAt > :since ORDER BY createdAt DESC")
suspend fun getRecentDeals(since: Long): List<HotDealEntity>
/**
* 알림 미발송 핫딜 조회
*/
@Query("SELECT * FROM hot_deals WHERE isNotified = 0 ORDER BY createdAt DESC")
suspend fun getUnnotifiedDeals(): List<HotDealEntity>
/**
* ID로 핫딜 조회
*/
@Query("SELECT * FROM hot_deals WHERE id = :id")
suspend fun getDealById(id: String): HotDealEntity?
/**
* URL로 핫딜 조회 (중복 체크용)
*/
@Query("SELECT * FROM hot_deals WHERE url = :url LIMIT 1")
suspend fun getDealByUrl(url: String): HotDealEntity?
/**
* 핫딜 일괄 저장 (중복 무시)
*/
@Insert(onConflict = OnConflictStrategy.IGNORE)
suspend fun insertDeals(deals: List<HotDealEntity>)
/**
* 단일 핫딜 저장
*/
@Insert(onConflict = OnConflictStrategy.REPLACE)
suspend fun insertDeal(deal: HotDealEntity)
/**
* 알림 발송 상태 업데이트
*/
@Query("UPDATE hot_deals SET isNotified = 1 WHERE id IN (:ids)")
suspend fun markAsNotified(ids: List<String>)
/**
* 키워드 매칭 상태 업데이트
*/
@Query("UPDATE hot_deals SET isKeywordMatch = 1 WHERE id IN (:ids)")
suspend fun markAsKeywordMatch(ids: List<String>)
/**
* 오래된 핫딜 삭제
*/
@Query("DELETE FROM hot_deals WHERE createdAt < :threshold")
suspend fun deleteOldDeals(threshold: Long)
/**
* 전체 핫딜 삭제
*/
@Query("DELETE FROM hot_deals")
suspend fun deleteAllDeals()
/**
* 핫딜 개수 조회
*/
@Query("SELECT COUNT(*) FROM hot_deals")
suspend fun getDealCount(): Int
/**
* 제목으로 검색
*/
@Query("SELECT * FROM hot_deals WHERE title LIKE '%' || :query || '%' ORDER BY createdAt DESC")
fun searchDeals(query: String): Flow<List<HotDealEntity>>
}
@@ -0,0 +1,84 @@
package com.hotdeal.alarm.data.local.db.dao
import androidx.room.*
import com.hotdeal.alarm.data.local.db.entity.KeywordEntity
import kotlinx.coroutines.flow.Flow
/**
* 키워드 DAO
*/
@Dao
interface KeywordDao {
/**
* 모든 키워드 조회 (Flow)
*/
@Query("SELECT * FROM keywords ORDER BY createdAt DESC")
fun observeAllKeywords(): Flow<List<KeywordEntity>>
/**
* 모든 키워드 조회
*/
@Query("SELECT * FROM keywords ORDER BY createdAt DESC")
suspend fun getAllKeywords(): List<KeywordEntity>
/**
* 활성화된 키워드 조회
*/
@Query("SELECT * FROM keywords WHERE isEnabled = 1 ORDER BY createdAt DESC")
suspend fun getEnabledKeywords(): List<KeywordEntity>
/**
* ID로 키워드 조회
*/
@Query("SELECT * FROM keywords WHERE id = :id")
suspend fun getKeywordById(id: Long): KeywordEntity?
/**
* 키워드 저장
*/
@Insert(onConflict = OnConflictStrategy.REPLACE)
suspend fun insertKeyword(keyword: KeywordEntity): Long
/**
* 키워드 일괄 저장
*/
@Insert(onConflict = OnConflictStrategy.REPLACE)
suspend fun insertKeywords(keywords: List<KeywordEntity>)
/**
* 키워드 업데이트
*/
@Update
suspend fun updateKeyword(keyword: KeywordEntity)
/**
* 활성화 상태 업데이트
*/
@Query("UPDATE keywords SET isEnabled = :enabled WHERE id = :id")
suspend fun updateEnabled(id: Long, enabled: Boolean)
/**
* 키워드 삭제
*/
@Delete
suspend fun deleteKeyword(keyword: KeywordEntity)
/**
* ID로 키워드 삭제
*/
@Query("DELETE FROM keywords WHERE id = :id")
suspend fun deleteKeywordById(id: Long)
/**
* 전체 키워드 삭제
*/
@Query("DELETE FROM keywords")
suspend fun deleteAllKeywords()
/**
* 키워드 개수 조회
*/
@Query("SELECT COUNT(*) FROM keywords")
suspend fun getKeywordCount(): Int
}
@@ -0,0 +1,72 @@
package com.hotdeal.alarm.data.local.db.dao
import androidx.room.*
import com.hotdeal.alarm.data.local.db.entity.SiteConfigEntity
import kotlinx.coroutines.flow.Flow
/**
* 사이트 설정 DAO
*/
@Dao
interface SiteConfigDao {
/**
* 모든 설정 조회
*/
@Query("SELECT * FROM site_configs")
suspend fun getAllConfigs(): List<SiteConfigEntity>
/**
* 모든 설정 조회 (Flow)
*/
@Query("SELECT * FROM site_configs")
fun observeAllConfigs(): Flow<List<SiteConfigEntity>>
/**
* 활성화된 설정 조회
*/
@Query("SELECT * FROM site_configs WHERE isEnabled = 1")
suspend fun getEnabledConfigs(): List<SiteConfigEntity>
/**
* 특정 사이트의 설정 조회
*/
@Query("SELECT * FROM site_configs WHERE siteName = :siteName")
suspend fun getConfigsBySite(siteName: String): List<SiteConfigEntity>
/**
* 특정 설정 조회
*/
@Query("SELECT * FROM site_configs WHERE siteBoardKey = :key")
suspend fun getConfigByKey(key: String): SiteConfigEntity?
/**
* 설정 저장
*/
@Insert(onConflict = OnConflictStrategy.REPLACE)
suspend fun insertConfig(config: SiteConfigEntity)
/**
* 설정 일괄 저장
*/
@Insert(onConflict = OnConflictStrategy.REPLACE)
suspend fun insertConfigs(configs: List<SiteConfigEntity>)
/**
* 활성화 상태 업데이트
*/
@Query("UPDATE site_configs SET isEnabled = :enabled WHERE siteBoardKey = :key")
suspend fun updateEnabled(key: String, enabled: Boolean)
/**
* 마지막 스크래핑 시간 업데이트
*/
@Query("UPDATE site_configs SET lastScrapedAt = :timestamp WHERE siteBoardKey = :key")
suspend fun updateLastScrapedAt(key: String, timestamp: Long)
/**
* 설정 삭제
*/
@Delete
suspend fun deleteConfig(config: SiteConfigEntity)
}
@@ -0,0 +1,15 @@
package com.hotdeal.alarm.data.local.db.entity
import androidx.room.Entity
import androidx.room.PrimaryKey
@Entity(tableName = "deals")
data class DealEntity(
@PrimaryKey val id: String,
val title: String,
val url: String,
val price: String?,
val source: String,
val timestamp: Long,
val isRead: Boolean = false
)
@@ -0,0 +1,65 @@
package com.hotdeal.alarm.data.local.db.entity
import androidx.room.Entity
import androidx.room.Index
import androidx.room.PrimaryKey
import com.hotdeal.alarm.domain.model.HotDeal
/**
* 핫딜 Entity
*/
@Entity(
tableName = "hot_deals",
indices = [
Index(value = ["siteName", "boardName"]),
Index(value = ["createdAt"])
]
)
data class HotDealEntity(
@PrimaryKey
val id: String,
val siteName: String,
val boardName: String,
val title: String,
val url: String,
val mallUrl: String?,
val createdAt: Long,
val isNotified: Boolean = false,
val isKeywordMatch: Boolean = false
) {
/**
* Domain 모델로 변환
*/
fun toDomain(): HotDeal {
return HotDeal(
id = id,
siteName = siteName,
boardName = boardName,
title = title,
url = url,
mallUrl = mallUrl,
createdAt = createdAt,
isNotified = isNotified,
isKeywordMatch = isKeywordMatch
)
}
companion object {
/**
* Domain 모델에서 Entity 생성
*/
fun fromDomain(domain: HotDeal): HotDealEntity {
return HotDealEntity(
id = domain.id,
siteName = domain.siteName,
boardName = domain.boardName,
title = domain.title,
url = domain.url,
mallUrl = domain.mallUrl,
createdAt = domain.createdAt,
isNotified = domain.isNotified,
isKeywordMatch = domain.isKeywordMatch
)
}
}
}
@@ -0,0 +1,45 @@
package com.hotdeal.alarm.data.local.db.entity
import androidx.room.Entity
import androidx.room.PrimaryKey
import com.hotdeal.alarm.domain.model.Keyword
import com.hotdeal.alarm.domain.model.MatchMode
/**
* 키워드 Entity
*/
@Entity(tableName = "keywords")
data class KeywordEntity(
@PrimaryKey(autoGenerate = true)
val id: Long = 0,
val keyword: String,
val isEnabled: Boolean = true,
val matchMode: String = MatchMode.CONTAINS.name,
val createdAt: Long = System.currentTimeMillis()
) {
fun toDomain(): Keyword {
return Keyword(
id = id,
keyword = keyword,
isEnabled = isEnabled,
matchMode = try {
MatchMode.valueOf(matchMode)
} catch (e: IllegalArgumentException) {
MatchMode.CONTAINS
},
createdAt = createdAt
)
}
companion object {
fun fromDomain(domain: Keyword): KeywordEntity {
return KeywordEntity(
id = domain.id,
keyword = domain.keyword,
isEnabled = domain.isEnabled,
matchMode = domain.matchMode.name,
createdAt = domain.createdAt
)
}
}
}
@@ -0,0 +1,43 @@
package com.hotdeal.alarm.data.local.db.entity
import androidx.room.Entity
import androidx.room.PrimaryKey
import com.hotdeal.alarm.domain.model.SiteConfig
/**
* 사이트 설정 Entity
*/
@Entity(tableName = "site_configs")
data class SiteConfigEntity(
@PrimaryKey
val siteBoardKey: String,
val siteName: String,
val boardName: String,
val displayName: String,
val isEnabled: Boolean = false,
val lastScrapedAt: Long? = null
) {
fun toDomain(): SiteConfig {
return SiteConfig(
siteBoardKey = siteBoardKey,
siteName = siteName,
boardName = boardName,
displayName = displayName,
isEnabled = isEnabled,
lastScrapedAt = lastScrapedAt
)
}
companion object {
fun fromDomain(domain: SiteConfig): SiteConfigEntity {
return SiteConfigEntity(
siteBoardKey = domain.siteBoardKey,
siteName = domain.siteName,
boardName = domain.boardName,
displayName = domain.displayName,
isEnabled = domain.isEnabled,
lastScrapedAt = domain.lastScrapedAt
)
}
}
}
@@ -0,0 +1,47 @@
package com.hotdeal.alarm.data.local.preferences
import android.content.Context
import androidx.datastore.core.DataStore
import androidx.datastore.preferences.core.Preferences
import androidx.datastore.preferences.core.booleanPreferencesKey
import androidx.datastore.preferences.core.edit
import androidx.datastore.preferences.core.stringPreferencesKey
import androidx.datastore.preferences.preferencesDataStore
import kotlinx.coroutines.flow.Flow
import kotlinx.coroutines.flow.map
class ThemePreferences(private val context: Context) {
private val Context.dataStore: DataStore<Preferences> by preferencesDataStore(name = "theme_preferences")
companion object {
private val THEME_MODE_KEY = stringPreferencesKey("theme_mode")
private val DYNAMIC_COLORS_KEY = booleanPreferencesKey("dynamic_colors")
const val THEME_LIGHT = "light"
const val THEME_DARK = "dark"
const val THEME_SYSTEM = "system"
}
val themeMode: Flow<String> = context.dataStore.data
.map { preferences ->
preferences[THEME_MODE_KEY] ?: THEME_SYSTEM
}
val dynamicColors: Flow<Boolean> = context.dataStore.data
.map { preferences ->
preferences[DYNAMIC_COLORS_KEY] ?: true
}
suspend fun setThemeMode(mode: String) {
context.dataStore.edit { preferences ->
preferences[THEME_MODE_KEY] = mode
}
}
suspend fun setDynamicColors(enabled: Boolean) {
context.dataStore.edit { preferences ->
preferences[DYNAMIC_COLORS_KEY] = enabled
}
}
}
@@ -0,0 +1,64 @@
package com.hotdeal.alarm.data.remote.cloudflare
import android.content.Context
import android.content.SharedPreferences
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.withContext
/**
* Cloudflare 우회 쿠키 관리자
*/
class BypassCookieManager(context: Context) {
private val prefs: SharedPreferences = context.getSharedPreferences(
"cloudflare_cookies",
Context.MODE_PRIVATE
)
/**
* 쿠키 저장
*/
suspend fun saveCookies(domain: String, cookies: Map<String, String>) = withContext(Dispatchers.IO) {
prefs.edit().apply {
cookies.forEach { (key, value) ->
putString("${domain}_$key", value)
}
apply()
}
}
/**
* 쿠키 로드
*/
suspend fun loadCookies(domain: String): Map<String, String> = withContext(Dispatchers.IO) {
val cookies = mutableMapOf<String, String>()
prefs.all.forEach { (key, value) ->
if (key.startsWith("${domain}_")) {
val cookieKey = key.removePrefix("${domain}_")
cookies[cookieKey] = value as String
}
}
cookies
}
/**
* 쿠키 삭제
*/
suspend fun clearCookies(domain: String) = withContext(Dispatchers.IO) {
prefs.edit().apply {
prefs.all.keys.forEach { key ->
if (key.startsWith("${domain}_")) {
remove(key)
}
}
apply()
}
}
/**
* 쿠키를 헤더 문자열로 변환
*/
fun toCookieHeader(cookies: Map<String, String>): String {
return cookies.entries.joinToString("; ") { "${it.key}=${it.value}" }
}
}
@@ -0,0 +1,83 @@
package com.hotdeal.alarm.data.remote.cloudflare
import android.content.Context
import android.webkit.WebView
import android.webkit.WebViewClient
import kotlinx.coroutines.suspendCancellableCoroutine
import kotlin.coroutines.resume
import kotlin.coroutines.resumeWithException
/**
* Cloudflare 우회 헬퍼
*/
class CloudflareBypass(private val context: Context) {
data class BypassResult(
val html: String,
val cookies: Map<String, String>
)
/**
* Cloudflare 챌린지 우회
*/
suspend fun bypass(url: String): BypassResult = suspendCancellableCoroutine { continuation ->
val webView = WebView(context)
webView.settings.javaScriptEnabled = true
webView.settings.domStorageEnabled = true
webView.settings.loadWithOverviewMode = true
var challengeCompleted = false
webView.webViewClient = object : WebViewClient() {
override fun onPageFinished(view: WebView?, url: String?) {
super.onPageFinished(view, url)
// Cloudflare 챌린지 완료 확인
view?.evaluateJavascript(
"(function() { return document.body !== null && !document.body.innerHTML.includes('cloudflare'); })();"
) { result ->
if (result == "true" && !challengeCompleted) {
challengeCompleted = true
// HTML 추출
view?.evaluateJavascript(
"(function() { return document.documentElement.outerHTML; })();"
) { html ->
// 쿠키 추출
val cookieManager = android.webkit.CookieManager.getInstance()
val cookies = cookieManager.getCookie(url)
val cookieMap = parseCookies(cookies)
continuation.resume(
BypassResult(
html = html.trim('"').replace("\\u003C", "<").replace("\\u003E", ">"),
cookies = cookieMap
)
)
}
}
}
}
}
webView.loadUrl(url)
continuation.invokeOnCancellation {
webView.stopLoading()
webView.destroy()
}
}
private fun parseCookies(cookieString: String?): Map<String, String> {
if (cookieString.isNullOrEmpty()) return emptyMap()
return cookieString.split(";")
.map { it.trim() }
.filter { it.contains("=") }
.associate {
val parts = it.split("=", limit = 2)
parts[0] to parts.getOrElse(1) { "" }
}
}
}
@@ -0,0 +1,31 @@
package com.hotdeal.alarm.data.remote.interceptor
import okhttp3.Interceptor
import okhttp3.Response
import java.util.concurrent.atomic.AtomicLong
/**
* Rate Limiting 인터셉터
*
* 동시성 안전을 위해 AtomicLong 사용
*/
class RateLimitInterceptor(
private val minIntervalMillis: Long = 3000L
) : Interceptor {
private val lastRequestTime = AtomicLong(0L)
override fun intercept(chain: Interceptor.Chain): Response {
val currentTime = System.currentTimeMillis()
val lastTime = lastRequestTime.get()
val timeSinceLastRequest = currentTime - lastTime
if (timeSinceLastRequest < minIntervalMillis) {
val sleepTime = minIntervalMillis - timeSinceLastRequest
Thread.sleep(sleepTime)
}
lastRequestTime.set(System.currentTimeMillis())
return chain.proceed(chain.request())
}
}
@@ -0,0 +1,48 @@
package com.hotdeal.alarm.data.remote.interceptor
import okhttp3.Interceptor
import okhttp3.Response
import java.io.IOException
/**
* 재시도 인터셉터 (지수 백오프)
*/
class RetryInterceptor(
private val maxRetries: Int = 3,
private val initialDelayMs: Long = 1000L
) : Interceptor {
override fun intercept(chain: Interceptor.Chain): Response {
val request = chain.request()
var lastException: IOException? = null
var response: Response? = null
for (attempt in 0..maxRetries) {
try {
response?.close()
response = chain.proceed(request)
if (response.isSuccessful) {
return response
}
// 서버 에러 또는 rate limiting인 경우 재시도
if (response.code in listOf(429, 500, 502, 503, 504)) {
response.close()
val delay = initialDelayMs * (1 shl attempt) // 지수 백오프
Thread.sleep(delay)
continue
}
return response
} catch (e: IOException) {
lastException = e
response?.close()
val delay = initialDelayMs * (1 shl attempt)
Thread.sleep(delay)
}
}
throw lastException ?: IOException("Max retries exceeded")
}
}
@@ -0,0 +1,39 @@
package com.hotdeal.alarm.data.remote.interceptor
import okhttp3.Interceptor
import okhttp3.Response
import kotlin.random.Random
/**
* User-Agent 회전 인터셉터
*/
class UserAgentInterceptor : Interceptor {
private val userAgents = listOf(
// Android Chrome
"Mozilla/5.0 (Linux; Android 14; SM-G991B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Mobile Safari/537.36",
"Mozilla/5.0 (Linux; Android 13; Pixel 7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Mobile Safari/537.36",
"Mozilla/5.0 (Linux; Android 14; SM-S918B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Mobile Safari/537.36",
// Desktop Chrome (일부 사이트는 데스크톱 선호)
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
)
override fun intercept(chain: Interceptor.Chain): Response {
val request = chain.request().newBuilder()
.header("User-Agent", userAgents.random())
.header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8")
.header("Accept-Language", "ko-KR,ko;q=0.9,en-US;q=0.8,en;q=0.7")
.header("Accept-Encoding", "gzip, deflate, br")
.header("Connection", "keep-alive")
.header("Upgrade-Insecure-Requests", "1")
.header("Sec-Fetch-Dest", "document")
.header("Sec-Fetch-Mode", "navigate")
.header("Sec-Fetch-Site", "none")
.header("Sec-Fetch-User", "?1")
.header("Cache-Control", "max-age=0")
.build()
return chain.proceed(request)
}
}
@@ -0,0 +1,103 @@
package com.hotdeal.alarm.data.remote.scraper
import android.util.Log
import com.hotdeal.alarm.domain.model.HotDeal
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.withContext
import okhttp3.OkHttpClient
import okhttp3.Request
import org.jsoup.Jsoup
/**
* 스크래퍼 기본 클래스
*/
abstract class BaseScraper(
protected val client: OkHttpClient
) {
/**
* 사이트 이름
*/
abstract val siteName: String
/**
* 기본 URL
*/
abstract val baseUrl: String
/**
* 게시판 URL 생성
*/
abstract fun getBoardUrl(board: String): String
/**
* 스크래핑 수행
*/
abstract suspend fun scrape(board: String): Result<List<HotDeal>>
/**
* HTML 인코딩 (기본값: UTF-8)
*/
protected open val charset: String = "UTF-8"
/**
* HTML 가져오기
*/
protected suspend fun fetchHtml(url: String): String? = withContext(Dispatchers.IO) {
try {
Log.d("Scraper", "[$siteName] 요청: $url")
val request = Request.Builder()
.url(url)
.header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
.header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8")
.header("Accept-Language", "ko-KR,ko;q=0.9,en-US;q=0.8,en;q=0.7")
.header("Referer", baseUrl)
.header("Connection", "keep-alive")
.header("Upgrade-Insecure-Requests", "1")
.build()
client.newCall(request).execute().use { response ->
Log.d("Scraper", "[$siteName] 응답 코드: ${response.code}")
if (!response.isSuccessful) {
Log.e("Scraper", "[$siteName] 요청 실패: ${response.code}")
return@withContext null
}
// 문자열로 직접 읽기 (OkHttp가 자동으로 인코딩 처리)
val html = response.body?.string()
if (html == null) {
Log.e("Scraper", "[$siteName] 응답 바디가 null입니다")
return@withContext null
}
Log.d("Scraper", "[$siteName] HTML 길이: ${html.length}")
// 디버깅: HTML 일부 출력
if (html.length > 0) {
Log.d("Scraper", "[$siteName] HTML 샘플: ${html.take(300)}")
}
html
}
} catch (e: Exception) {
Log.e("Scraper", "[$siteName] 요청 예외: ${e.message}")
null
}
}
/**
* HTML 파싱
*/
protected fun parseHtml(html: String, baseUrl: String): org.jsoup.nodes.Document {
return Jsoup.parse(html, baseUrl)
}
/**
* 상대 URL을 절대 URL로 변환
*/
protected fun resolveUrl(baseUrl: String, relativeUrl: String): String {
if (relativeUrl.startsWith("http")) return relativeUrl
if (relativeUrl.startsWith("//")) return "https:$relativeUrl"
if (relativeUrl.startsWith("/")) return baseUrl.trimEnd('/') + relativeUrl
return baseUrl.trimEnd('/') + "/" + relativeUrl
}
}
@@ -0,0 +1,105 @@
package com.hotdeal.alarm.data.remote.scraper
import android.util.Log
import com.hotdeal.alarm.domain.model.HotDeal
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.delay
import kotlinx.coroutines.withContext
import okhttp3.OkHttpClient
import org.jsoup.Jsoup
import org.jsoup.nodes.Document
import org.jsoup.select.Elements
import kotlin.random.Random
/**
* 클리앙 스크래퍼
*/
class ClienScraper(client: OkHttpClient) : BaseScraper(client) {
override val siteName: String = "clien"
override val baseUrl: String = "https://www.clien.net"
override fun getBoardUrl(board: String): String {
return when (board) {
"allsell" -> "$baseUrl/service/group/allsell"
else -> "$baseUrl/service/board/$board"
}
}
override suspend fun scrape(board: String): Result<List<HotDeal>> = withContext(Dispatchers.IO) {
try {
val url = getBoardUrl(board)
Log.d("Clien", "스크래핑 시작: $url")
// 요청 간격 랜덤화 (2~4초)
val delayTime = Random.nextLong(2000, 4000)
Log.d("Clien", "요청 대기: ${delayTime}ms")
delay(delayTime)
val userAgent = getRandomUserAgent()
val doc: Document = Jsoup.connect(url)
.userAgent(userAgent)
.header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
.header("Accept-Language", "ko-KR,ko;q=0.9,en-US;q=0.8,en;q=0.7")
.header("Referer", "https://www.clien.net/")
.timeout(30000)
.followRedirects(true)
.get()
Log.d("Clien", "문서 파싱 성공, 길이: ${doc.html().length}")
val deals = mutableListOf<HotDeal>()
val elements: Elements = doc.select("div.list_item.symph_row")
Log.d("Clien", "찾은 요소: ${elements.size}")
var count = 0
elements.forEach { item ->
if (count >= 20) return@forEach
try {
val titleElement = item.selectFirst("a.list_subject, span.list_subject a")
val title = titleElement?.text()?.trim() ?: return@forEach
if (title.isEmpty()) return@forEach
val href = titleElement.attr("href")
val dealUrl = resolveUrl(baseUrl, href)
val postId = href.substringAfterLast("/").substringBefore("?").ifEmpty {
href.substringAfterLast("/").ifEmpty { return@forEach }
}
val deal = HotDeal(
id = HotDeal.generateId(siteName, postId),
siteName = siteName,
boardName = board,
title = title,
url = dealUrl,
createdAt = System.currentTimeMillis()
)
deals.add(deal)
count++
Log.d("Clien", "[$count] $title")
} catch (e: Exception) {
Log.e("Clien", "파싱 에러: ${e.message}")
}
}
Log.d("Clien", "파싱 완료: ${deals.size}")
Result.success(deals)
} catch (e: Exception) {
Log.e("Clien", "스크래핑 실패: ${e.message}", e)
Result.failure(e)
}
}
private fun getRandomUserAgent(): String {
val userAgents = listOf(
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:120.0) Gecko/20100101 Firefox/120.0"
)
return userAgents.random()
}
}
@@ -0,0 +1,112 @@
package com.hotdeal.alarm.data.remote.scraper
import android.util.Log
import com.hotdeal.alarm.domain.model.HotDeal
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.delay
import kotlinx.coroutines.withContext
import okhttp3.OkHttpClient
import org.jsoup.Jsoup
import org.jsoup.nodes.Document
import org.jsoup.select.Elements
import kotlin.random.Random
/**
* 쿨엔조이 스크래퍼
*/
class CoolenjoyScraper(client: OkHttpClient) : BaseScraper(client) {
override val siteName: String = "coolenjoy"
override val baseUrl: String = "https://coolenjoy.net"
override fun getBoardUrl(board: String): String {
return "$baseUrl/bbs/$board"
}
override suspend fun scrape(board: String): Result<List<HotDeal>> = withContext(Dispatchers.IO) {
try {
val url = getBoardUrl(board)
Log.d("Coolenjoy", "스크래핑 시작: $url")
// 요청 간격 랜덤화 (2~4초)
val delayTime = Random.nextLong(2000, 4000)
Log.d("Coolenjoy", "요청 대기: ${delayTime}ms")
delay(delayTime)
val userAgent = getRandomUserAgent()
val doc: Document = Jsoup.connect(url)
.userAgent(userAgent)
.header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
.header("Accept-Language", "ko-KR,ko;q=0.9,en-US;q=0.8,en;q=0.7")
.header("Referer", "https://coolenjoy.net/")
.timeout(30000)
.followRedirects(true)
.get()
Log.d("Coolenjoy", "문서 파싱 성공, 길이: ${doc.html().length}")
val deals = mutableListOf<HotDeal>()
val elements: Elements = doc.select("a.na-subject")
Log.d("Coolenjoy", "찾은 요소: ${elements.size}")
var count = 0
elements.forEach { element ->
if (count >= 20) return@forEach
try {
val title = element.text().trim()
if (title.isEmpty()) return@forEach
val href = element.attr("href")
val dealUrl = resolveUrl(baseUrl, href)
val postId = extractPostId(href)
if (postId.isEmpty()) return@forEach
val deal = HotDeal(
id = HotDeal.generateId(siteName, postId),
siteName = siteName,
boardName = board,
title = title,
url = dealUrl,
createdAt = System.currentTimeMillis()
)
deals.add(deal)
count++
Log.d("Coolenjoy", "[$count] $title")
} catch (e: Exception) {
Log.e("Coolenjoy", "파싱 에러: ${e.message}")
}
}
Log.d("Coolenjoy", "파싱 완료: ${deals.size}")
Result.success(deals)
} catch (e: Exception) {
Log.e("Coolenjoy", "스크래핑 실패: ${e.message}", e)
Result.failure(e)
}
}
private fun extractPostId(href: String): String {
val fromPath = href.substringAfterLast("/").substringBefore("?")
if (fromPath.isNotEmpty() && fromPath.all { it.isDigit() }) {
return fromPath
}
val fromWrId = href.substringAfter("wr_id=").substringBefore("&")
if (fromWrId.isNotEmpty()) {
return fromWrId
}
return ""
}
private fun getRandomUserAgent(): String {
val userAgents = listOf(
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:120.0) Gecko/20100101 Firefox/120.0"
)
return userAgents.random()
}
}
@@ -0,0 +1,122 @@
package com.hotdeal.alarm.data.remote.scraper
import android.util.Log
import com.hotdeal.alarm.domain.model.HotDeal
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.delay
import kotlinx.coroutines.withContext
import okhttp3.OkHttpClient
import org.jsoup.Jsoup
import org.jsoup.nodes.Document
import org.jsoup.select.Elements
import kotlin.random.Random
/**
* 뽐뿌 스크래퍼
*/
class PpomppuScraper(client: OkHttpClient) : BaseScraper(client) {
override val siteName: String = "ppomppu"
override val baseUrl: String = "https://www.ppomppu.co.kr/zboard/"
override fun getBoardUrl(board: String): String {
return "${baseUrl}zboard.php?id=$board"
}
override suspend fun scrape(board: String): Result<List<HotDeal>> = withContext(Dispatchers.IO) {
try {
val url = getBoardUrl(board)
Log.d("Ppomppu", "스크래핑 시작: $url")
// 요청 간격 랜덤화 (2~4초) - 차단 방지
val delayTime = Random.nextLong(2000, 4000)
Log.d("Ppomppu", "요청 대기: ${delayTime}ms")
delay(delayTime)
// Jsoup으로 직접 연결 (User-Agent 회전)
val userAgent = getRandomUserAgent()
Log.d("Ppomppu", "User-Agent: $userAgent")
val doc: Document = Jsoup.connect(url)
.userAgent(userAgent)
.header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
.header("Accept-Language", "ko-KR,ko;q=0.9,en-US;q=0.8,en;q=0.7")
.header("Accept-Charset", "utf-8,ISO-8859-1")
.header("Referer", "https://www.ppomppu.co.kr/")
.timeout(30000)
.followRedirects(true)
.get()
Log.d("Ppomppu", "문서 파싱 성공, 길이: ${doc.html().length}")
val deals = mutableListOf<HotDeal>()
// 셀렉터로 요소 찾기
val elements: Elements = doc.select("a.baseList-title")
Log.d("Ppomppu", "찾은 요소: ${elements.size}")
// 최대 20개까지만 처리
var count = 0
elements.forEach { element ->
if (count >= 20) return@forEach
try {
val title = element.text().trim()
if (title.isEmpty()) return@forEach
val href = element.attr("href")
// 공지사항 제외
if (href.contains("regulation") || href.contains("notice")) return@forEach
val dealUrl = resolveUrl(baseUrl, href)
// postId 추출
val postId = extractPostId(href)
if (postId.isEmpty()) {
Log.w("Ppomppu", "postId 추출 실패: href=$href")
return@forEach
}
val deal = HotDeal(
id = HotDeal.generateId(siteName, postId),
siteName = siteName,
boardName = board,
title = title,
url = dealUrl,
createdAt = System.currentTimeMillis()
)
deals.add(deal)
count++
Log.d("Ppomppu", "[$count] $title")
} catch (e: Exception) {
Log.e("Ppomppu", "파싱 에러: ${e.message}")
}
}
Log.d("Ppomppu", "파싱 완료: ${deals.size}")
Result.success(deals)
} catch (e: Exception) {
Log.e("Ppomppu", "스크래핑 실패: ${e.message}", e)
Result.failure(e)
}
}
private fun extractPostId(href: String): String {
val afterNo = href.substringAfter("no=", "")
if (afterNo.isEmpty()) return ""
val postId = afterNo.takeWhile { it != '&' && it != '/' && it != '#' }
return if (postId.isNotEmpty() && postId.all { it.isDigit() }) postId else ""
}
private fun getRandomUserAgent(): String {
val userAgents = listOf(
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:120.0) Gecko/20100101 Firefox/120.0",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.1 Safari/605.1.15",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36 Edg/119.0.0.0"
)
return userAgents.random()
}
}
@@ -0,0 +1,113 @@
package com.hotdeal.alarm.data.remote.scraper
import android.util.Log
import com.hotdeal.alarm.domain.model.HotDeal
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.delay
import kotlinx.coroutines.withContext
import okhttp3.OkHttpClient
import org.jsoup.Jsoup
import org.jsoup.nodes.Document
import org.jsoup.select.Elements
import kotlin.random.Random
/**
* 루리웹 스크래퍼
*/
class RuriwebScraper(client: OkHttpClient) : BaseScraper(client) {
override val siteName: String = "ruriweb"
override val baseUrl: String = "https://bbs.ruliweb.com"
override fun getBoardUrl(board: String): String {
return "$baseUrl/market/board/$board"
}
override suspend fun scrape(board: String): Result<List<HotDeal>> = withContext(Dispatchers.IO) {
try {
val url = getBoardUrl(board)
Log.d("Ruriweb", "스크래핑 시작: $url")
// 요청 간격 랜덤화 (2~4초)
val delayTime = Random.nextLong(2000, 4000)
Log.d("Ruriweb", "요청 대기: ${delayTime}ms")
delay(delayTime)
val userAgent = getRandomUserAgent()
val doc: Document = Jsoup.connect(url)
.userAgent(userAgent)
.header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
.header("Accept-Language", "ko-KR,ko;q=0.9,en-US;q=0.8,en;q=0.7")
.header("Referer", "https://bbs.ruliweb.com/")
.timeout(30000)
.followRedirects(true)
.get()
Log.d("Ruriweb", "문서 파싱 성공, 길이: ${doc.html().length}")
val deals = mutableListOf<HotDeal>()
val elements: Elements = doc.select("td.subject a.subject_link.deco")
Log.d("Ruriweb", "찾은 요소: ${elements.size}")
var count = 0
elements.forEach { element ->
if (count >= 20) return@forEach
try {
val strongEl = element.selectFirst("strong")
val title = (strongEl?.text() ?: element.text()).trim()
if (title.isEmpty()) return@forEach
val href = element.attr("href")
val dealUrl = resolveUrl(baseUrl, href)
val postId = extractPostId(href)
if (postId.isEmpty()) return@forEach
val deal = HotDeal(
id = HotDeal.generateId(siteName, postId),
siteName = siteName,
boardName = board,
title = title,
url = dealUrl,
createdAt = System.currentTimeMillis()
)
deals.add(deal)
count++
Log.d("Ruriweb", "[$count] $title")
} catch (e: Exception) {
Log.e("Ruriweb", "파싱 에러: ${e.message}")
}
}
Log.d("Ruriweb", "파싱 완료: ${deals.size}")
Result.success(deals)
} catch (e: Exception) {
Log.e("Ruriweb", "스크래핑 실패: ${e.message}", e)
Result.failure(e)
}
}
private fun extractPostId(href: String): String {
val fromRead = href.substringAfter("/read/").substringBefore("?")
if (fromRead.isNotEmpty() && fromRead.all { it.isDigit() }) {
return fromRead
}
val fromPath = href.substringAfterLast("/").substringBefore("?")
if (fromPath.isNotEmpty() && fromPath.all { it.isDigit() }) {
return fromPath
}
return ""
}
private fun getRandomUserAgent(): String {
val userAgents = listOf(
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:120.0) Gecko/20100101 Firefox/120.0"
)
return userAgents.random()
}
}
@@ -0,0 +1,45 @@
package com.hotdeal.alarm.data.remote.scraper
import com.hotdeal.alarm.domain.model.SiteType
/**
* 스크래퍼 팩토리
*/
class ScraperFactory(
private val ppomppu: PpomppuScraper,
private val clien: ClienScraper,
private val ruriweb: RuriwebScraper,
private val coolenjoy: CoolenjoyScraper
) {
/**
* 사이트 타입에 따른 스크래퍼 반환
*/
fun getScraper(siteType: SiteType): BaseScraper {
return when (siteType) {
SiteType.PPOMPPU -> ppomppu
SiteType.CLIEN -> clien
SiteType.RURIWEB -> ruriweb
SiteType.COOLENJOY -> coolenjoy
}
}
/**
* 사이트 이름으로 스크래퍼 반환
*/
fun getScraper(siteName: String): BaseScraper? {
return when (siteName.lowercase()) {
"ppomppu" -> ppomppu
"clien" -> clien
"ruriweb" -> ruriweb
"coolenjoy" -> coolenjoy
else -> null
}
}
/**
* 모든 스크래퍼 반환
*/
fun getAllScrapers(): List<BaseScraper> {
return listOf(ppomppu, clien, ruriweb, coolenjoy)
}
}