Initial commit: HotDeal Alarm Android App
Features: - Multi-site hot deal scraping (Ppomppu, Clien, Ruriweb, Coolenjoy) - Site filter with color-coded badges - Board display names (e.g., ppomppu8 -> 알리뽐뿌) - Anti-bot protection with request delays and User-Agent rotation - Keyword matching and notifications - Material Design 3 UI
This commit is contained in:
@@ -0,0 +1,56 @@
|
||||
package com.hotdeal.alarm.data.local.db
|
||||
|
||||
import androidx.room.Database
|
||||
import androidx.room.RoomDatabase
|
||||
import androidx.room.TypeConverters
|
||||
import androidx.room.migration.Migration
|
||||
import androidx.sqlite.db.SupportSQLiteDatabase
|
||||
import com.hotdeal.alarm.data.local.db.dao.HotDealDao
|
||||
import com.hotdeal.alarm.data.local.db.dao.KeywordDao
|
||||
import com.hotdeal.alarm.data.local.db.dao.SiteConfigDao
|
||||
import com.hotdeal.alarm.data.local.db.entity.HotDealEntity
|
||||
import com.hotdeal.alarm.data.local.db.entity.KeywordEntity
|
||||
import com.hotdeal.alarm.data.local.db.entity.SiteConfigEntity
|
||||
|
||||
/**
|
||||
* 핫딜 알람 데이터베이스
|
||||
*/
|
||||
@Database(
|
||||
entities = [
|
||||
HotDealEntity::class,
|
||||
SiteConfigEntity::class,
|
||||
KeywordEntity::class
|
||||
],
|
||||
version = 3,
|
||||
exportSchema = false
|
||||
)
|
||||
@TypeConverters(Converters::class)
|
||||
abstract class AppDatabase : RoomDatabase() {
|
||||
|
||||
abstract fun hotDealDao(): HotDealDao
|
||||
abstract fun siteConfigDao(): SiteConfigDao
|
||||
abstract fun keywordDao(): KeywordDao
|
||||
|
||||
companion object {
|
||||
const val DATABASE_NAME = "hotdeal_alarm_db"
|
||||
|
||||
/**
|
||||
* Migration 1 -> 2: 키워드 매칭 모드 추가
|
||||
*/
|
||||
val MIGRATION_1_2 = object : Migration(1, 2) {
|
||||
override fun migrate(db: SupportSQLiteDatabase) {
|
||||
db.execSQL("ALTER TABLE keywords ADD COLUMN matchMode TEXT NOT NULL DEFAULT 'CONTAINS'")
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Migration 2 -> 3: 인덱스 추가
|
||||
*/
|
||||
val MIGRATION_2_3 = object : Migration(2, 3) {
|
||||
override fun migrate(db: SupportSQLiteDatabase) {
|
||||
db.execSQL("CREATE INDEX IF NOT EXISTS index_hot_deals_siteName_boardName ON hot_deals(siteName, boardName)")
|
||||
db.execSQL("CREATE INDEX IF NOT EXISTS index_hot_deals_createdAt ON hot_deals(createdAt)")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
package com.hotdeal.alarm.data.local.db
|
||||
|
||||
import androidx.room.TypeConverter
|
||||
|
||||
/**
|
||||
* Room Type Converters
|
||||
*/
|
||||
class Converters {
|
||||
|
||||
@TypeConverter
|
||||
fun fromStringList(value: String?): List<String> {
|
||||
return value?.split(",")?.map { it.trim() } ?: emptyList()
|
||||
}
|
||||
|
||||
@TypeConverter
|
||||
fun toStringList(list: List<String>?): String? {
|
||||
return list?.joinToString(",")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
package com.hotdeal.alarm.data.local.db.dao
|
||||
|
||||
import androidx.room.Dao
|
||||
import androidx.room.Insert
|
||||
import androidx.room.OnConflictStrategy
|
||||
import androidx.room.Query
|
||||
import com.hotdeal.alarm.data.local.db.entity.DealEntity
|
||||
import kotlinx.coroutines.flow.Flow
|
||||
|
||||
@Dao
|
||||
interface DealDao {
|
||||
@Query("SELECT * FROM deals ORDER BY timestamp DESC")
|
||||
fun getAllDeals(): Flow<List<DealEntity>>
|
||||
|
||||
@Insert(onConflict = OnConflictStrategy.REPLACE)
|
||||
suspend fun insertDeals(deals: List<DealEntity>)
|
||||
|
||||
@Query("DELETE FROM deals WHERE timestamp < :threshold")
|
||||
suspend fun deleteOldDeals(threshold: Long)
|
||||
|
||||
@Query("UPDATE deals SET isRead = 1 WHERE id = :id")
|
||||
suspend fun markAsRead(id: String)
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
package com.hotdeal.alarm.data.local.db.dao
|
||||
|
||||
import androidx.room.*
|
||||
import com.hotdeal.alarm.data.local.db.entity.HotDealEntity
|
||||
import kotlinx.coroutines.flow.Flow
|
||||
|
||||
/**
|
||||
* 핫딜 DAO
|
||||
*/
|
||||
@Dao
|
||||
interface HotDealDao {
|
||||
|
||||
/**
|
||||
* 모든 핫딜 조회 (Flow)
|
||||
*/
|
||||
@Query("SELECT * FROM hot_deals ORDER BY createdAt DESC")
|
||||
fun observeAllDeals(): Flow<List<HotDealEntity>>
|
||||
|
||||
/**
|
||||
* 특정 사이트의 핫딜 조회
|
||||
*/
|
||||
@Query("SELECT * FROM hot_deals WHERE siteName = :siteName ORDER BY createdAt DESC")
|
||||
fun observeDealsBySite(siteName: String): Flow<List<HotDealEntity>>
|
||||
|
||||
/**
|
||||
* 특정 사이트/게시판의 핫딜 조회
|
||||
*/
|
||||
@Query("SELECT * FROM hot_deals WHERE siteName = :siteName AND boardName = :boardName ORDER BY createdAt DESC")
|
||||
fun observeDealsBySiteAndBoard(siteName: String, boardName: String): Flow<List<HotDealEntity>>
|
||||
|
||||
/**
|
||||
* 최근 핫딜 조회
|
||||
*/
|
||||
@Query("SELECT * FROM hot_deals WHERE createdAt > :since ORDER BY createdAt DESC")
|
||||
suspend fun getRecentDeals(since: Long): List<HotDealEntity>
|
||||
|
||||
/**
|
||||
* 알림 미발송 핫딜 조회
|
||||
*/
|
||||
@Query("SELECT * FROM hot_deals WHERE isNotified = 0 ORDER BY createdAt DESC")
|
||||
suspend fun getUnnotifiedDeals(): List<HotDealEntity>
|
||||
|
||||
/**
|
||||
* ID로 핫딜 조회
|
||||
*/
|
||||
@Query("SELECT * FROM hot_deals WHERE id = :id")
|
||||
suspend fun getDealById(id: String): HotDealEntity?
|
||||
|
||||
/**
|
||||
* URL로 핫딜 조회 (중복 체크용)
|
||||
*/
|
||||
@Query("SELECT * FROM hot_deals WHERE url = :url LIMIT 1")
|
||||
suspend fun getDealByUrl(url: String): HotDealEntity?
|
||||
|
||||
/**
|
||||
* 핫딜 일괄 저장 (중복 무시)
|
||||
*/
|
||||
@Insert(onConflict = OnConflictStrategy.IGNORE)
|
||||
suspend fun insertDeals(deals: List<HotDealEntity>)
|
||||
|
||||
/**
|
||||
* 단일 핫딜 저장
|
||||
*/
|
||||
@Insert(onConflict = OnConflictStrategy.REPLACE)
|
||||
suspend fun insertDeal(deal: HotDealEntity)
|
||||
|
||||
/**
|
||||
* 알림 발송 상태 업데이트
|
||||
*/
|
||||
@Query("UPDATE hot_deals SET isNotified = 1 WHERE id IN (:ids)")
|
||||
suspend fun markAsNotified(ids: List<String>)
|
||||
|
||||
/**
|
||||
* 키워드 매칭 상태 업데이트
|
||||
*/
|
||||
@Query("UPDATE hot_deals SET isKeywordMatch = 1 WHERE id IN (:ids)")
|
||||
suspend fun markAsKeywordMatch(ids: List<String>)
|
||||
|
||||
/**
|
||||
* 오래된 핫딜 삭제
|
||||
*/
|
||||
@Query("DELETE FROM hot_deals WHERE createdAt < :threshold")
|
||||
suspend fun deleteOldDeals(threshold: Long)
|
||||
|
||||
/**
|
||||
* 전체 핫딜 삭제
|
||||
*/
|
||||
@Query("DELETE FROM hot_deals")
|
||||
suspend fun deleteAllDeals()
|
||||
|
||||
/**
|
||||
* 핫딜 개수 조회
|
||||
*/
|
||||
@Query("SELECT COUNT(*) FROM hot_deals")
|
||||
suspend fun getDealCount(): Int
|
||||
|
||||
/**
|
||||
* 제목으로 검색
|
||||
*/
|
||||
@Query("SELECT * FROM hot_deals WHERE title LIKE '%' || :query || '%' ORDER BY createdAt DESC")
|
||||
fun searchDeals(query: String): Flow<List<HotDealEntity>>
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
package com.hotdeal.alarm.data.local.db.dao
|
||||
|
||||
import androidx.room.*
|
||||
import com.hotdeal.alarm.data.local.db.entity.KeywordEntity
|
||||
import kotlinx.coroutines.flow.Flow
|
||||
|
||||
/**
|
||||
* 키워드 DAO
|
||||
*/
|
||||
@Dao
|
||||
interface KeywordDao {
|
||||
|
||||
/**
|
||||
* 모든 키워드 조회 (Flow)
|
||||
*/
|
||||
@Query("SELECT * FROM keywords ORDER BY createdAt DESC")
|
||||
fun observeAllKeywords(): Flow<List<KeywordEntity>>
|
||||
|
||||
/**
|
||||
* 모든 키워드 조회
|
||||
*/
|
||||
@Query("SELECT * FROM keywords ORDER BY createdAt DESC")
|
||||
suspend fun getAllKeywords(): List<KeywordEntity>
|
||||
|
||||
/**
|
||||
* 활성화된 키워드 조회
|
||||
*/
|
||||
@Query("SELECT * FROM keywords WHERE isEnabled = 1 ORDER BY createdAt DESC")
|
||||
suspend fun getEnabledKeywords(): List<KeywordEntity>
|
||||
|
||||
/**
|
||||
* ID로 키워드 조회
|
||||
*/
|
||||
@Query("SELECT * FROM keywords WHERE id = :id")
|
||||
suspend fun getKeywordById(id: Long): KeywordEntity?
|
||||
|
||||
/**
|
||||
* 키워드 저장
|
||||
*/
|
||||
@Insert(onConflict = OnConflictStrategy.REPLACE)
|
||||
suspend fun insertKeyword(keyword: KeywordEntity): Long
|
||||
|
||||
/**
|
||||
* 키워드 일괄 저장
|
||||
*/
|
||||
@Insert(onConflict = OnConflictStrategy.REPLACE)
|
||||
suspend fun insertKeywords(keywords: List<KeywordEntity>)
|
||||
|
||||
/**
|
||||
* 키워드 업데이트
|
||||
*/
|
||||
@Update
|
||||
suspend fun updateKeyword(keyword: KeywordEntity)
|
||||
|
||||
/**
|
||||
* 활성화 상태 업데이트
|
||||
*/
|
||||
@Query("UPDATE keywords SET isEnabled = :enabled WHERE id = :id")
|
||||
suspend fun updateEnabled(id: Long, enabled: Boolean)
|
||||
|
||||
/**
|
||||
* 키워드 삭제
|
||||
*/
|
||||
@Delete
|
||||
suspend fun deleteKeyword(keyword: KeywordEntity)
|
||||
|
||||
/**
|
||||
* ID로 키워드 삭제
|
||||
*/
|
||||
@Query("DELETE FROM keywords WHERE id = :id")
|
||||
suspend fun deleteKeywordById(id: Long)
|
||||
|
||||
/**
|
||||
* 전체 키워드 삭제
|
||||
*/
|
||||
@Query("DELETE FROM keywords")
|
||||
suspend fun deleteAllKeywords()
|
||||
|
||||
/**
|
||||
* 키워드 개수 조회
|
||||
*/
|
||||
@Query("SELECT COUNT(*) FROM keywords")
|
||||
suspend fun getKeywordCount(): Int
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
package com.hotdeal.alarm.data.local.db.dao
|
||||
|
||||
import androidx.room.*
|
||||
import com.hotdeal.alarm.data.local.db.entity.SiteConfigEntity
|
||||
import kotlinx.coroutines.flow.Flow
|
||||
|
||||
/**
|
||||
* 사이트 설정 DAO
|
||||
*/
|
||||
@Dao
|
||||
interface SiteConfigDao {
|
||||
|
||||
/**
|
||||
* 모든 설정 조회
|
||||
*/
|
||||
@Query("SELECT * FROM site_configs")
|
||||
suspend fun getAllConfigs(): List<SiteConfigEntity>
|
||||
|
||||
/**
|
||||
* 모든 설정 조회 (Flow)
|
||||
*/
|
||||
@Query("SELECT * FROM site_configs")
|
||||
fun observeAllConfigs(): Flow<List<SiteConfigEntity>>
|
||||
|
||||
/**
|
||||
* 활성화된 설정 조회
|
||||
*/
|
||||
@Query("SELECT * FROM site_configs WHERE isEnabled = 1")
|
||||
suspend fun getEnabledConfigs(): List<SiteConfigEntity>
|
||||
|
||||
/**
|
||||
* 특정 사이트의 설정 조회
|
||||
*/
|
||||
@Query("SELECT * FROM site_configs WHERE siteName = :siteName")
|
||||
suspend fun getConfigsBySite(siteName: String): List<SiteConfigEntity>
|
||||
|
||||
/**
|
||||
* 특정 설정 조회
|
||||
*/
|
||||
@Query("SELECT * FROM site_configs WHERE siteBoardKey = :key")
|
||||
suspend fun getConfigByKey(key: String): SiteConfigEntity?
|
||||
|
||||
/**
|
||||
* 설정 저장
|
||||
*/
|
||||
@Insert(onConflict = OnConflictStrategy.REPLACE)
|
||||
suspend fun insertConfig(config: SiteConfigEntity)
|
||||
|
||||
/**
|
||||
* 설정 일괄 저장
|
||||
*/
|
||||
@Insert(onConflict = OnConflictStrategy.REPLACE)
|
||||
suspend fun insertConfigs(configs: List<SiteConfigEntity>)
|
||||
|
||||
/**
|
||||
* 활성화 상태 업데이트
|
||||
*/
|
||||
@Query("UPDATE site_configs SET isEnabled = :enabled WHERE siteBoardKey = :key")
|
||||
suspend fun updateEnabled(key: String, enabled: Boolean)
|
||||
|
||||
/**
|
||||
* 마지막 스크래핑 시간 업데이트
|
||||
*/
|
||||
@Query("UPDATE site_configs SET lastScrapedAt = :timestamp WHERE siteBoardKey = :key")
|
||||
suspend fun updateLastScrapedAt(key: String, timestamp: Long)
|
||||
|
||||
/**
|
||||
* 설정 삭제
|
||||
*/
|
||||
@Delete
|
||||
suspend fun deleteConfig(config: SiteConfigEntity)
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
package com.hotdeal.alarm.data.local.db.entity
|
||||
|
||||
import androidx.room.Entity
|
||||
import androidx.room.PrimaryKey
|
||||
|
||||
@Entity(tableName = "deals")
|
||||
data class DealEntity(
|
||||
@PrimaryKey val id: String,
|
||||
val title: String,
|
||||
val url: String,
|
||||
val price: String?,
|
||||
val source: String,
|
||||
val timestamp: Long,
|
||||
val isRead: Boolean = false
|
||||
)
|
||||
@@ -0,0 +1,65 @@
|
||||
package com.hotdeal.alarm.data.local.db.entity
|
||||
|
||||
import androidx.room.Entity
|
||||
import androidx.room.Index
|
||||
import androidx.room.PrimaryKey
|
||||
import com.hotdeal.alarm.domain.model.HotDeal
|
||||
|
||||
/**
|
||||
* 핫딜 Entity
|
||||
*/
|
||||
@Entity(
|
||||
tableName = "hot_deals",
|
||||
indices = [
|
||||
Index(value = ["siteName", "boardName"]),
|
||||
Index(value = ["createdAt"])
|
||||
]
|
||||
)
|
||||
data class HotDealEntity(
|
||||
@PrimaryKey
|
||||
val id: String,
|
||||
val siteName: String,
|
||||
val boardName: String,
|
||||
val title: String,
|
||||
val url: String,
|
||||
val mallUrl: String?,
|
||||
val createdAt: Long,
|
||||
val isNotified: Boolean = false,
|
||||
val isKeywordMatch: Boolean = false
|
||||
) {
|
||||
/**
|
||||
* Domain 모델로 변환
|
||||
*/
|
||||
fun toDomain(): HotDeal {
|
||||
return HotDeal(
|
||||
id = id,
|
||||
siteName = siteName,
|
||||
boardName = boardName,
|
||||
title = title,
|
||||
url = url,
|
||||
mallUrl = mallUrl,
|
||||
createdAt = createdAt,
|
||||
isNotified = isNotified,
|
||||
isKeywordMatch = isKeywordMatch
|
||||
)
|
||||
}
|
||||
|
||||
companion object {
|
||||
/**
|
||||
* Domain 모델에서 Entity 생성
|
||||
*/
|
||||
fun fromDomain(domain: HotDeal): HotDealEntity {
|
||||
return HotDealEntity(
|
||||
id = domain.id,
|
||||
siteName = domain.siteName,
|
||||
boardName = domain.boardName,
|
||||
title = domain.title,
|
||||
url = domain.url,
|
||||
mallUrl = domain.mallUrl,
|
||||
createdAt = domain.createdAt,
|
||||
isNotified = domain.isNotified,
|
||||
isKeywordMatch = domain.isKeywordMatch
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
package com.hotdeal.alarm.data.local.db.entity
|
||||
|
||||
import androidx.room.Entity
|
||||
import androidx.room.PrimaryKey
|
||||
import com.hotdeal.alarm.domain.model.Keyword
|
||||
import com.hotdeal.alarm.domain.model.MatchMode
|
||||
|
||||
/**
|
||||
* 키워드 Entity
|
||||
*/
|
||||
@Entity(tableName = "keywords")
|
||||
data class KeywordEntity(
|
||||
@PrimaryKey(autoGenerate = true)
|
||||
val id: Long = 0,
|
||||
val keyword: String,
|
||||
val isEnabled: Boolean = true,
|
||||
val matchMode: String = MatchMode.CONTAINS.name,
|
||||
val createdAt: Long = System.currentTimeMillis()
|
||||
) {
|
||||
fun toDomain(): Keyword {
|
||||
return Keyword(
|
||||
id = id,
|
||||
keyword = keyword,
|
||||
isEnabled = isEnabled,
|
||||
matchMode = try {
|
||||
MatchMode.valueOf(matchMode)
|
||||
} catch (e: IllegalArgumentException) {
|
||||
MatchMode.CONTAINS
|
||||
},
|
||||
createdAt = createdAt
|
||||
)
|
||||
}
|
||||
|
||||
companion object {
|
||||
fun fromDomain(domain: Keyword): KeywordEntity {
|
||||
return KeywordEntity(
|
||||
id = domain.id,
|
||||
keyword = domain.keyword,
|
||||
isEnabled = domain.isEnabled,
|
||||
matchMode = domain.matchMode.name,
|
||||
createdAt = domain.createdAt
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
package com.hotdeal.alarm.data.local.db.entity
|
||||
|
||||
import androidx.room.Entity
|
||||
import androidx.room.PrimaryKey
|
||||
import com.hotdeal.alarm.domain.model.SiteConfig
|
||||
|
||||
/**
|
||||
* 사이트 설정 Entity
|
||||
*/
|
||||
@Entity(tableName = "site_configs")
|
||||
data class SiteConfigEntity(
|
||||
@PrimaryKey
|
||||
val siteBoardKey: String,
|
||||
val siteName: String,
|
||||
val boardName: String,
|
||||
val displayName: String,
|
||||
val isEnabled: Boolean = false,
|
||||
val lastScrapedAt: Long? = null
|
||||
) {
|
||||
fun toDomain(): SiteConfig {
|
||||
return SiteConfig(
|
||||
siteBoardKey = siteBoardKey,
|
||||
siteName = siteName,
|
||||
boardName = boardName,
|
||||
displayName = displayName,
|
||||
isEnabled = isEnabled,
|
||||
lastScrapedAt = lastScrapedAt
|
||||
)
|
||||
}
|
||||
|
||||
companion object {
|
||||
fun fromDomain(domain: SiteConfig): SiteConfigEntity {
|
||||
return SiteConfigEntity(
|
||||
siteBoardKey = domain.siteBoardKey,
|
||||
siteName = domain.siteName,
|
||||
boardName = domain.boardName,
|
||||
displayName = domain.displayName,
|
||||
isEnabled = domain.isEnabled,
|
||||
lastScrapedAt = domain.lastScrapedAt
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
package com.hotdeal.alarm.data.local.preferences
|
||||
|
||||
import android.content.Context
|
||||
import androidx.datastore.core.DataStore
|
||||
import androidx.datastore.preferences.core.Preferences
|
||||
import androidx.datastore.preferences.core.booleanPreferencesKey
|
||||
import androidx.datastore.preferences.core.edit
|
||||
import androidx.datastore.preferences.core.stringPreferencesKey
|
||||
import androidx.datastore.preferences.preferencesDataStore
|
||||
import kotlinx.coroutines.flow.Flow
|
||||
import kotlinx.coroutines.flow.map
|
||||
|
||||
class ThemePreferences(private val context: Context) {
|
||||
|
||||
private val Context.dataStore: DataStore<Preferences> by preferencesDataStore(name = "theme_preferences")
|
||||
|
||||
companion object {
|
||||
private val THEME_MODE_KEY = stringPreferencesKey("theme_mode")
|
||||
private val DYNAMIC_COLORS_KEY = booleanPreferencesKey("dynamic_colors")
|
||||
|
||||
const val THEME_LIGHT = "light"
|
||||
const val THEME_DARK = "dark"
|
||||
const val THEME_SYSTEM = "system"
|
||||
}
|
||||
|
||||
val themeMode: Flow<String> = context.dataStore.data
|
||||
.map { preferences ->
|
||||
preferences[THEME_MODE_KEY] ?: THEME_SYSTEM
|
||||
}
|
||||
|
||||
val dynamicColors: Flow<Boolean> = context.dataStore.data
|
||||
.map { preferences ->
|
||||
preferences[DYNAMIC_COLORS_KEY] ?: true
|
||||
}
|
||||
|
||||
suspend fun setThemeMode(mode: String) {
|
||||
context.dataStore.edit { preferences ->
|
||||
preferences[THEME_MODE_KEY] = mode
|
||||
}
|
||||
}
|
||||
|
||||
suspend fun setDynamicColors(enabled: Boolean) {
|
||||
context.dataStore.edit { preferences ->
|
||||
preferences[DYNAMIC_COLORS_KEY] = enabled
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
package com.hotdeal.alarm.data.remote.cloudflare
|
||||
|
||||
import android.content.Context
|
||||
import android.content.SharedPreferences
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.withContext
|
||||
|
||||
/**
|
||||
* Cloudflare 우회 쿠키 관리자
|
||||
*/
|
||||
class BypassCookieManager(context: Context) {
|
||||
|
||||
private val prefs: SharedPreferences = context.getSharedPreferences(
|
||||
"cloudflare_cookies",
|
||||
Context.MODE_PRIVATE
|
||||
)
|
||||
|
||||
/**
|
||||
* 쿠키 저장
|
||||
*/
|
||||
suspend fun saveCookies(domain: String, cookies: Map<String, String>) = withContext(Dispatchers.IO) {
|
||||
prefs.edit().apply {
|
||||
cookies.forEach { (key, value) ->
|
||||
putString("${domain}_$key", value)
|
||||
}
|
||||
apply()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 쿠키 로드
|
||||
*/
|
||||
suspend fun loadCookies(domain: String): Map<String, String> = withContext(Dispatchers.IO) {
|
||||
val cookies = mutableMapOf<String, String>()
|
||||
prefs.all.forEach { (key, value) ->
|
||||
if (key.startsWith("${domain}_")) {
|
||||
val cookieKey = key.removePrefix("${domain}_")
|
||||
cookies[cookieKey] = value as String
|
||||
}
|
||||
}
|
||||
cookies
|
||||
}
|
||||
|
||||
/**
|
||||
* 쿠키 삭제
|
||||
*/
|
||||
suspend fun clearCookies(domain: String) = withContext(Dispatchers.IO) {
|
||||
prefs.edit().apply {
|
||||
prefs.all.keys.forEach { key ->
|
||||
if (key.startsWith("${domain}_")) {
|
||||
remove(key)
|
||||
}
|
||||
}
|
||||
apply()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 쿠키를 헤더 문자열로 변환
|
||||
*/
|
||||
fun toCookieHeader(cookies: Map<String, String>): String {
|
||||
return cookies.entries.joinToString("; ") { "${it.key}=${it.value}" }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
package com.hotdeal.alarm.data.remote.cloudflare
|
||||
|
||||
import android.content.Context
|
||||
import android.webkit.WebView
|
||||
import android.webkit.WebViewClient
|
||||
import kotlinx.coroutines.suspendCancellableCoroutine
|
||||
import kotlin.coroutines.resume
|
||||
import kotlin.coroutines.resumeWithException
|
||||
|
||||
/**
|
||||
* Cloudflare 우회 헬퍼
|
||||
*/
|
||||
class CloudflareBypass(private val context: Context) {
|
||||
|
||||
data class BypassResult(
|
||||
val html: String,
|
||||
val cookies: Map<String, String>
|
||||
)
|
||||
|
||||
/**
|
||||
* Cloudflare 챌린지 우회
|
||||
*/
|
||||
suspend fun bypass(url: String): BypassResult = suspendCancellableCoroutine { continuation ->
|
||||
val webView = WebView(context)
|
||||
webView.settings.javaScriptEnabled = true
|
||||
webView.settings.domStorageEnabled = true
|
||||
webView.settings.loadWithOverviewMode = true
|
||||
|
||||
var challengeCompleted = false
|
||||
|
||||
webView.webViewClient = object : WebViewClient() {
|
||||
override fun onPageFinished(view: WebView?, url: String?) {
|
||||
super.onPageFinished(view, url)
|
||||
|
||||
// Cloudflare 챌린지 완료 확인
|
||||
view?.evaluateJavascript(
|
||||
"(function() { return document.body !== null && !document.body.innerHTML.includes('cloudflare'); })();"
|
||||
) { result ->
|
||||
if (result == "true" && !challengeCompleted) {
|
||||
challengeCompleted = true
|
||||
|
||||
// HTML 추출
|
||||
view?.evaluateJavascript(
|
||||
"(function() { return document.documentElement.outerHTML; })();"
|
||||
) { html ->
|
||||
// 쿠키 추출
|
||||
val cookieManager = android.webkit.CookieManager.getInstance()
|
||||
val cookies = cookieManager.getCookie(url)
|
||||
|
||||
val cookieMap = parseCookies(cookies)
|
||||
|
||||
continuation.resume(
|
||||
BypassResult(
|
||||
html = html.trim('"').replace("\\u003C", "<").replace("\\u003E", ">"),
|
||||
cookies = cookieMap
|
||||
)
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
webView.loadUrl(url)
|
||||
|
||||
continuation.invokeOnCancellation {
|
||||
webView.stopLoading()
|
||||
webView.destroy()
|
||||
}
|
||||
}
|
||||
|
||||
private fun parseCookies(cookieString: String?): Map<String, String> {
|
||||
if (cookieString.isNullOrEmpty()) return emptyMap()
|
||||
|
||||
return cookieString.split(";")
|
||||
.map { it.trim() }
|
||||
.filter { it.contains("=") }
|
||||
.associate {
|
||||
val parts = it.split("=", limit = 2)
|
||||
parts[0] to parts.getOrElse(1) { "" }
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
package com.hotdeal.alarm.data.remote.interceptor
|
||||
|
||||
import okhttp3.Interceptor
|
||||
import okhttp3.Response
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
/**
|
||||
* Rate Limiting 인터셉터
|
||||
*
|
||||
* 동시성 안전을 위해 AtomicLong 사용
|
||||
*/
|
||||
class RateLimitInterceptor(
|
||||
private val minIntervalMillis: Long = 3000L
|
||||
) : Interceptor {
|
||||
|
||||
private val lastRequestTime = AtomicLong(0L)
|
||||
|
||||
override fun intercept(chain: Interceptor.Chain): Response {
|
||||
val currentTime = System.currentTimeMillis()
|
||||
val lastTime = lastRequestTime.get()
|
||||
val timeSinceLastRequest = currentTime - lastTime
|
||||
|
||||
if (timeSinceLastRequest < minIntervalMillis) {
|
||||
val sleepTime = minIntervalMillis - timeSinceLastRequest
|
||||
Thread.sleep(sleepTime)
|
||||
}
|
||||
|
||||
lastRequestTime.set(System.currentTimeMillis())
|
||||
return chain.proceed(chain.request())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
package com.hotdeal.alarm.data.remote.interceptor
|
||||
|
||||
import okhttp3.Interceptor
|
||||
import okhttp3.Response
|
||||
import java.io.IOException
|
||||
|
||||
/**
|
||||
* 재시도 인터셉터 (지수 백오프)
|
||||
*/
|
||||
class RetryInterceptor(
|
||||
private val maxRetries: Int = 3,
|
||||
private val initialDelayMs: Long = 1000L
|
||||
) : Interceptor {
|
||||
|
||||
override fun intercept(chain: Interceptor.Chain): Response {
|
||||
val request = chain.request()
|
||||
var lastException: IOException? = null
|
||||
var response: Response? = null
|
||||
|
||||
for (attempt in 0..maxRetries) {
|
||||
try {
|
||||
response?.close()
|
||||
response = chain.proceed(request)
|
||||
|
||||
if (response.isSuccessful) {
|
||||
return response
|
||||
}
|
||||
|
||||
// 서버 에러 또는 rate limiting인 경우 재시도
|
||||
if (response.code in listOf(429, 500, 502, 503, 504)) {
|
||||
response.close()
|
||||
val delay = initialDelayMs * (1 shl attempt) // 지수 백오프
|
||||
Thread.sleep(delay)
|
||||
continue
|
||||
}
|
||||
|
||||
return response
|
||||
} catch (e: IOException) {
|
||||
lastException = e
|
||||
response?.close()
|
||||
val delay = initialDelayMs * (1 shl attempt)
|
||||
Thread.sleep(delay)
|
||||
}
|
||||
}
|
||||
|
||||
throw lastException ?: IOException("Max retries exceeded")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
package com.hotdeal.alarm.data.remote.interceptor
|
||||
|
||||
import okhttp3.Interceptor
|
||||
import okhttp3.Response
|
||||
import kotlin.random.Random
|
||||
|
||||
/**
|
||||
* User-Agent 회전 인터셉터
|
||||
*/
|
||||
class UserAgentInterceptor : Interceptor {
|
||||
|
||||
private val userAgents = listOf(
|
||||
// Android Chrome
|
||||
"Mozilla/5.0 (Linux; Android 14; SM-G991B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Mobile Safari/537.36",
|
||||
"Mozilla/5.0 (Linux; Android 13; Pixel 7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Mobile Safari/537.36",
|
||||
"Mozilla/5.0 (Linux; Android 14; SM-S918B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Mobile Safari/537.36",
|
||||
// Desktop Chrome (일부 사이트는 데스크톱 선호)
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
|
||||
)
|
||||
|
||||
override fun intercept(chain: Interceptor.Chain): Response {
|
||||
val request = chain.request().newBuilder()
|
||||
.header("User-Agent", userAgents.random())
|
||||
.header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8")
|
||||
.header("Accept-Language", "ko-KR,ko;q=0.9,en-US;q=0.8,en;q=0.7")
|
||||
.header("Accept-Encoding", "gzip, deflate, br")
|
||||
.header("Connection", "keep-alive")
|
||||
.header("Upgrade-Insecure-Requests", "1")
|
||||
.header("Sec-Fetch-Dest", "document")
|
||||
.header("Sec-Fetch-Mode", "navigate")
|
||||
.header("Sec-Fetch-Site", "none")
|
||||
.header("Sec-Fetch-User", "?1")
|
||||
.header("Cache-Control", "max-age=0")
|
||||
.build()
|
||||
|
||||
return chain.proceed(request)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
package com.hotdeal.alarm.data.remote.scraper
|
||||
|
||||
import android.util.Log
|
||||
import com.hotdeal.alarm.domain.model.HotDeal
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.withContext
|
||||
import okhttp3.OkHttpClient
|
||||
import okhttp3.Request
|
||||
import org.jsoup.Jsoup
|
||||
|
||||
/**
|
||||
* 스크래퍼 기본 클래스
|
||||
*/
|
||||
abstract class BaseScraper(
|
||||
protected val client: OkHttpClient
|
||||
) {
|
||||
/**
|
||||
* 사이트 이름
|
||||
*/
|
||||
abstract val siteName: String
|
||||
|
||||
/**
|
||||
* 기본 URL
|
||||
*/
|
||||
abstract val baseUrl: String
|
||||
|
||||
/**
|
||||
* 게시판 URL 생성
|
||||
*/
|
||||
abstract fun getBoardUrl(board: String): String
|
||||
|
||||
/**
|
||||
* 스크래핑 수행
|
||||
*/
|
||||
abstract suspend fun scrape(board: String): Result<List<HotDeal>>
|
||||
|
||||
/**
|
||||
* HTML 인코딩 (기본값: UTF-8)
|
||||
*/
|
||||
protected open val charset: String = "UTF-8"
|
||||
|
||||
/**
|
||||
* HTML 가져오기
|
||||
*/
|
||||
protected suspend fun fetchHtml(url: String): String? = withContext(Dispatchers.IO) {
|
||||
try {
|
||||
Log.d("Scraper", "[$siteName] 요청: $url")
|
||||
val request = Request.Builder()
|
||||
.url(url)
|
||||
.header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
|
||||
.header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8")
|
||||
.header("Accept-Language", "ko-KR,ko;q=0.9,en-US;q=0.8,en;q=0.7")
|
||||
.header("Referer", baseUrl)
|
||||
.header("Connection", "keep-alive")
|
||||
.header("Upgrade-Insecure-Requests", "1")
|
||||
.build()
|
||||
|
||||
client.newCall(request).execute().use { response ->
|
||||
Log.d("Scraper", "[$siteName] 응답 코드: ${response.code}")
|
||||
if (!response.isSuccessful) {
|
||||
Log.e("Scraper", "[$siteName] 요청 실패: ${response.code}")
|
||||
return@withContext null
|
||||
}
|
||||
|
||||
// 문자열로 직접 읽기 (OkHttp가 자동으로 인코딩 처리)
|
||||
val html = response.body?.string()
|
||||
if (html == null) {
|
||||
Log.e("Scraper", "[$siteName] 응답 바디가 null입니다")
|
||||
return@withContext null
|
||||
}
|
||||
|
||||
Log.d("Scraper", "[$siteName] HTML 길이: ${html.length}")
|
||||
|
||||
// 디버깅: HTML 일부 출력
|
||||
if (html.length > 0) {
|
||||
Log.d("Scraper", "[$siteName] HTML 샘플: ${html.take(300)}")
|
||||
}
|
||||
|
||||
html
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
Log.e("Scraper", "[$siteName] 요청 예외: ${e.message}")
|
||||
null
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* HTML 파싱
|
||||
*/
|
||||
protected fun parseHtml(html: String, baseUrl: String): org.jsoup.nodes.Document {
|
||||
return Jsoup.parse(html, baseUrl)
|
||||
}
|
||||
|
||||
/**
|
||||
* 상대 URL을 절대 URL로 변환
|
||||
*/
|
||||
protected fun resolveUrl(baseUrl: String, relativeUrl: String): String {
|
||||
if (relativeUrl.startsWith("http")) return relativeUrl
|
||||
if (relativeUrl.startsWith("//")) return "https:$relativeUrl"
|
||||
if (relativeUrl.startsWith("/")) return baseUrl.trimEnd('/') + relativeUrl
|
||||
return baseUrl.trimEnd('/') + "/" + relativeUrl
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
package com.hotdeal.alarm.data.remote.scraper
|
||||
|
||||
import android.util.Log
|
||||
import com.hotdeal.alarm.domain.model.HotDeal
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.delay
|
||||
import kotlinx.coroutines.withContext
|
||||
import okhttp3.OkHttpClient
|
||||
import org.jsoup.Jsoup
|
||||
import org.jsoup.nodes.Document
|
||||
import org.jsoup.select.Elements
|
||||
import kotlin.random.Random
|
||||
|
||||
/**
|
||||
* 클리앙 스크래퍼
|
||||
*/
|
||||
class ClienScraper(client: OkHttpClient) : BaseScraper(client) {
|
||||
|
||||
override val siteName: String = "clien"
|
||||
override val baseUrl: String = "https://www.clien.net"
|
||||
|
||||
override fun getBoardUrl(board: String): String {
|
||||
return when (board) {
|
||||
"allsell" -> "$baseUrl/service/group/allsell"
|
||||
else -> "$baseUrl/service/board/$board"
|
||||
}
|
||||
}
|
||||
|
||||
override suspend fun scrape(board: String): Result<List<HotDeal>> = withContext(Dispatchers.IO) {
|
||||
try {
|
||||
val url = getBoardUrl(board)
|
||||
Log.d("Clien", "스크래핑 시작: $url")
|
||||
|
||||
// 요청 간격 랜덤화 (2~4초)
|
||||
val delayTime = Random.nextLong(2000, 4000)
|
||||
Log.d("Clien", "요청 대기: ${delayTime}ms")
|
||||
delay(delayTime)
|
||||
|
||||
val userAgent = getRandomUserAgent()
|
||||
|
||||
val doc: Document = Jsoup.connect(url)
|
||||
.userAgent(userAgent)
|
||||
.header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
|
||||
.header("Accept-Language", "ko-KR,ko;q=0.9,en-US;q=0.8,en;q=0.7")
|
||||
.header("Referer", "https://www.clien.net/")
|
||||
.timeout(30000)
|
||||
.followRedirects(true)
|
||||
.get()
|
||||
|
||||
Log.d("Clien", "문서 파싱 성공, 길이: ${doc.html().length}")
|
||||
|
||||
val deals = mutableListOf<HotDeal>()
|
||||
|
||||
val elements: Elements = doc.select("div.list_item.symph_row")
|
||||
Log.d("Clien", "찾은 요소: ${elements.size}개")
|
||||
|
||||
var count = 0
|
||||
elements.forEach { item ->
|
||||
if (count >= 20) return@forEach
|
||||
|
||||
try {
|
||||
val titleElement = item.selectFirst("a.list_subject, span.list_subject a")
|
||||
val title = titleElement?.text()?.trim() ?: return@forEach
|
||||
if (title.isEmpty()) return@forEach
|
||||
|
||||
val href = titleElement.attr("href")
|
||||
val dealUrl = resolveUrl(baseUrl, href)
|
||||
|
||||
val postId = href.substringAfterLast("/").substringBefore("?").ifEmpty {
|
||||
href.substringAfterLast("/").ifEmpty { return@forEach }
|
||||
}
|
||||
|
||||
val deal = HotDeal(
|
||||
id = HotDeal.generateId(siteName, postId),
|
||||
siteName = siteName,
|
||||
boardName = board,
|
||||
title = title,
|
||||
url = dealUrl,
|
||||
createdAt = System.currentTimeMillis()
|
||||
)
|
||||
deals.add(deal)
|
||||
count++
|
||||
Log.d("Clien", "[$count] $title")
|
||||
} catch (e: Exception) {
|
||||
Log.e("Clien", "파싱 에러: ${e.message}")
|
||||
}
|
||||
}
|
||||
|
||||
Log.d("Clien", "파싱 완료: ${deals.size}개")
|
||||
Result.success(deals)
|
||||
} catch (e: Exception) {
|
||||
Log.e("Clien", "스크래핑 실패: ${e.message}", e)
|
||||
Result.failure(e)
|
||||
}
|
||||
}
|
||||
|
||||
private fun getRandomUserAgent(): String {
|
||||
val userAgents = listOf(
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:120.0) Gecko/20100101 Firefox/120.0"
|
||||
)
|
||||
return userAgents.random()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
package com.hotdeal.alarm.data.remote.scraper
|
||||
|
||||
import android.util.Log
|
||||
import com.hotdeal.alarm.domain.model.HotDeal
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.delay
|
||||
import kotlinx.coroutines.withContext
|
||||
import okhttp3.OkHttpClient
|
||||
import org.jsoup.Jsoup
|
||||
import org.jsoup.nodes.Document
|
||||
import org.jsoup.select.Elements
|
||||
import kotlin.random.Random
|
||||
|
||||
/**
|
||||
* 쿨엔조이 스크래퍼
|
||||
*/
|
||||
class CoolenjoyScraper(client: OkHttpClient) : BaseScraper(client) {
|
||||
|
||||
override val siteName: String = "coolenjoy"
|
||||
override val baseUrl: String = "https://coolenjoy.net"
|
||||
|
||||
override fun getBoardUrl(board: String): String {
|
||||
return "$baseUrl/bbs/$board"
|
||||
}
|
||||
|
||||
override suspend fun scrape(board: String): Result<List<HotDeal>> = withContext(Dispatchers.IO) {
|
||||
try {
|
||||
val url = getBoardUrl(board)
|
||||
Log.d("Coolenjoy", "스크래핑 시작: $url")
|
||||
|
||||
// 요청 간격 랜덤화 (2~4초)
|
||||
val delayTime = Random.nextLong(2000, 4000)
|
||||
Log.d("Coolenjoy", "요청 대기: ${delayTime}ms")
|
||||
delay(delayTime)
|
||||
|
||||
val userAgent = getRandomUserAgent()
|
||||
|
||||
val doc: Document = Jsoup.connect(url)
|
||||
.userAgent(userAgent)
|
||||
.header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
|
||||
.header("Accept-Language", "ko-KR,ko;q=0.9,en-US;q=0.8,en;q=0.7")
|
||||
.header("Referer", "https://coolenjoy.net/")
|
||||
.timeout(30000)
|
||||
.followRedirects(true)
|
||||
.get()
|
||||
|
||||
Log.d("Coolenjoy", "문서 파싱 성공, 길이: ${doc.html().length}")
|
||||
|
||||
val deals = mutableListOf<HotDeal>()
|
||||
|
||||
val elements: Elements = doc.select("a.na-subject")
|
||||
Log.d("Coolenjoy", "찾은 요소: ${elements.size}개")
|
||||
|
||||
var count = 0
|
||||
elements.forEach { element ->
|
||||
if (count >= 20) return@forEach
|
||||
|
||||
try {
|
||||
val title = element.text().trim()
|
||||
if (title.isEmpty()) return@forEach
|
||||
|
||||
val href = element.attr("href")
|
||||
val dealUrl = resolveUrl(baseUrl, href)
|
||||
|
||||
val postId = extractPostId(href)
|
||||
if (postId.isEmpty()) return@forEach
|
||||
|
||||
val deal = HotDeal(
|
||||
id = HotDeal.generateId(siteName, postId),
|
||||
siteName = siteName,
|
||||
boardName = board,
|
||||
title = title,
|
||||
url = dealUrl,
|
||||
createdAt = System.currentTimeMillis()
|
||||
)
|
||||
deals.add(deal)
|
||||
count++
|
||||
Log.d("Coolenjoy", "[$count] $title")
|
||||
} catch (e: Exception) {
|
||||
Log.e("Coolenjoy", "파싱 에러: ${e.message}")
|
||||
}
|
||||
}
|
||||
|
||||
Log.d("Coolenjoy", "파싱 완료: ${deals.size}개")
|
||||
Result.success(deals)
|
||||
} catch (e: Exception) {
|
||||
Log.e("Coolenjoy", "스크래핑 실패: ${e.message}", e)
|
||||
Result.failure(e)
|
||||
}
|
||||
}
|
||||
|
||||
private fun extractPostId(href: String): String {
|
||||
val fromPath = href.substringAfterLast("/").substringBefore("?")
|
||||
if (fromPath.isNotEmpty() && fromPath.all { it.isDigit() }) {
|
||||
return fromPath
|
||||
}
|
||||
val fromWrId = href.substringAfter("wr_id=").substringBefore("&")
|
||||
if (fromWrId.isNotEmpty()) {
|
||||
return fromWrId
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
private fun getRandomUserAgent(): String {
|
||||
val userAgents = listOf(
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:120.0) Gecko/20100101 Firefox/120.0"
|
||||
)
|
||||
return userAgents.random()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
package com.hotdeal.alarm.data.remote.scraper
|
||||
|
||||
import android.util.Log
|
||||
import com.hotdeal.alarm.domain.model.HotDeal
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.delay
|
||||
import kotlinx.coroutines.withContext
|
||||
import okhttp3.OkHttpClient
|
||||
import org.jsoup.Jsoup
|
||||
import org.jsoup.nodes.Document
|
||||
import org.jsoup.select.Elements
|
||||
import kotlin.random.Random
|
||||
|
||||
/**
|
||||
* 뽐뿌 스크래퍼
|
||||
*/
|
||||
class PpomppuScraper(client: OkHttpClient) : BaseScraper(client) {
|
||||
|
||||
override val siteName: String = "ppomppu"
|
||||
override val baseUrl: String = "https://www.ppomppu.co.kr/zboard/"
|
||||
|
||||
override fun getBoardUrl(board: String): String {
|
||||
return "${baseUrl}zboard.php?id=$board"
|
||||
}
|
||||
|
||||
override suspend fun scrape(board: String): Result<List<HotDeal>> = withContext(Dispatchers.IO) {
|
||||
try {
|
||||
val url = getBoardUrl(board)
|
||||
Log.d("Ppomppu", "스크래핑 시작: $url")
|
||||
|
||||
// 요청 간격 랜덤화 (2~4초) - 차단 방지
|
||||
val delayTime = Random.nextLong(2000, 4000)
|
||||
Log.d("Ppomppu", "요청 대기: ${delayTime}ms")
|
||||
delay(delayTime)
|
||||
|
||||
// Jsoup으로 직접 연결 (User-Agent 회전)
|
||||
val userAgent = getRandomUserAgent()
|
||||
Log.d("Ppomppu", "User-Agent: $userAgent")
|
||||
|
||||
val doc: Document = Jsoup.connect(url)
|
||||
.userAgent(userAgent)
|
||||
.header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
|
||||
.header("Accept-Language", "ko-KR,ko;q=0.9,en-US;q=0.8,en;q=0.7")
|
||||
.header("Accept-Charset", "utf-8,ISO-8859-1")
|
||||
.header("Referer", "https://www.ppomppu.co.kr/")
|
||||
.timeout(30000)
|
||||
.followRedirects(true)
|
||||
.get()
|
||||
|
||||
Log.d("Ppomppu", "문서 파싱 성공, 길이: ${doc.html().length}")
|
||||
|
||||
val deals = mutableListOf<HotDeal>()
|
||||
|
||||
// 셀렉터로 요소 찾기
|
||||
val elements: Elements = doc.select("a.baseList-title")
|
||||
Log.d("Ppomppu", "찾은 요소: ${elements.size}개")
|
||||
|
||||
// 최대 20개까지만 처리
|
||||
var count = 0
|
||||
elements.forEach { element ->
|
||||
if (count >= 20) return@forEach
|
||||
|
||||
try {
|
||||
val title = element.text().trim()
|
||||
if (title.isEmpty()) return@forEach
|
||||
|
||||
val href = element.attr("href")
|
||||
|
||||
// 공지사항 제외
|
||||
if (href.contains("regulation") || href.contains("notice")) return@forEach
|
||||
|
||||
val dealUrl = resolveUrl(baseUrl, href)
|
||||
|
||||
// postId 추출
|
||||
val postId = extractPostId(href)
|
||||
if (postId.isEmpty()) {
|
||||
Log.w("Ppomppu", "postId 추출 실패: href=$href")
|
||||
return@forEach
|
||||
}
|
||||
|
||||
val deal = HotDeal(
|
||||
id = HotDeal.generateId(siteName, postId),
|
||||
siteName = siteName,
|
||||
boardName = board,
|
||||
title = title,
|
||||
url = dealUrl,
|
||||
createdAt = System.currentTimeMillis()
|
||||
)
|
||||
deals.add(deal)
|
||||
count++
|
||||
Log.d("Ppomppu", "[$count] $title")
|
||||
} catch (e: Exception) {
|
||||
Log.e("Ppomppu", "파싱 에러: ${e.message}")
|
||||
}
|
||||
}
|
||||
|
||||
Log.d("Ppomppu", "파싱 완료: ${deals.size}개")
|
||||
Result.success(deals)
|
||||
} catch (e: Exception) {
|
||||
Log.e("Ppomppu", "스크래핑 실패: ${e.message}", e)
|
||||
Result.failure(e)
|
||||
}
|
||||
}
|
||||
|
||||
private fun extractPostId(href: String): String {
|
||||
val afterNo = href.substringAfter("no=", "")
|
||||
if (afterNo.isEmpty()) return ""
|
||||
val postId = afterNo.takeWhile { it != '&' && it != '/' && it != '#' }
|
||||
return if (postId.isNotEmpty() && postId.all { it.isDigit() }) postId else ""
|
||||
}
|
||||
|
||||
private fun getRandomUserAgent(): String {
|
||||
val userAgents = listOf(
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:120.0) Gecko/20100101 Firefox/120.0",
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.1 Safari/605.1.15",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36 Edg/119.0.0.0"
|
||||
)
|
||||
return userAgents.random()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
package com.hotdeal.alarm.data.remote.scraper
|
||||
|
||||
import android.util.Log
|
||||
import com.hotdeal.alarm.domain.model.HotDeal
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.delay
|
||||
import kotlinx.coroutines.withContext
|
||||
import okhttp3.OkHttpClient
|
||||
import org.jsoup.Jsoup
|
||||
import org.jsoup.nodes.Document
|
||||
import org.jsoup.select.Elements
|
||||
import kotlin.random.Random
|
||||
|
||||
/**
|
||||
* 루리웹 스크래퍼
|
||||
*/
|
||||
class RuriwebScraper(client: OkHttpClient) : BaseScraper(client) {
|
||||
|
||||
override val siteName: String = "ruriweb"
|
||||
override val baseUrl: String = "https://bbs.ruliweb.com"
|
||||
|
||||
override fun getBoardUrl(board: String): String {
|
||||
return "$baseUrl/market/board/$board"
|
||||
}
|
||||
|
||||
override suspend fun scrape(board: String): Result<List<HotDeal>> = withContext(Dispatchers.IO) {
|
||||
try {
|
||||
val url = getBoardUrl(board)
|
||||
Log.d("Ruriweb", "스크래핑 시작: $url")
|
||||
|
||||
// 요청 간격 랜덤화 (2~4초)
|
||||
val delayTime = Random.nextLong(2000, 4000)
|
||||
Log.d("Ruriweb", "요청 대기: ${delayTime}ms")
|
||||
delay(delayTime)
|
||||
|
||||
val userAgent = getRandomUserAgent()
|
||||
|
||||
val doc: Document = Jsoup.connect(url)
|
||||
.userAgent(userAgent)
|
||||
.header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
|
||||
.header("Accept-Language", "ko-KR,ko;q=0.9,en-US;q=0.8,en;q=0.7")
|
||||
.header("Referer", "https://bbs.ruliweb.com/")
|
||||
.timeout(30000)
|
||||
.followRedirects(true)
|
||||
.get()
|
||||
|
||||
Log.d("Ruriweb", "문서 파싱 성공, 길이: ${doc.html().length}")
|
||||
|
||||
val deals = mutableListOf<HotDeal>()
|
||||
|
||||
val elements: Elements = doc.select("td.subject a.subject_link.deco")
|
||||
Log.d("Ruriweb", "찾은 요소: ${elements.size}개")
|
||||
|
||||
var count = 0
|
||||
elements.forEach { element ->
|
||||
if (count >= 20) return@forEach
|
||||
|
||||
try {
|
||||
val strongEl = element.selectFirst("strong")
|
||||
val title = (strongEl?.text() ?: element.text()).trim()
|
||||
if (title.isEmpty()) return@forEach
|
||||
|
||||
val href = element.attr("href")
|
||||
val dealUrl = resolveUrl(baseUrl, href)
|
||||
|
||||
val postId = extractPostId(href)
|
||||
if (postId.isEmpty()) return@forEach
|
||||
|
||||
val deal = HotDeal(
|
||||
id = HotDeal.generateId(siteName, postId),
|
||||
siteName = siteName,
|
||||
boardName = board,
|
||||
title = title,
|
||||
url = dealUrl,
|
||||
createdAt = System.currentTimeMillis()
|
||||
)
|
||||
deals.add(deal)
|
||||
count++
|
||||
Log.d("Ruriweb", "[$count] $title")
|
||||
} catch (e: Exception) {
|
||||
Log.e("Ruriweb", "파싱 에러: ${e.message}")
|
||||
}
|
||||
}
|
||||
|
||||
Log.d("Ruriweb", "파싱 완료: ${deals.size}개")
|
||||
Result.success(deals)
|
||||
} catch (e: Exception) {
|
||||
Log.e("Ruriweb", "스크래핑 실패: ${e.message}", e)
|
||||
Result.failure(e)
|
||||
}
|
||||
}
|
||||
|
||||
private fun extractPostId(href: String): String {
|
||||
val fromRead = href.substringAfter("/read/").substringBefore("?")
|
||||
if (fromRead.isNotEmpty() && fromRead.all { it.isDigit() }) {
|
||||
return fromRead
|
||||
}
|
||||
val fromPath = href.substringAfterLast("/").substringBefore("?")
|
||||
if (fromPath.isNotEmpty() && fromPath.all { it.isDigit() }) {
|
||||
return fromPath
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
private fun getRandomUserAgent(): String {
|
||||
val userAgents = listOf(
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:120.0) Gecko/20100101 Firefox/120.0"
|
||||
)
|
||||
return userAgents.random()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
package com.hotdeal.alarm.data.remote.scraper
|
||||
|
||||
import com.hotdeal.alarm.domain.model.SiteType
|
||||
|
||||
/**
|
||||
* 스크래퍼 팩토리
|
||||
*/
|
||||
class ScraperFactory(
|
||||
private val ppomppu: PpomppuScraper,
|
||||
private val clien: ClienScraper,
|
||||
private val ruriweb: RuriwebScraper,
|
||||
private val coolenjoy: CoolenjoyScraper
|
||||
) {
|
||||
/**
|
||||
* 사이트 타입에 따른 스크래퍼 반환
|
||||
*/
|
||||
fun getScraper(siteType: SiteType): BaseScraper {
|
||||
return when (siteType) {
|
||||
SiteType.PPOMPPU -> ppomppu
|
||||
SiteType.CLIEN -> clien
|
||||
SiteType.RURIWEB -> ruriweb
|
||||
SiteType.COOLENJOY -> coolenjoy
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 사이트 이름으로 스크래퍼 반환
|
||||
*/
|
||||
fun getScraper(siteName: String): BaseScraper? {
|
||||
return when (siteName.lowercase()) {
|
||||
"ppomppu" -> ppomppu
|
||||
"clien" -> clien
|
||||
"ruriweb" -> ruriweb
|
||||
"coolenjoy" -> coolenjoy
|
||||
else -> null
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 모든 스크래퍼 반환
|
||||
*/
|
||||
fun getAllScrapers(): List<BaseScraper> {
|
||||
return listOf(ppomppu, clien, ruriweb, coolenjoy)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user