Add support for ksoup and migrate core extractors to it

app/build.gradle.kts

@@ -253,7 +253,8 @@ dependencies {
     implementation(libs.qrcode.kotlin) // QR Code for PIN Auth on TV
 
     // Extensions & Other Libs
-    implementation(libs.jsoup) // HTML Parser
+    implementation(libs.jsoup) // HTML Parser (Deprecated usage)
+    implementation(libs.ksoup) // HTML Parser (KMP support)
     implementation(libs.rhino) // Run JavaScript
     implementation(libs.fuzzywuzzy) // Library/Ext Searching with Levenshtein Distance
     implementation(libs.safefile) // To Prevent the URI File Fu*kery

app/src/main/java/com/lagradost/cloudstream3/syncproviders/providers/Addic7ed.kt

@@ -2,6 +2,7 @@ package com.lagradost.cloudstream3.syncproviders.providers
 
 import com.lagradost.cloudstream3.AllLanguagesName
 import com.lagradost.cloudstream3.app
+import com.lagradost.cloudstream3.ksoupDocument
 import com.lagradost.cloudstream3.subtitles.AbstractSubtitleEntities.SubtitleEntity
 import com.lagradost.cloudstream3.subtitles.AbstractSubtitleEntities.SubtitleSearch
 import com.lagradost.cloudstream3.syncproviders.AuthData
@@ -66,7 +67,7 @@ class Addic7ed : SubtitleAPI() {
         }
 
         val response = app.get(url = "$HOST/search.php?search=$searchQuery&Submit=Search")
-        val hostDocument = response.document
+        val hostDocument = response.ksoupDocument
 
         // 1st case: found one movie or episode. Redirected to $HOST/movie/1234 or $HOST/serie/show-name/$seasonNum/$epNum/ep-name
         if (response.url.contains("/movie/") || response.url.contains("/serie/"))
@@ -78,7 +79,7 @@ class Addic7ed : SubtitleAPI() {
             val doc = app.get(
                 "$HOST/ajax_loadShow.php?show=$showId&season=$seasonNum&langs=|$langNumAddic7ed|&hd=0&hi=0",
                 referer = "$HOST/"
-            ).document
+            ).ksoupDocument
 
             // get direct subtitles links from list
             return doc.select("#season tbody tr").mapNotNull { node ->
@@ -105,7 +106,7 @@ class Addic7ed : SubtitleAPI() {
         // filter download page by language. Do not work for movies :/
         if (downloadPage.contains("/serie/"))
             downloadPage = downloadPage.substringBeforeLast("/") + "/$langNumAddic7ed"
-        val doc = app.get(url = downloadPage).document
+        val doc = app.get(url = downloadPage).ksoupDocument
 
         // get subtitles links from download page
         return doc.select(".tabel95 .tabel95 tr:has(.language):contains($langName)").mapNotNull { node ->

gradle/libs.versions.toml

@@ -28,6 +28,7 @@ juniversalchardet = "2.5.0"
 kotlinGradlePlugin = "2.3.20"
 kotlinxCollectionsImmutable = "0.4.0"
 kotlinxCoroutinesCore = "1.10.2"
+ksoup = "0.2.6"
 lifecycleKtx = "2.10.0"
 material = "1.14.0-beta01"
 media3 = "1.9.3"
@@ -83,6 +84,7 @@ junit-ktx = { module = "androidx.test.ext:junit-ktx", version.ref = "junitKtx" }
 juniversalchardet = { module = "com.github.albfernandez:juniversalchardet", version.ref = "juniversalchardet" }
 kotlinx-collections-immutable = { module = "org.jetbrains.kotlinx:kotlinx-collections-immutable", version.ref = "kotlinxCollectionsImmutable" }
 kotlinx-coroutines-core = { module = "org.jetbrains.kotlinx:kotlinx-coroutines-core", version.ref = "kotlinxCoroutinesCore" }
+ksoup = { module = "com.fleeksoft.ksoup:ksoup", version.ref = "ksoup" }
 lifecycle-livedata-ktx = { module = "androidx.lifecycle:lifecycle-livedata-ktx", version.ref = "lifecycleKtx" }
 lifecycle-viewmodel-ktx = { module = "androidx.lifecycle:lifecycle-viewmodel-ktx", version.ref = "lifecycleKtx" }
 material = { module = "com.google.android.material:material", version.ref = "material" }

library/build.gradle.kts

@@ -58,7 +58,7 @@ kotlin {
             implementation(libs.jackson.module.kotlin) // JSON Parser
             implementation(libs.kotlinx.coroutines.core)
             implementation(libs.fuzzywuzzy) // Match Extractors
-            implementation(libs.jsoup) // HTML Parser
+            implementation(libs.ksoup) // HTML Parser
             implementation(libs.rhino) // Run JavaScript
             implementation(libs.newpipeextractor)
             implementation(libs.tmdb.java) // TMDB API v3 Wrapper Made with RetroFit

library/src/commonMain/kotlin/com/lagradost/cloudstream3/MainAPI.kt

@@ -177,7 +177,7 @@ object APIHolder {
                     .substringBefore("/")
             val recapToken =
                 app.get("https://www.google.com/recaptcha/api2/anchor?ar=1&hl=en&size=invisible&cb=cs3&k=$key&co=$domain&v=$vToken")
-                    .document
+                    .ksoupDocument
                     .selectFirst("#recaptcha-token")?.attr("value")
             if (recapToken != null) {
                 return app.post(

library/src/commonMain/kotlin/com/lagradost/cloudstream3/MainActivity.kt

@@ -3,6 +3,9 @@ package com.lagradost.cloudstream3
 import com.fasterxml.jackson.databind.DeserializationFeature
 import com.fasterxml.jackson.databind.ObjectMapper
 import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
+import com.fleeksoft.ksoup.Ksoup
+import com.fleeksoft.ksoup.nodes.Document
+import com.lagradost.nicehttp.NiceResponse
 import com.lagradost.nicehttp.Requests
 import com.lagradost.nicehttp.ResponseParser
 import kotlin.reflect.KClass
@@ -37,6 +40,10 @@ var app = Requests(responseParser = jacksonResponseParser).apply {
     defaultHeaders = mapOf("user-agent" to USER_AGENT)
 }
 
+/** Parses the response body as a Ksoup Document. */
+val NiceResponse.ksoupDocument: Document
+    get() = Ksoup.parse(text)
+
 /** Same as the default app networking helper, but this instance ignores SSL certificates.
  * This should NEVER be used for sensitive networking operations such as logins. Only use this when required. */
 @Prerelease

library/src/commonMain/kotlin/com/lagradost/cloudstream3/extractors/Blogger.kt

@@ -2,6 +2,7 @@ package com.lagradost.cloudstream3.extractors
 
 import com.fasterxml.jackson.annotation.JsonProperty
 import com.lagradost.cloudstream3.app
+import com.lagradost.cloudstream3.ksoupDocument
 import com.lagradost.cloudstream3.utils.*
 import com.lagradost.cloudstream3.utils.AppUtils.tryParseJson
 
@@ -12,7 +13,7 @@ open class Blogger : ExtractorApi() {
 
     override suspend fun getUrl(url: String, referer: String?): List<ExtractorLink> {
         val sources = mutableListOf<ExtractorLink>()
-        with(app.get(url).document) {
+        with(app.get(url).ksoupDocument) {
             this.select("script").map { script ->
                 if (script.data().contains("\"streams\":[")) {
                     val data = script.data().substringAfter("\"streams\":[")

library/src/commonMain/kotlin/com/lagradost/cloudstream3/extractors/Cda.kt

@@ -2,6 +2,7 @@ package com.lagradost.cloudstream3.extractors
 
 import com.lagradost.cloudstream3.USER_AGENT
 import com.lagradost.cloudstream3.app
+import com.lagradost.cloudstream3.ksoupDocument
 import com.lagradost.cloudstream3.utils.AppUtils.tryParseJson
 import com.lagradost.cloudstream3.utils.ExtractorApi
 import com.lagradost.cloudstream3.utils.ExtractorLink
@@ -25,7 +26,7 @@ open class Cda : ExtractorApi() {
                 "User-Agent" to USER_AGENT,
                 "Cookie" to "cda.player=html5"
             )
-        ).document
+        ).ksoupDocument
         val dataRaw = doc.selectFirst("[player_data]")?.attr("player_data") ?: return null
         val playerData = tryParseJson<PlayerData>(dataRaw) ?: return null
         return listOf(

library/src/commonMain/kotlin/com/lagradost/cloudstream3/extractors/Embedgram.kt

@@ -2,6 +2,7 @@ package com.lagradost.cloudstream3.extractors
 
 import com.lagradost.cloudstream3.SubtitleFile
 import com.lagradost.cloudstream3.app
+import com.lagradost.cloudstream3.ksoupDocument
 import com.lagradost.cloudstream3.utils.ExtractorApi
 import com.lagradost.cloudstream3.utils.ExtractorLink
 import com.lagradost.cloudstream3.utils.getQualityFromName
@@ -19,7 +20,7 @@ open class Embedgram : ExtractorApi() {
         subtitleCallback: (SubtitleFile) -> Unit,
         callback: (ExtractorLink) -> Unit
     ) {
-        val document = app.get(url, referer = referer).document
+        val document = app.get(url, referer = referer).ksoupDocument
         val link = document.select("video source:last-child").attr("src")
         val quality = document.select("video source:last-child").attr("title")
         callback.invoke(

library/src/commonMain/kotlin/com/lagradost/cloudstream3/extractors/EmturbovidExtractor.kt

@@ -1,6 +1,7 @@
 package com.lagradost.cloudstream3.extractors
 
 import com.lagradost.cloudstream3.app
+import com.lagradost.cloudstream3.ksoupDocument
 import com.lagradost.cloudstream3.utils.ExtractorApi
 import com.lagradost.cloudstream3.utils.ExtractorLink
 import com.lagradost.cloudstream3.utils.ExtractorLinkType
@@ -16,9 +17,9 @@ open class EmturbovidExtractor : ExtractorApi() {
         val response = app.get(
             url, referer = referer ?: "$mainUrl/"
         )
-        val playerScript =
-            response.document.selectXpath("//script[contains(text(),'var urlPlay')]")
-                .html()
+        val playerScript = response.ksoupDocument
+            .select("script:contains(var urlPlay)")
+            .html()
 
         val sources = mutableListOf<ExtractorLink>()
         if (playerScript.isNotBlank()) {

library/src/commonMain/kotlin/com/lagradost/cloudstream3/extractors/Fastream.kt

@@ -4,6 +4,7 @@ import com.lagradost.cloudstream3.SubtitleFile
 import com.lagradost.cloudstream3.amap
 import com.lagradost.cloudstream3.app
 import com.lagradost.cloudstream3.extractors.helper.JwPlayerHelper
+import com.lagradost.cloudstream3.ksoupDocument
 import com.lagradost.cloudstream3.utils.ExtractorApi
 import com.lagradost.cloudstream3.utils.ExtractorLink
 import com.lagradost.cloudstream3.utils.getAndUnpack
@@ -30,9 +31,9 @@ open class Fastream : ExtractorApi() {
                     "file_code" to id,
                     "auto" to "1"
                 )
-            ).document
+            ).ksoupDocument
         } else {
-            app.get(url, referer = url).document
+            app.get(url, referer = url).ksoupDocument
         }
         response.select("script").amap { script ->
             if (getPacked(script.data()) != null) {

library/src/commonMain/kotlin/com/lagradost/cloudstream3/extractors/Filegram.kt

@@ -1,13 +1,14 @@
 package com.lagradost.cloudstream3.extractors
 
+import com.fleeksoft.ksoup.nodes.Element
 import com.lagradost.cloudstream3.SubtitleFile
 import com.lagradost.cloudstream3.USER_AGENT
 import com.lagradost.cloudstream3.app
 import com.lagradost.cloudstream3.extractors.helper.JwPlayerHelper
+import com.lagradost.cloudstream3.ksoupDocument
 import com.lagradost.cloudstream3.utils.ExtractorApi
 import com.lagradost.cloudstream3.utils.ExtractorLink
 import com.lagradost.cloudstream3.utils.getAndUnpack
-import org.jsoup.nodes.Element
 
 open class Filegram : ExtractorApi() {
     override val name = "Filegram"
@@ -32,7 +33,7 @@ open class Filegram : ExtractorApi() {
             "user-agent" to USER_AGENT,
         )
 
-        val doc = app.get(getEmbedUrl(url), referer = referer).document
+        val doc = app.get(getEmbedUrl(url), referer = referer).ksoupDocument
         val unpackedJs = unpackJs(doc).toString()
 
         JwPlayerHelper.extractStreamLinks(unpackedJs, name, mainUrl, callback, subtitleCallback, headers = header)

library/src/commonMain/kotlin/com/lagradost/cloudstream3/extractors/Filemoon.kt

@@ -4,6 +4,7 @@ import com.lagradost.api.Log
 import com.lagradost.cloudstream3.SubtitleFile
 import com.lagradost.cloudstream3.app
 import com.lagradost.cloudstream3.extractors.helper.JwPlayerHelper
+import com.lagradost.cloudstream3.ksoupDocument
 import com.lagradost.cloudstream3.network.WebViewResolver
 import com.lagradost.cloudstream3.utils.ExtractorApi
 import com.lagradost.cloudstream3.utils.ExtractorLink
@@ -47,10 +48,10 @@ open class FilemoonV2 : ExtractorApi() {
         )
 
         val initialResponse = app.get(url, defaultHeaders)
-        val iframeSrcUrl = initialResponse.document.selectFirst("iframe")?.attr("src")
+        val iframeSrcUrl = initialResponse.ksoupDocument.selectFirst("iframe")?.attr("src")
 
         if (iframeSrcUrl.isNullOrEmpty()) {
-            val fallbackScriptData = initialResponse.document
+            val fallbackScriptData = initialResponse.ksoupDocument
                 .selectFirst("script:containsData(function(p,a,c,k,e,d))")
                 ?.data().orEmpty()
             val unpackedScript = JsUnpacker(fallbackScriptData).unpack()
@@ -74,7 +75,7 @@ open class FilemoonV2 : ExtractorApi() {
         val iframeHeaders = defaultHeaders + ("Accept-Language" to "en-US,en;q=0.5")
         val iframeResponse = app.get(iframeSrcUrl, headers = iframeHeaders)
 
-        val iframeScriptData = iframeResponse.document
+        val iframeScriptData = iframeResponse.ksoupDocument
             .selectFirst("script:containsData(function(p,a,c,k,e,d))")
             ?.data().orEmpty()
 

library/src/commonMain/kotlin/com/lagradost/cloudstream3/extractors/Filesim.kt

@@ -2,6 +2,7 @@ package com.lagradost.cloudstream3.extractors
 
 import com.lagradost.cloudstream3.SubtitleFile
 import com.lagradost.cloudstream3.app
+import com.lagradost.cloudstream3.ksoupDocument
 import com.lagradost.cloudstream3.utils.*
 import com.lagradost.api.Log
 import com.lagradost.cloudstream3.extractors.helper.JwPlayerHelper
@@ -60,7 +61,7 @@ open class Filesim : ExtractorApi() {
         val embedUrl = url.replace("/download/", "/e/")
         var pageResponse = app.get(embedUrl, referer = referer)
 
-        val iframeElement = pageResponse.document.selectFirst("iframe")
+        val iframeElement = pageResponse.ksoupDocument.selectFirst("iframe")
         if (iframeElement != null) {
             val iframeUrl = iframeElement.attr("src")
             pageResponse = app.get(
@@ -76,7 +77,7 @@ open class Filesim : ExtractorApi() {
         val scriptData = if (!getPacked(pageResponse.text).isNullOrEmpty()) {
             getAndUnpack(pageResponse.text)
         } else {
-            pageResponse.document.selectFirst("script:containsData(sources:)")?.data()
+            pageResponse.ksoupDocument.selectFirst("script:containsData(sources:)")?.data()
         }
 
         val linkFound = JwPlayerHelper.extractStreamLinks(scriptData.orEmpty(), name, mainUrl, callback, subtitleCallback)

library/src/commonMain/kotlin/com/lagradost/cloudstream3/extractors/GamoVideo.kt

@@ -3,6 +3,7 @@ package com.lagradost.cloudstream3.extractors
 import com.lagradost.cloudstream3.SubtitleFile
 import com.lagradost.cloudstream3.app
 import com.lagradost.cloudstream3.extractors.helper.JwPlayerHelper
+import com.lagradost.cloudstream3.ksoupDocument
 import com.lagradost.cloudstream3.utils.ExtractorApi
 import com.lagradost.cloudstream3.utils.ExtractorLink
 
@@ -18,7 +19,7 @@ open class GamoVideo : ExtractorApi() {
         subtitleCallback: (SubtitleFile) -> Unit,
         callback: (ExtractorLink) -> Unit
     ) {
-        app.get(url, referer = referer).document.select("script")
+        app.get(url, referer = referer).ksoupDocument.select("script")
             .firstOrNull { JwPlayerHelper.canParseJwScript(it.data()) }!!.let {
                 JwPlayerHelper.extractStreamLinks(it.data(), name, mainUrl, callback, subtitleCallback)
             }

library/src/commonMain/kotlin/com/lagradost/cloudstream3/extractors/Gdriveplayer.kt

@@ -1,11 +1,11 @@
 package com.lagradost.cloudstream3.extractors
 
 import com.fasterxml.jackson.annotation.JsonProperty
+import com.fleeksoft.ksoup.nodes.Element
 import com.lagradost.cloudstream3.*
 import com.lagradost.cloudstream3.extractors.helper.AesHelper.cryptoAESHandler
 import com.lagradost.cloudstream3.utils.*
 import com.lagradost.cloudstream3.utils.AppUtils.tryParseJson
-import org.jsoup.nodes.Element
 
 class DatabaseGdrive2 : Gdriveplayer() {
     override var mainUrl = "https://databasegdriveplayer.co"
@@ -75,7 +75,7 @@ open class Gdriveplayer : ExtractorApi() {
         subtitleCallback: (SubtitleFile) -> Unit,
         callback: (ExtractorLink) -> Unit
     ) {
-        val document = app.get(url).document
+        val document = app.get(url).ksoupDocument
 
         val eval = unpackJs(document)?.replace("\\", "") ?: return
         val data = Regex("data='(\\S+?)'").first(eval) ?: return

library/src/commonMain/kotlin/com/lagradost/cloudstream3/extractors/GoodstreamExtractor.kt

@@ -2,6 +2,7 @@ package com.lagradost.cloudstream3.extractors
 
 import com.lagradost.cloudstream3.SubtitleFile
 import com.lagradost.cloudstream3.app
+import com.lagradost.cloudstream3.ksoupDocument
 import com.lagradost.cloudstream3.utils.ExtractorApi
 import com.lagradost.cloudstream3.utils.ExtractorLink
 import com.lagradost.cloudstream3.utils.Qualities
@@ -18,7 +19,7 @@ class GoodstreamExtractor : ExtractorApi() {
         subtitleCallback: (SubtitleFile) -> Unit,
         callback: (ExtractorLink) -> Unit
     ) {
-        app.get(url).document.select("script").map { script ->
+        app.get(url).ksoupDocument.select("script").map { script ->
             if (script.data().contains(Regex("file|player"))) {
                 val urlRegex = Regex("file: \"(https:\\/\\/[a-z0-9.\\/-_?=&]+)\",")
                 urlRegex.find(script.data())?.groupValues?.get(1).let { link ->

library/src/commonMain/kotlin/com/lagradost/cloudstream3/extractors/HubCloud.kt

@@ -4,6 +4,7 @@ import com.lagradost.api.Log
 import com.lagradost.cloudstream3.SubtitleFile
 import com.lagradost.cloudstream3.amap
 import com.lagradost.cloudstream3.app
+import com.lagradost.cloudstream3.ksoupDocument
 import com.lagradost.cloudstream3.utils.ExtractorApi
 import com.lagradost.cloudstream3.utils.ExtractorLink
 import com.lagradost.cloudstream3.utils.Qualities
@@ -33,7 +34,7 @@ class HubCloud : ExtractorApi() {
             if ("hubcloud.php" in realUrl) {
                 realUrl
             } else {
-                val rawHref = app.get(realUrl).document.select("#download").attr("href")
+                val rawHref = app.get(realUrl).ksoupDocument.select("#download").attr("href")
                 if (rawHref.startsWith("http", ignoreCase = true)) {
                     rawHref
                 } else {
@@ -50,7 +51,7 @@ class HubCloud : ExtractorApi() {
             return
         }
 
-        val document = app.get(href).document
+        val document = app.get(href).ksoupDocument
         val size = document.selectFirst("i#size")?.text().orEmpty()
         val header = document.selectFirst("div.card-header")?.text().orEmpty()
 

library/src/commonMain/kotlin/com/lagradost/cloudstream3/extractors/Hxfile.kt

@@ -3,6 +3,7 @@ package com.lagradost.cloudstream3.extractors
 import com.lagradost.cloudstream3.SubtitleFile
 import com.lagradost.cloudstream3.app
 import com.lagradost.cloudstream3.extractors.helper.JwPlayerHelper
+import com.lagradost.cloudstream3.ksoupDocument
 import com.lagradost.cloudstream3.utils.*
 
 class Neonime7n : Hxfile() {
@@ -45,7 +46,7 @@ open class Hxfile : ExtractorApi() {
         subtitleCallback: (SubtitleFile) -> Unit,
         callback: (ExtractorLink) -> Unit
     ) {
-        val document = app.get(url, allowRedirects = redirect, referer = referer).document
+        val document = app.get(url, allowRedirects = redirect, referer = referer).ksoupDocument
         with(document) {
             this.select("script").map { script ->
                 if (getPacked(script.data()) != null) {

library/src/commonMain/kotlin/com/lagradost/cloudstream3/extractors/InternetArchive.kt

@@ -1,15 +1,16 @@
 package com.lagradost.cloudstream3.extractors
 
+import com.fleeksoft.ksoup.nodes.Document
 import com.lagradost.cloudstream3.SubtitleFile
 import com.lagradost.cloudstream3.app
 import com.lagradost.cloudstream3.mvvm.logError
 import com.lagradost.cloudstream3.newSubtitleFile
+import com.lagradost.cloudstream3.ksoupDocument
 import com.lagradost.cloudstream3.utils.ExtractorApi
 import com.lagradost.cloudstream3.utils.ExtractorLink
 import com.lagradost.cloudstream3.utils.Qualities
 import com.lagradost.cloudstream3.utils.StringUtils.decodeUri
 import com.lagradost.cloudstream3.utils.newExtractorLink
-import org.jsoup.nodes.Document
 
 open class InternetArchive : ExtractorApi() {
     override val mainUrl = "https://archive.org"
@@ -34,7 +35,7 @@ open class InternetArchive : ExtractorApi() {
     ) {
         val document = archivedItems[url] ?: run {
             try {
-                val doc = app.get(url).document
+                val doc = app.get(url).ksoupDocument
                 archivedItems[url] = doc
                 doc
             } catch (e: Exception) {