Add playlist interceptor

MediaThor-Commons/src/main/resources/logback.xml

@@ -6,11 +6,15 @@
         </encoder>
     </appender>
 
-    <root level="trace">
+    <root name="io.github.freya022" level="trace" additivity="false">
         <appender-ref ref="STDOUT"/>
     </root>
 
     <root name="io.github.freya022.mediathor.record.memfs" level="debug" additivity="false">
         <appender-ref ref="STDOUT"/>
     </root>
+
+    <root level="info">
+        <appender-ref ref="STDOUT"/>
+    </root>
 </configuration>

MediaThor-Interceptor/.gitignore

@@ -0,0 +1 @@
+/master_playlists.txt

MediaThor-Interceptor/README.md

@@ -0,0 +1,9 @@
+# MediaThor-Interceptor
+
+A utility to intercept HLS stream requests and append them to a list
+
+## Limitations
+* Master playlists won't be distinguished from media playlists
+
+## Requirements
+* Chrome on Linux/Mac (Optional), or Chromium-based Edge on Windows

MediaThor-Interceptor/pom.xml

@@ -0,0 +1,80 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>io.github.freya022</groupId>
+        <artifactId>mediathor</artifactId>
+        <version>1.0</version>
+    </parent>
+
+    <artifactId>mediathor-interceptor</artifactId>
+    <name>MediaThor-Interceptor</name>
+
+    <properties>
+        <mainClass>io.github.freya022.mediathor.interceptor.InterceptorMain</mainClass>
+    </properties>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-shade-plugin</artifactId>
+                <configuration>
+                    <transformers>
+                        <transformer
+                                implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+                            <mainClass>${mainClass}</mainClass>
+                        </transformer>
+                    </transformers>
+
+                    <createDependencyReducedPom>false</createDependencyReducedPom>
+                    <finalName>${project.name}</finalName>
+                </configuration>
+            </plugin>
+        </plugins>
+    </build>
+
+    <dependencies>
+        <dependency>
+            <groupId>io.github.freya022</groupId>
+            <artifactId>mediathor-commons</artifactId>
+            <version>1.0</version>
+        </dependency>
+        <dependency>
+            <groupId>com.google.code.gson</groupId>
+            <artifactId>gson</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.squareup.okhttp3</groupId>
+            <artifactId>okhttp</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.seleniumhq.selenium</groupId>
+            <artifactId>selenium-java</artifactId>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.seleniumhq.selenium</groupId>
+                    <artifactId>selenium-devtools-v85</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.seleniumhq.selenium</groupId>
+                    <artifactId>selenium-devtools-v114</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.seleniumhq.selenium</groupId>
+                    <artifactId>selenium-devtools-v115</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.seleniumhq.selenium</groupId>
+                    <artifactId>selenium-firefox-driver</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.seleniumhq.selenium</groupId>
+                    <artifactId>selenium-ie-driver</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+    </dependencies>
+</project>

MediaThor-Interceptor/src/main/kotlin/io/github/freya022/mediathor/interceptor/Interceptor.kt

@@ -0,0 +1,94 @@
+package io.github.freya022.mediathor.interceptor
+
+import mu.two.KotlinLogging
+import okhttp3.HttpUrl.Companion.toHttpUrl
+import org.openqa.selenium.Platform
+import org.openqa.selenium.chrome.ChromeDriver
+import org.openqa.selenium.devtools.v116.network.Network
+import org.openqa.selenium.devtools.v116.target.Target
+import org.openqa.selenium.edge.EdgeDriver
+import org.openqa.selenium.remote.DriverCommand
+import org.openqa.selenium.remote.RemoteWebDriver
+import java.nio.file.StandardOpenOption
+import java.util.*
+import java.util.logging.Level
+import kotlin.coroutines.Continuation
+import kotlin.coroutines.resume
+import kotlin.coroutines.suspendCoroutine
+import kotlin.io.path.Path
+import kotlin.io.path.bufferedWriter
+import kotlin.jvm.optionals.getOrNull
+
+private val logger = KotlinLogging.logger { }
+
+class Interceptor private constructor() {
+    private val writer = outputPath.bufferedWriter(options = arrayOf(StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING))
+
+    private lateinit var continuation: Continuation<Unit>
+
+    private val driver = when (Platform.getCurrent().family()) {
+        Platform.WINDOWS -> EdgeDriver()
+        else -> ChromeDriver()
+    }
+
+    private val hook = Thread {
+        driver.maybeGetDevTools().getOrNull()?.clearListeners()
+        driver.quit()
+        close()
+    }
+
+    init {
+        driver.setLogLevel(Level.INFO)
+        Runtime.getRuntime().addShutdownHook(hook)
+
+        val devTools = driver.devTools
+        devTools.createSession()
+        devTools.send(Network.enable(Optional.empty(), Optional.empty(), Optional.empty()))
+
+        devTools.addListener(Network.requestWillBeSent()) { futureRequest ->
+            val url = futureRequest.request.url
+            if (!url.startsWith("http")) {
+                logger.trace { "Discarding $url as it is not an http url" }
+                return@addListener
+            }
+
+            if (!url.toHttpUrl().pathSegments.last().endsWith(".m3u8")) {
+                logger.trace { "Discarding $url as it is not a '*.m3u8' url" }
+                return@addListener
+            }
+
+            logger.info { "Added $url" }
+            writer.appendLine(url)
+        }
+
+        devTools.addListener(Target.detachedFromTarget()) {
+            logger.info { "Exiting" }
+            Runtime.getRuntime().removeShutdownHook(hook)
+            continuation.resume(Unit)
+        }
+    }
+
+    private suspend fun awaitTermination(): Unit = suspendCoroutine {
+        continuation = it
+    }
+
+    // This method is only used when the browser is already closed.
+    private fun close() {
+        writer.close()
+    }
+
+    companion object {
+        val outputPath = Path("master_playlists.txt")
+
+        suspend fun intercept() {
+            val interceptor = Interceptor()
+            interceptor.awaitTermination()
+            // If the browser is closed, the driver is still running
+            // Using the QUIT command makes it exit
+            val executeMethod = RemoteWebDriver::class.java.getDeclaredMethod("execute", String::class.java)
+            executeMethod.isAccessible = true
+            executeMethod.invoke(interceptor.driver, DriverCommand.QUIT)
+            interceptor.close()
+        }
+    }
+}

MediaThor-Interceptor/src/main/kotlin/io/github/freya022/mediathor/interceptor/InterceptorMain.kt

@@ -0,0 +1,18 @@
+package io.github.freya022.mediathor.interceptor
+
+import kotlinx.coroutines.runBlocking
+import java.io.IOException
+import kotlin.io.path.absolutePathString
+import kotlin.io.path.exists
+
+object InterceptorMain {
+    @JvmStatic
+    fun main(args: Array<String>) {
+        if (Interceptor.outputPath.exists() && "-y" !in args)
+            throw IOException("${Interceptor.outputPath.absolutePathString()} already exists")
+
+        runBlocking {
+            Interceptor.intercept()
+        }
+    }
+}

README.md

@@ -6,5 +6,6 @@ These utilities are focused on specific use cases, the goal is NOT a swiss army
 
 ## Subprojects
 * [MediaThor-DL](MediaThor-DL): A downloader for HLS streams
+* [MediaThor-Interceptor](MediaThor-Interceptor): Writes *.m3u8 files in a list from browser usage
 * [MediaThor-Volume](MediaThor-Volume): Helps you manage the volume of your music files
 * [MediaThor-Record-Helper](MediaThor-Record-Helper): Merges overlapping clips using a temporary ramdisk

pom.xml

@@ -14,6 +14,7 @@
         <module>MediaThor-Volume</module>
         <module>MediaThor-Record-Helper</module>
         <module>MediaThor-Commons</module>
+        <module>MediaThor-Interceptor</module>
     </modules>
 
     <name>MediaThor</name>
@@ -210,6 +211,11 @@
                 <artifactId>jnr-winfsp</artifactId>
                 <version>c7be8901e8</version>
             </dependency>
+            <dependency>
+                <groupId>org.seleniumhq.selenium</groupId>
+                <artifactId>selenium-java</artifactId>
+                <version>4.12.0</version>
+            </dependency>
         </dependencies>
     </dependencyManagement>