diff --git a/checkstyle/checkstyle.xml b/checkstyle/checkstyle.xml
index 3b5825a3be..822e0688de 100644
--- a/checkstyle/checkstyle.xml
+++ b/checkstyle/checkstyle.xml
@@ -192,4 +192,9 @@
+
+
+
+
+
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/ClientsConstants.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/ClientsConstants.java
new file mode 100644
index 0000000000..41aac33b3e
--- /dev/null
+++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/ClientsConstants.java
@@ -0,0 +1,114 @@
+package org.schabi.newpipe.extractor.services.youtube;
+
+final class ClientsConstants {
+ private ClientsConstants() {
+ }
+
+ // Common client fields
+
+ static final String DESKTOP_CLIENT_PLATFORM = "DESKTOP";
+ static final String MOBILE_CLIENT_PLATFORM = "MOBILE";
+ static final String WATCH_CLIENT_SCREEN = "WATCH";
+ static final String EMBED_CLIENT_SCREEN = "EMBED";
+
+ // WEB (YouTube desktop) client fields
+
+ static final String WEB_CLIENT_ID = "1";
+ static final String WEB_CLIENT_NAME = "WEB";
+ /**
+ * The client version for InnerTube requests with the {@code WEB} client, used as the last
+ * fallback if the extraction of the real one failed.
+ */
+ static final String WEB_HARDCODED_CLIENT_VERSION = "2.20250122.04.00";
+
+ // WEB_REMIX (YouTube Music) client fields
+
+ static final String WEB_REMIX_CLIENT_ID = "67";
+ static final String WEB_REMIX_CLIENT_NAME = "WEB_REMIX";
+ static final String WEB_REMIX_HARDCODED_CLIENT_VERSION = "1.20250122.01.00";
+
+ // TVHTML5 (YouTube web on TV and consoles) client fields
+ static final String TVHTML5_CLIENT_ID = "7";
+ static final String TVHTML5_CLIENT_NAME = "TVHTML5";
+ static final String TVHTML5_CLIENT_VERSION = "7.20250122.15.00";
+ static final String TVHTML5_CLIENT_PLATFORM = "CONSOLE";
+ // CHECKSTYLE:OFF
+ static final String TVHTML5_USER_AGENT =
+ "Mozilla/5.0 (PlayStation; PlayStation 4/8.50) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.0 Safari/605.1.15";
+ // CHECKSTYLE:ON
+
+ // WEB_EMBEDDED_PLAYER (YouTube embeds)
+
+ static final String WEB_EMBEDDED_CLIENT_ID = "56";
+ static final String WEB_EMBEDDED_CLIENT_NAME = "WEB_EMBEDDED_PLAYER";
+ static final String WEB_EMBEDDED_CLIENT_VERSION = "1.20250121.00.00";
+
+ // IOS (iOS YouTube app) client fields
+
+ static final String IOS_CLIENT_NAME = "IOS";
+
+ /**
+ * The hardcoded client version of the iOS app used for InnerTube requests with this client.
+ *
+ *
+ * It can be extracted by getting the latest release version of the app on
+ * the App
+ * Store page of the YouTube app, in the {@code What’s New} section.
+ *
+ */
+ static final String IOS_CLIENT_VERSION = "19.28.1";
+
+ /**
+ * The device machine id for the iPhone 15 Pro Max, used to get 60fps with the {@code iOS}
+ * client.
+ *
+ *
+ * See this GitHub Gist for more
+ * information.
+ *
+ */
+ static final String IOS_DEVICE_MODEL = "iPhone16,2";
+
+ /**
+ * The iOS version to be used in JSON POST requests, the one of an iPhone 15 Pro Max running
+ * iOS 18.2.1 with the hardcoded version of the iOS app (for the {@code "osVersion"} field).
+ *
+ *
+ * The value of this field seems to use the following structure:
+ * "iOS major version.minor version.patch version.build version", where
+ * "patch version" is equal to 0 if it isn't set
+ * The build version corresponding to the iOS version used can be found on
+ *
+ * https://theapplewiki.com/wiki/Firmware/iPhone/18.x#iPhone_15_Pro_Max
+ *
+ *
+ * @see #IOS_USER_AGENT_VERSION
+ */
+ static final String IOS_OS_VERSION = "18.2.1.22C161";
+
+ /**
+ * The iOS version to be used in the HTTP user agent for requests.
+ *
+ *
+ * This should be the same of as {@link #IOS_OS_VERSION}.
+ *
+ *
+ * @see #IOS_OS_VERSION
+ */
+ static final String IOS_USER_AGENT_VERSION = "18_2_1";
+
+ // ANDROID (Android YouTube app) client fields
+
+ static final String ANDROID_CLIENT_NAME = "ANDROID";
+
+ /**
+ * The hardcoded client version of the Android app used for InnerTube requests with this
+ * client.
+ *
+ *
+ * It can be extracted by getting the latest release version of the app in an APK repository
+ * such as APKMirror.
+ *
+ */
+ static final String ANDROID_CLIENT_VERSION = "19.28.35";
+}
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/PoTokenProvider.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/PoTokenProvider.java
new file mode 100644
index 0000000000..2d1e4a0d7f
--- /dev/null
+++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/PoTokenProvider.java
@@ -0,0 +1,113 @@
+package org.schabi.newpipe.extractor.services.youtube;
+
+import javax.annotation.Nullable;
+
+/**
+ * An interface to provide poTokens to YouTube player requests.
+ *
+ *
+ *
+ * On some major clients, YouTube requires that the integrity of the device passes some checks to
+ * allow playback.
+ *
+ *
+ *
+ * These checks involve running codes to verify the integrity and using their result to generate a
+ * poToken (which likely stands for proof of origin token), using a visitor data ID for logged-out
+ * users.
+ *
+ *
+ *
+ * These tokens may have a role in triggering the sign in requirement.
+ *
+ *
+ *
+ * Implementations of this interface are expected to be thread-safe, as they may be accessed by
+ * multiple threads.
+ *
+ */
+public interface PoTokenProvider {
+
+ /**
+ * Get a {@link PoTokenResult} specific to the desktop website, a.k.a. the WEB InnerTube client.
+ *
+ *
+ * To be generated and valid, poTokens from this client must be generated using Google's
+ * BotGuard machine, which requires a JavaScript engine with a good DOM implementation. They
+ * must be added to adaptive/DASH streaming URLs with the {@code pot} parameter.
+ *
+ *
+ *
+ * Note that YouTube desktop website generates two poTokens:
+ * - one for the player requests poTokens, using the videoId as the minter value;
+ * - one for the streaming URLs, using a visitor data for logged-out users.
+ *
+ *
+ * @return a {@link PoTokenResult} specific to the WEB InnerTube client
+ */
+ @Nullable
+ PoTokenResult getWebClientPoToken(String videoId);
+
+ /**
+ * Get a {@link PoTokenResult} specific to the web embeds, a.k.a. the WEB_EMBEDDED_PLAYER
+ * InnerTube client.
+ *
+ *
+ * To be generated and valid, poTokens from this client must be generated using Google's
+ * BotGuard machine, which requires a JavaScript engine with a good DOM implementation. They
+ * should be added to adaptive/DASH streaming URLs with the {@code pot} parameter and do not
+ * seem to be mandatory for now.
+ *
+ *
+ *
+ * As of writing, like the YouTube desktop website previously did, it generates only one
+ * poToken, sent in player requests and streaming URLs, using a visitor data for logged-out
+ * users.
+ *
+ *
+ * @return a {@link PoTokenResult} specific to the WEB_EMBEDDED_PLAYER InnerTube client
+ */
+ @Nullable
+ PoTokenResult getWebEmbedClientPoToken(String videoId);
+
+ /**
+ * Get a {@link PoTokenResult} specific to the Android app, a.k.a. the ANDROID InnerTube client.
+ *
+ *
+ * Implementation details are not known, the app uses DroidGuard, a native virtual machine
+ * ran by Google Play Services for which its code is updated pretty frequently.
+ *
+ *
+ *
+ * As of writing, DroidGuard seem to check for the Android app signature and package ID, as
+ * unrooted YouTube patched with reVanced doesn't work without spoofing another InnerTube
+ * client while the rooted version works without any client spoofing.
+ *
+ *
+ *
+ * There should be only poToken needed, for the player requests.
+ *
+ *
+ * @return a {@link PoTokenResult} specific to the ANDROID InnerTube client
+ */
+ @Nullable
+ PoTokenResult getAndroidClientPoToken(String videoId);
+
+ /**
+ * Get a {@link PoTokenResult} specific to the Android app, a.k.a. the ANDROID InnerTube client.
+ *
+ *
+ * Implementation details are not really known, the app seem to use something called
+ * iosGuard which should be something similar to Android's DroidGuard. It may rely on Apple's
+ * attestation APIs.
+ *
+ *
+ *
+ * There should be only poToken needed, for the player requests.
+ *
+ *
+ * @return a {@link PoTokenResult} specific to the IOS InnerTube client
+ */
+ @Nullable
+ PoTokenResult getIosClientPoToken(String videoId);
+}
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/PoTokenResult.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/PoTokenResult.java
new file mode 100644
index 0000000000..aa21e74324
--- /dev/null
+++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/PoTokenResult.java
@@ -0,0 +1,40 @@
+package org.schabi.newpipe.extractor.services.youtube;
+
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import java.util.Objects;
+
+public final class PoTokenResult {
+
+ /**
+ * The visitor data associated with a {@code poToken}.
+ */
+ @Nonnull
+ public final String visitorData;
+
+ /**
+ * The {@code poToken} of a player request, a Protobuf object encoded as a base 64 string.
+ */
+ @Nonnull
+ public final String playerRequestPoToken;
+
+ /**
+ * The {@code poToken} to be appended to streaming URLs, a Protobuf object encoded as a base
+ * 64 string.
+ *
+ *
+ * It may be required on some clients such as HTML5 ones and may also differ from the player
+ * request {@code poToken}.
+ *
+ */
+ @Nullable
+ public final String streamingDataPoToken;
+
+ public PoTokenResult(@Nonnull final String visitorData,
+ @Nonnull final String playerRequestPoToken,
+ @Nullable final String streamingDataPoToken) {
+ this.visitorData = Objects.requireNonNull(visitorData);
+ this.playerRequestPoToken = Objects.requireNonNull(playerRequestPoToken);
+ this.streamingDataPoToken = streamingDataPoToken;
+ }
+}
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java
index 0cc9391be6..6b87425319 100644
--- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java
+++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java
@@ -21,6 +21,17 @@
package org.schabi.newpipe.extractor.services.youtube;
import static org.schabi.newpipe.extractor.NewPipe.getDownloader;
+import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.ANDROID_CLIENT_VERSION;
+import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.DESKTOP_CLIENT_PLATFORM;
+import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_CLIENT_VERSION;
+import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_DEVICE_MODEL;
+import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_USER_AGENT_VERSION;
+import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_CLIENT_ID;
+import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_CLIENT_NAME;
+import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_HARDCODED_CLIENT_VERSION;
+import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_REMIX_CLIENT_ID;
+import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_REMIX_CLIENT_NAME;
+import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_REMIX_HARDCODED_CLIENT_VERSION;
import static org.schabi.newpipe.extractor.utils.Utils.HTTP;
import static org.schabi.newpipe.extractor.utils.Utils.HTTPS;
import static org.schabi.newpipe.extractor.utils.Utils.getStringResultFromRegexArray;
@@ -144,55 +155,6 @@ private YoutubeParsingHelper() {
*/
public static final String RACY_CHECK_OK = "racyCheckOk";
- /**
- * The hardcoded client ID used for InnerTube requests with the {@code WEB} client.
- */
- private static final String WEB_CLIENT_ID = "1";
-
- /**
- * The client version for InnerTube requests with the {@code WEB} client, used as the last
- * fallback if the extraction of the real one failed.
- */
- private static final String HARDCODED_CLIENT_VERSION = "2.20240718.01.00";
-
- /**
- * The hardcoded client version of the Android app used for InnerTube requests with this
- * client.
- *
- *
- * It can be extracted by getting the latest release version of the app in an APK repository
- * such as APKMirror.
- *
- */
- private static final String ANDROID_YOUTUBE_CLIENT_VERSION = "19.28.35";
-
- /**
- * The hardcoded client version of the iOS app used for InnerTube requests with this client.
- *
- *
- * It can be extracted by getting the latest release version of the app on
- * the App
- * Store page of the YouTube app, in the {@code What’s New} section.
- *
- */
- private static final String IOS_YOUTUBE_CLIENT_VERSION = "19.45.4";
-
- /**
- * The hardcoded client version used for InnerTube requests with the TV HTML5 embed client.
- */
- private static final String TVHTML5_SIMPLY_EMBED_CLIENT_VERSION = "2.0";
-
- /**
- * The hardcoded client ID used for InnerTube requests with the YouTube Music desktop client.
- */
- private static final String YOUTUBE_MUSIC_CLIENT_ID = "67";
-
- /**
- * The hardcoded client version used for InnerTube requests with the YouTube Music desktop
- * client.
- */
- private static final String HARDCODED_YOUTUBE_MUSIC_CLIENT_VERSION = "1.20240715.01.00";
-
private static String clientVersion;
private static String youtubeMusicClientVersion;
@@ -212,41 +174,6 @@ private YoutubeParsingHelper() {
private static final String CONTENT_PLAYBACK_NONCE_ALPHABET =
"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_";
- /**
- * The device machine id for the iPhone 15 Pro Max,
- * used to get 60fps with the {@code iOS} client.
- *
- *
- * See this GitHub Gist for more
- * information.
- *
- */
- private static final String IOS_DEVICE_MODEL = "iPhone16,2";
-
- /**
- * Spoofing an iPhone 15 Pro Max running iOS 18.1.0 with the hardcoded version of the iOS app.
- * To be used for the {@code "osVersion"} field in JSON POST requests.
- *
- * The value of this field seems to use the following structure:
- * "iOS major version.minor version.patch version.build version", where
- * "patch version" is equal to 0 if it isn't set
- * The build version corresponding to the iOS version used can be found on
- *
- * https://theapplewiki.com/wiki/Firmware/iPhone/18.x#iPhone_15_Pro_Max
- *
- *
- * @see #IOS_USER_AGENT_VERSION
- */
- private static final String IOS_OS_VERSION = "18.1.0.22B83";
-
- /**
- * Spoofing an iPhone 15 Pro Max running iOS 18.1.0 with the hardcoded version of the iOS app.
- * To be used in the user agent for requests.
- *
- * @see #IOS_OS_VERSION
- */
- private static final String IOS_USER_AGENT_VERSION = "18_1_0";
-
private static Random numberGenerator = new Random();
private static final String FEED_BASE_CHANNEL_ID =
@@ -561,9 +488,9 @@ public static boolean isHardcodedClientVersionValid()
.object("client")
.value("hl", "en-GB")
.value("gl", "GB")
- .value("clientName", "WEB")
- .value("clientVersion", HARDCODED_CLIENT_VERSION)
- .value("platform", "DESKTOP")
+ .value("clientName", WEB_CLIENT_NAME)
+ .value("clientVersion", WEB_HARDCODED_CLIENT_VERSION)
+ .value("platform", DESKTOP_CLIENT_PLATFORM)
.value("utcOffsetMinutes", 0)
.end()
.object("request")
@@ -581,7 +508,7 @@ public static boolean isHardcodedClientVersionValid()
.end().done().getBytes(StandardCharsets.UTF_8);
// @formatter:on
- final var headers = getClientHeaders(WEB_CLIENT_ID, HARDCODED_CLIENT_VERSION);
+ final var headers = getClientHeaders(WEB_CLIENT_ID, WEB_HARDCODED_CLIENT_VERSION);
// This endpoint is fetched by the YouTube website to get the items of its main menu and is
// pretty lightweight (around 30kB)
@@ -705,7 +632,7 @@ public static String getClientVersion() throws IOException, ExtractionException
// Fallback to the hardcoded one if it is valid
if (isHardcodedClientVersionValid()) {
- clientVersion = HARDCODED_CLIENT_VERSION;
+ clientVersion = WEB_HARDCODED_CLIENT_VERSION;
return clientVersion;
}
@@ -752,11 +679,11 @@ public static boolean isHardcodedYoutubeMusicClientVersionValid() throws IOExcep
.object()
.object("context")
.object("client")
- .value("clientName", "WEB_REMIX")
- .value("clientVersion", HARDCODED_YOUTUBE_MUSIC_CLIENT_VERSION)
+ .value("clientName", WEB_REMIX_CLIENT_NAME)
+ .value("clientVersion", WEB_REMIX_HARDCODED_CLIENT_VERSION)
.value("hl", "en-GB")
.value("gl", "GB")
- .value("platform", "DESKTOP")
+ .value("platform", DESKTOP_CLIENT_PLATFORM)
.value("utcOffsetMinutes", 0)
.end()
.object("request")
@@ -775,8 +702,7 @@ public static boolean isHardcodedYoutubeMusicClientVersionValid() throws IOExcep
// @formatter:on
final var headers = new HashMap<>(getOriginReferrerHeaders(YOUTUBE_MUSIC_URL));
- headers.putAll(getClientHeaders(YOUTUBE_MUSIC_CLIENT_ID,
- HARDCODED_YOUTUBE_MUSIC_CLIENT_VERSION));
+ headers.putAll(getClientHeaders(WEB_REMIX_CLIENT_ID, WEB_HARDCODED_CLIENT_VERSION));
final Response response = getDownloader().postWithContentTypeJson(url, headers, json);
// Ensure to have a valid response
@@ -789,7 +715,7 @@ public static String getYoutubeMusicClientVersion()
return youtubeMusicClientVersion;
}
if (isHardcodedYoutubeMusicClientVersionValid()) {
- youtubeMusicClientVersion = HARDCODED_YOUTUBE_MUSIC_CLIENT_VERSION;
+ youtubeMusicClientVersion = WEB_REMIX_HARDCODED_CLIENT_VERSION;
return youtubeMusicClientVersion;
}
@@ -1134,43 +1060,6 @@ public static JsonObject getJsonPostResponse(final String endpoint,
+ DISABLE_PRETTY_PRINT_PARAMETER, headers, body, localization)));
}
- public static JsonObject getJsonAndroidPostResponse(
- final String endpoint,
- final byte[] body,
- @Nonnull final Localization localization,
- @Nullable final String endPartOfUrlRequest) throws IOException, ExtractionException {
- return getMobilePostResponse(endpoint, body, localization,
- getAndroidUserAgent(localization), endPartOfUrlRequest);
- }
-
- public static JsonObject getJsonIosPostResponse(
- final String endpoint,
- final byte[] body,
- @Nonnull final Localization localization,
- @Nullable final String endPartOfUrlRequest) throws IOException, ExtractionException {
- return getMobilePostResponse(endpoint, body, localization, getIosUserAgent(localization),
- endPartOfUrlRequest);
- }
-
- private static JsonObject getMobilePostResponse(
- final String endpoint,
- final byte[] body,
- @Nonnull final Localization localization,
- @Nonnull final String userAgent,
- @Nullable final String endPartOfUrlRequest) throws IOException, ExtractionException {
- final var headers = Map.of("User-Agent", List.of(userAgent),
- "X-Goog-Api-Format-Version", List.of("2"));
-
- final String baseEndpointUrl = YOUTUBEI_V1_GAPIS_URL + endpoint + "?"
- + DISABLE_PRETTY_PRINT_PARAMETER;
-
- return JsonUtils.toJsonObject(getValidJsonResponseBody(
- getDownloader().postWithContentTypeJson(isNullOrEmpty(endPartOfUrlRequest)
- ? baseEndpointUrl
- : baseEndpointUrl + endPartOfUrlRequest,
- headers, body, localization)));
- }
-
@Nonnull
public static JsonBuilder prepareDesktopJsonBuilder(
@Nonnull final Localization localization,
@@ -1196,10 +1085,10 @@ public static JsonBuilder prepareDesktopJsonBuilder(
.object("client")
.value("hl", localization.getLocalizationCode())
.value("gl", contentCountry.getCountryCode())
- .value("clientName", "WEB")
+ .value("clientName", WEB_CLIENT_NAME)
.value("clientVersion", getClientVersion())
.value("originalUrl", "https://www.youtube.com")
- .value("platform", "DESKTOP")
+ .value("platform", DESKTOP_CLIENT_PLATFORM)
.value("utcOffsetMinutes", 0)
.value("visitorData", vData)
.end()
@@ -1217,165 +1106,6 @@ public static JsonBuilder prepareDesktopJsonBuilder(
// @formatter:on
}
- @Nonnull
- public static JsonBuilder prepareAndroidMobileJsonBuilder(
- @Nonnull final Localization localization,
- @Nonnull final ContentCountry contentCountry) {
- // @formatter:off
- return JsonObject.builder()
- .object("context")
- .object("client")
- .value("clientName", "ANDROID")
- .value("clientVersion", ANDROID_YOUTUBE_CLIENT_VERSION)
- .value("platform", "MOBILE")
- .value("osName", "Android")
- .value("osVersion", "14")
- /*
- A valid Android SDK version is required to be sure to get a valid player
- response
- If this parameter is not provided, the player response is replaced by an
- error saying the message "The following content is not available on this
- app. Watch this content on the latest version on YouTube" (it was
- previously a 5-minute video with this message)
- See https://github.com/TeamNewPipe/NewPipe/issues/8713
- The Android SDK version corresponding to the Android version used in
- requests is sent
- */
- .value("androidSdkVersion", 34)
- .value("hl", localization.getLocalizationCode())
- .value("gl", contentCountry.getCountryCode())
- .value("utcOffsetMinutes", 0)
- .end()
- .object("request")
- .array("internalExperimentFlags")
- .end()
- .value("useSsl", true)
- .end()
- .object("user")
- // TODO: provide a way to enable restricted mode with:
- // .value("enableSafetyMode", boolean)
- .value("lockedSafetyMode", false)
- .end()
- .end();
- // @formatter:on
- }
-
- @Nonnull
- public static JsonBuilder prepareIosMobileJsonBuilder(
- @Nonnull final Localization localization,
- @Nonnull final ContentCountry contentCountry) {
- // @formatter:off
- return JsonObject.builder()
- .object("context")
- .object("client")
- .value("clientName", "IOS")
- .value("clientVersion", IOS_YOUTUBE_CLIENT_VERSION)
- .value("deviceMake", "Apple")
- // Device model is required to get 60fps streams
- .value("deviceModel", IOS_DEVICE_MODEL)
- .value("platform", "MOBILE")
- .value("osName", "iOS")
- .value("osVersion", IOS_OS_VERSION)
- .value("visitorData", randomVisitorData(contentCountry))
- .value("hl", localization.getLocalizationCode())
- .value("gl", contentCountry.getCountryCode())
- .value("utcOffsetMinutes", 0)
- .end()
- .object("request")
- .array("internalExperimentFlags")
- .end()
- .value("useSsl", true)
- .end()
- .object("user")
- // TODO: provide a way to enable restricted mode with:
- // .value("enableSafetyMode", boolean)
- .value("lockedSafetyMode", false)
- .end()
- .end();
- // @formatter:on
- }
-
- @Nonnull
- public static JsonBuilder prepareTvHtml5EmbedJsonBuilder(
- @Nonnull final Localization localization,
- @Nonnull final ContentCountry contentCountry,
- @Nonnull final String videoId) {
- // @formatter:off
- return JsonObject.builder()
- .object("context")
- .object("client")
- .value("clientName", "TVHTML5_SIMPLY_EMBEDDED_PLAYER")
- .value("clientVersion", TVHTML5_SIMPLY_EMBED_CLIENT_VERSION)
- .value("clientScreen", "EMBED")
- .value("platform", "TV")
- .value("hl", localization.getLocalizationCode())
- .value("gl", contentCountry.getCountryCode())
- .value("utcOffsetMinutes", 0)
- .end()
- .object("thirdParty")
- .value("embedUrl", "https://www.youtube.com/watch?v=" + videoId)
- .end()
- .object("request")
- .array("internalExperimentFlags")
- .end()
- .value("useSsl", true)
- .end()
- .object("user")
- // TODO: provide a way to enable restricted mode with:
- // .value("enableSafetyMode", boolean)
- .value("lockedSafetyMode", false)
- .end()
- .end();
- // @formatter:on
- }
-
- @Nonnull
- public static JsonObject getWebPlayerResponse(
- @Nonnull final Localization localization,
- @Nonnull final ContentCountry contentCountry,
- @Nonnull final String videoId) throws IOException, ExtractionException {
- final byte[] body = JsonWriter.string(
- prepareDesktopJsonBuilder(localization, contentCountry)
- .value(VIDEO_ID, videoId)
- .value(CONTENT_CHECK_OK, true)
- .value(RACY_CHECK_OK, true)
- .done())
- .getBytes(StandardCharsets.UTF_8);
- final String url = YOUTUBEI_V1_URL + "player" + "?" + DISABLE_PRETTY_PRINT_PARAMETER
- + "&$fields=microformat,playabilityStatus,storyboards,videoDetails";
-
- return JsonUtils.toJsonObject(getValidJsonResponseBody(
- getDownloader().postWithContentTypeJson(
- url, getYouTubeHeaders(), body, localization)));
- }
-
- @Nonnull
- public static byte[] createTvHtml5EmbedPlayerBody(
- @Nonnull final Localization localization,
- @Nonnull final ContentCountry contentCountry,
- @Nonnull final String videoId,
- @Nonnull final Integer sts,
- @Nonnull final String contentPlaybackNonce) {
- // @formatter:off
- return JsonWriter.string(
- prepareTvHtml5EmbedJsonBuilder(localization, contentCountry, videoId)
- .object("playbackContext")
- .object("contentPlaybackContext")
- // Signature timestamp from the JavaScript base player is needed to get
- // working obfuscated URLs
- .value("signatureTimestamp", sts)
- .value("referer", "https://www.youtube.com/watch?v=" + videoId)
- .end()
- .end()
- .value(CPN, contentPlaybackNonce)
- .value(VIDEO_ID, videoId)
- .value(CONTENT_CHECK_OK, true)
- .value(RACY_CHECK_OK, true)
- .done())
- .getBytes(StandardCharsets.UTF_8);
- // @formatter:on
- }
-
/**
* Get the user-agent string used as the user-agent for InnerTube requests with the Android
* client.
@@ -1391,9 +1121,8 @@ public static byte[] createTvHtml5EmbedPlayerBody(
*/
@Nonnull
public static String getAndroidUserAgent(@Nullable final Localization localization) {
- // Spoofing an Android 14 device with the hardcoded version of the Android app
- return "com.google.android.youtube/" + ANDROID_YOUTUBE_CLIENT_VERSION
- + " (Linux; U; Android 14; "
+ return "com.google.android.youtube/" + ANDROID_CLIENT_VERSION
+ + " (Linux; U; Android 15; "
+ (localization != null ? localization : Localization.DEFAULT).getCountryCode()
+ ") gzip";
}
@@ -1413,11 +1142,8 @@ public static String getAndroidUserAgent(@Nullable final Localization localizati
*/
@Nonnull
public static String getIosUserAgent(@Nullable final Localization localization) {
- // Spoofing an iPhone 15 Pro Max running iOS 18.1.0
- // with the hardcoded version of the iOS app
- return "com.google.ios.youtube/" + IOS_YOUTUBE_CLIENT_VERSION
- + "(" + IOS_DEVICE_MODEL + "; U; CPU iOS "
- + IOS_USER_AGENT_VERSION + " like Mac OS X; "
+ return "com.google.ios.youtube/" + IOS_CLIENT_VERSION + "(" + IOS_DEVICE_MODEL
+ + "; U; CPU iOS " + IOS_USER_AGENT_VERSION + " like Mac OS X; "
+ (localization != null ? localization : Localization.DEFAULT).getCountryCode()
+ ")";
}
@@ -1428,8 +1154,7 @@ public static String getIosUserAgent(@Nullable final Localization localization)
@Nonnull
public static Map> getYoutubeMusicHeaders() {
final var headers = new HashMap<>(getOriginReferrerHeaders(YOUTUBE_MUSIC_URL));
- headers.putAll(getClientHeaders(YOUTUBE_MUSIC_CLIENT_ID,
- youtubeMusicClientVersion));
+ headers.putAll(getClientHeaders(WEB_REMIX_CLIENT_ID, youtubeMusicClientVersion));
return headers;
}
@@ -1461,7 +1186,7 @@ public static Map> getClientInfoHeaders()
*
* @param url The URL to be set as the origin and referrer.
*/
- private static Map> getOriginReferrerHeaders(@Nonnull final String url) {
+ static Map> getOriginReferrerHeaders(@Nonnull final String url) {
final var urlList = List.of(url);
return Map.of("Origin", urlList, "Referer", urlList);
}
@@ -1473,8 +1198,8 @@ private static Map> getOriginReferrerHeaders(@Nonnull final
* @param name The X-YouTube-Client-Name value.
* @param version X-YouTube-Client-Version value.
*/
- private static Map> getClientHeaders(@Nonnull final String name,
- @Nonnull final String version) {
+ static Map> getClientHeaders(@Nonnull final String name,
+ @Nonnull final String version) {
return Map.of("X-YouTube-Client-Name", List.of(name),
"X-YouTube-Client-Version", List.of(version));
}
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamHelper.java
new file mode 100644
index 0000000000..5b4276e628
--- /dev/null
+++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamHelper.java
@@ -0,0 +1,470 @@
+package org.schabi.newpipe.extractor.services.youtube;
+
+import com.grack.nanojson.JsonBuilder;
+import com.grack.nanojson.JsonObject;
+import com.grack.nanojson.JsonWriter;
+import org.schabi.newpipe.extractor.exceptions.ExtractionException;
+import org.schabi.newpipe.extractor.localization.ContentCountry;
+import org.schabi.newpipe.extractor.localization.Localization;
+import org.schabi.newpipe.extractor.utils.JsonUtils;
+
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.schabi.newpipe.extractor.NewPipe.getDownloader;
+import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.ANDROID_CLIENT_NAME;
+import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.ANDROID_CLIENT_VERSION;
+import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.DESKTOP_CLIENT_PLATFORM;
+import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.EMBED_CLIENT_SCREEN;
+import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_CLIENT_NAME;
+import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_CLIENT_VERSION;
+import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_DEVICE_MODEL;
+import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.IOS_OS_VERSION;
+import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.MOBILE_CLIENT_PLATFORM;
+import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.TVHTML5_CLIENT_ID;
+import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.TVHTML5_CLIENT_NAME;
+import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.TVHTML5_CLIENT_PLATFORM;
+import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.TVHTML5_CLIENT_VERSION;
+import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.TVHTML5_USER_AGENT;
+import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WATCH_CLIENT_SCREEN;
+import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_CLIENT_NAME;
+import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_EMBEDDED_CLIENT_ID;
+import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_EMBEDDED_CLIENT_NAME;
+import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_EMBEDDED_CLIENT_VERSION;
+import static org.schabi.newpipe.extractor.services.youtube.ClientsConstants.WEB_REMIX_HARDCODED_CLIENT_VERSION;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.CONTENT_CHECK_OK;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.CPN;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.RACY_CHECK_OK;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.VIDEO_ID;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_GAPIS_URL;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.generateTParameter;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getAndroidUserAgent;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getClientHeaders;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getClientVersion;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getIosUserAgent;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getOriginReferrerHeaders;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getYouTubeHeaders;
+import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
+
+public final class YoutubeStreamHelper {
+
+ private static final String PLAYER = "player";
+ private static final String SERVICE_INTEGRITY_DIMENSIONS = "serviceIntegrityDimensions";
+ private static final String PO_TOKEN = "poToken";
+
+ private YoutubeStreamHelper() {
+ }
+
+ @Nonnull
+ public static JsonObject getWebMetadataPlayerResponse(
+ @Nonnull final Localization localization,
+ @Nonnull final ContentCountry contentCountry,
+ @Nonnull final String videoId) throws IOException, ExtractionException {
+ final JsonBuilder builder = prepareJsonBuilder(
+ localization,
+ contentCountry,
+ WEB_CLIENT_NAME,
+ getClientVersion(),
+ WATCH_CLIENT_SCREEN,
+ DESKTOP_CLIENT_PLATFORM,
+ "", // TODO
+ null,
+ null,
+ null,
+ null,
+ null,
+ -1);
+
+ addVideoIdCpnAndOkChecks(builder, videoId, null);
+
+ final byte[] body = JsonWriter.string(builder.done())
+ .getBytes(StandardCharsets.UTF_8);
+
+ final String url = YOUTUBEI_V1_URL + PLAYER + "?" + DISABLE_PRETTY_PRINT_PARAMETER
+ + "&$fields=microformat,playabilityStatus,storyboards,videoDetails";
+
+ return JsonUtils.toJsonObject(getValidJsonResponseBody(
+ getDownloader().postWithContentTypeJson(
+ url, getYouTubeHeaders(), body, localization)));
+ }
+
+ @Nonnull
+ public static JsonObject getTvHtml5PlayerResponse(
+ @Nonnull final Localization localization,
+ @Nonnull final ContentCountry contentCountry,
+ @Nonnull final String videoId,
+ @Nonnull final String cpn) throws IOException, ExtractionException {
+ final JsonBuilder builder = prepareJsonBuilder(
+ localization,
+ contentCountry,
+ TVHTML5_CLIENT_NAME,
+ TVHTML5_CLIENT_VERSION,
+ WATCH_CLIENT_SCREEN,
+ TVHTML5_CLIENT_PLATFORM,
+ YoutubeParsingHelper.randomVisitorData(contentCountry),
+ null,
+ null,
+ null,
+ null,
+ null,
+ -1);
+ addVideoIdCpnAndOkChecks(builder, videoId, cpn);
+
+ final byte[] body = JsonWriter.string(builder.done())
+ .getBytes(StandardCharsets.UTF_8);
+
+ final String url = YOUTUBEI_V1_URL + PLAYER + "?" + DISABLE_PRETTY_PRINT_PARAMETER;
+
+ final Map> headers = new HashMap<>(
+ getClientHeaders(TVHTML5_CLIENT_ID, TVHTML5_CLIENT_VERSION));
+ headers.putAll(getOriginReferrerHeaders("https://www.youtube.com"));
+ headers.put("User-Agent", List.of(TVHTML5_USER_AGENT));
+
+ return JsonUtils.toJsonObject(getValidJsonResponseBody(
+ getDownloader().postWithContentTypeJson(
+ url, headers, body, localization)));
+ }
+
+ @Nonnull
+ public static JsonObject getWebFullPlayerResponse(
+ @Nonnull final Localization localization,
+ @Nonnull final ContentCountry contentCountry,
+ @Nonnull final String videoId,
+ @Nonnull final String cpn,
+ @Nonnull final PoTokenResult webPoTokenResult,
+ final int signatureTimestamp) throws IOException, ExtractionException {
+ final JsonBuilder builder = prepareJsonBuilder(
+ localization,
+ contentCountry,
+ WEB_CLIENT_NAME,
+ getClientVersion(),
+ WATCH_CLIENT_SCREEN,
+ DESKTOP_CLIENT_PLATFORM,
+ webPoTokenResult.visitorData,
+ null,
+ null,
+ null,
+ null,
+ null,
+ -1);
+
+ addVideoIdCpnAndOkChecks(builder, videoId, cpn);
+
+ addPlaybackContext(
+ builder,
+ "https://www.youtube.com/watch?v=" + videoId,
+ signatureTimestamp);
+
+ addPoToken(builder, webPoTokenResult.playerRequestPoToken);
+
+ final byte[] body = JsonWriter.string(builder.done())
+ .getBytes(StandardCharsets.UTF_8);
+ final String url = YOUTUBEI_V1_URL + PLAYER + "?" + DISABLE_PRETTY_PRINT_PARAMETER;
+
+ return JsonUtils.toJsonObject(getValidJsonResponseBody(
+ getDownloader().postWithContentTypeJson(
+ url, getYouTubeHeaders(), body, localization)));
+ }
+
+ @Nonnull
+ public static JsonObject getWebEmbeddedPlayerResponse(
+ @Nonnull final Localization localization,
+ @Nonnull final ContentCountry contentCountry,
+ @Nonnull final String videoId,
+ @Nonnull final String cpn,
+ @Nullable final PoTokenResult webEmbeddedPoTokenResult,
+ final int signatureTimestamp) throws IOException, ExtractionException {
+ final JsonBuilder builder = prepareJsonBuilder(
+ localization,
+ contentCountry,
+ WEB_EMBEDDED_CLIENT_NAME,
+ WEB_REMIX_HARDCODED_CLIENT_VERSION,
+ EMBED_CLIENT_SCREEN,
+ DESKTOP_CLIENT_PLATFORM,
+ webEmbeddedPoTokenResult == null
+ ? YoutubeParsingHelper.randomVisitorData(contentCountry)
+ : webEmbeddedPoTokenResult.visitorData,
+ null,
+ null,
+ null,
+ null,
+ null,
+ -1);
+
+ addVideoIdCpnAndOkChecks(builder, videoId, cpn);
+
+ addPlaybackContext(
+ builder,
+ "https://www.youtube.com/watch?v=" + videoId,
+ signatureTimestamp);
+
+ if (webEmbeddedPoTokenResult != null) {
+ addPoToken(builder, webEmbeddedPoTokenResult.playerRequestPoToken);
+ }
+
+ final byte[] body = JsonWriter.string(builder.end().done())
+ .getBytes(StandardCharsets.UTF_8);
+ final String url = YOUTUBEI_V1_URL + PLAYER + "?" + DISABLE_PRETTY_PRINT_PARAMETER;
+
+ final Map> headers = new HashMap<>(
+ getClientHeaders(WEB_EMBEDDED_CLIENT_ID, WEB_EMBEDDED_CLIENT_VERSION));
+ headers.putAll(getOriginReferrerHeaders("https://www.youtube.com"));
+
+ return JsonUtils.toJsonObject(getValidJsonResponseBody(
+ getDownloader().postWithContentTypeJson(
+ url, headers, body, localization)));
+ }
+
+ public static JsonObject getAndroidPlayerResponse(
+ @Nonnull final ContentCountry contentCountry,
+ @Nonnull final Localization localization,
+ @Nonnull final String videoId,
+ @Nonnull final String cpn,
+ @Nonnull final PoTokenResult androidPoTokenResult)
+ throws IOException, ExtractionException {
+
+ final JsonBuilder builder = prepareJsonBuilder(
+ localization,
+ contentCountry,
+ ANDROID_CLIENT_NAME,
+ ANDROID_CLIENT_VERSION,
+ WATCH_CLIENT_SCREEN,
+ MOBILE_CLIENT_PLATFORM,
+ androidPoTokenResult.visitorData,
+ null,
+ null,
+ "Android",
+ "15",
+ null,
+ 35);
+
+ addVideoIdCpnAndOkChecks(builder, videoId, cpn);
+
+ addPoToken(builder, androidPoTokenResult.playerRequestPoToken);
+
+ final byte[] body = JsonWriter.string(builder.end().done())
+ .getBytes(StandardCharsets.UTF_8);
+
+ return getJsonAndroidPostResponse(
+ PLAYER,
+ body,
+ localization,
+ "&t=" + generateTParameter() + "&id=" + videoId);
+ }
+
+ public static JsonObject getAndroidReelPlayerResponse(
+ @Nonnull final ContentCountry contentCountry,
+ @Nonnull final Localization localization,
+ @Nonnull final String videoId,
+ @Nonnull final String cpn) throws IOException, ExtractionException {
+ final JsonBuilder builder = prepareJsonBuilder(
+ localization,
+ contentCountry,
+ ANDROID_CLIENT_NAME,
+ ANDROID_CLIENT_VERSION,
+ WATCH_CLIENT_SCREEN,
+ MOBILE_CLIENT_PLATFORM,
+ "", // TODO
+ null,
+ null,
+ "Android",
+ "15",
+ null,
+ 35);
+
+ addVideoIdCpnAndOkChecks(builder, videoId, cpn);
+
+ builder.object("playerRequest")
+ .value(VIDEO_ID, videoId)
+ .end()
+ .value("disablePlayerResponse", false);
+
+ final byte[] mobileBody = JsonWriter.string(builder.done())
+ .getBytes(StandardCharsets.UTF_8);
+
+ final JsonObject androidPlayerResponse = getJsonAndroidPostResponse(
+ "reel/reel_item_watch",
+ mobileBody,
+ localization,
+ "&t=" + generateTParameter() + "&id=" + videoId + "&$fields=playerResponse");
+
+ return androidPlayerResponse.getObject("playerResponse");
+ }
+
+ public static JsonObject getIosPlayerResponse(@Nonnull final ContentCountry contentCountry,
+ @Nonnull final Localization localization,
+ @Nonnull final String videoId,
+ @Nonnull final String cpn,
+ @Nullable final PoTokenResult iosPoTokenResult)
+ throws IOException, ExtractionException {
+ final boolean noPoTokenResult = iosPoTokenResult == null;
+ final JsonBuilder builder = prepareJsonBuilder(
+ localization,
+ contentCountry,
+ IOS_CLIENT_NAME,
+ IOS_CLIENT_VERSION,
+ WATCH_CLIENT_SCREEN,
+ MOBILE_CLIENT_PLATFORM,
+ noPoTokenResult ? "" : iosPoTokenResult.visitorData, // TODO
+ "Apple",
+ IOS_DEVICE_MODEL,
+ "iOS",
+ IOS_OS_VERSION,
+ null,
+ -1);
+
+ addVideoIdCpnAndOkChecks(builder, videoId, cpn);
+ if (!noPoTokenResult) {
+ addPoToken(builder, iosPoTokenResult.playerRequestPoToken);
+ }
+
+ final byte[] mobileBody = JsonWriter.string(builder.done())
+ .getBytes(StandardCharsets.UTF_8);
+
+ return getJsonIosPostResponse(
+ mobileBody, localization, "&t=" + generateTParameter()
+ + "&id=" + videoId + "&fields=streamingData.hlsManifestUrl");
+ }
+
+ public static JsonObject getJsonAndroidPostResponse(final String endpoint,
+ final byte[] body,
+ @Nonnull final Localization localization,
+ @Nullable final String endPartOfUrlRequest)
+ throws IOException, ExtractionException {
+ return getMobilePostResponse(endpoint, body, localization,
+ getAndroidUserAgent(localization), endPartOfUrlRequest);
+ }
+
+ private static JsonObject getJsonIosPostResponse(final byte[] body,
+ @Nonnull final Localization localization,
+ @Nullable final String endPartOfUrlRequest)
+ throws IOException, ExtractionException {
+ return getMobilePostResponse(YoutubeStreamHelper.PLAYER, body, localization,
+ getIosUserAgent(localization),
+ endPartOfUrlRequest);
+ }
+
+ @Nonnull
+ private static JsonBuilder prepareJsonBuilder(
+ @Nonnull final Localization localization,
+ @Nonnull final ContentCountry contentCountry,
+ @Nonnull final String clientName,
+ @Nonnull final String clientVersion,
+ @Nonnull final String clientScreen,
+ @Nonnull final String platform,
+ @Nonnull final String visitorData,
+ @Nullable final String deviceMake,
+ @Nullable final String deviceModel,
+ @Nullable final String osName,
+ @Nullable final String osVersion,
+ @Nullable final String embedUrl,
+ final int androidSdkVersion) {
+ final JsonBuilder builder = JsonObject.builder()
+ .object("context")
+ .object("client")
+ .value("clientName", clientName)
+ .value("clientVersion", clientVersion)
+ .value("clientScreen", clientScreen)
+ .value("platform", platform)
+ .value("visitorData", visitorData);
+
+ if (deviceMake != null) {
+ builder.value("deviceMake", deviceMake);
+ }
+ if (deviceModel != null) {
+ builder.value("deviceModel", deviceModel);
+ }
+ if (osName != null) {
+ builder.value("osName", osName);
+ }
+ if (osVersion != null) {
+ builder.value("osVersion", osVersion);
+ }
+ if (androidSdkVersion > 0) {
+ builder.value("androidSdkVersion", androidSdkVersion);
+ }
+
+ builder.value("hl", localization.getLocalizationCode())
+ .value("gl", contentCountry.getCountryCode())
+ .value("utcOffsetMinutes", 0)
+ .end();
+
+ if (embedUrl != null) {
+ builder.object("thirdParty")
+ .value("embedUrl", embedUrl)
+ .end();
+ }
+
+ builder.object("request")
+ .array("internalExperimentFlags")
+ .end()
+ .value("useSsl", true)
+ .end()
+ .object("user")
+ // TODO: provide a way to enable restricted mode with:
+ // .value("enableSafetyMode", boolean)
+ .value("lockedSafetyMode", false)
+ .end()
+ .end();
+
+ return builder;
+ }
+
+ private static JsonObject getMobilePostResponse(final String endpoint,
+ final byte[] body,
+ @Nonnull final Localization localization,
+ @Nonnull final String userAgent,
+ @Nullable final String endPartOfUrlRequest)
+ throws IOException, ExtractionException {
+ final var headers = Map.of("User-Agent", List.of(userAgent),
+ "X-Goog-Api-Format-Version", List.of("2"));
+
+ final String baseEndpointUrl = YOUTUBEI_V1_GAPIS_URL + endpoint + "?"
+ + DISABLE_PRETTY_PRINT_PARAMETER;
+
+ return JsonUtils.toJsonObject(getValidJsonResponseBody(
+ getDownloader().postWithContentTypeJson(isNullOrEmpty(endPartOfUrlRequest)
+ ? baseEndpointUrl
+ : baseEndpointUrl + endPartOfUrlRequest,
+ headers, body, localization)));
+ }
+
+ private static void addVideoIdCpnAndOkChecks(@Nonnull final JsonBuilder builder,
+ @Nonnull final String videoId,
+ @Nullable final String cpn) {
+ builder.value(VIDEO_ID, videoId);
+
+ if (cpn != null) {
+ builder.value(CPN, cpn);
+ }
+
+ builder.value(CONTENT_CHECK_OK, true)
+ .value(RACY_CHECK_OK, true);
+ }
+
+ private static void addPlaybackContext(@Nonnull final JsonBuilder builder,
+ @Nonnull final String referer,
+ final int signatureTimestamp) {
+ builder.object("playbackContext")
+ .object("contentPlaybackContext")
+ .value("signatureTimestamp", signatureTimestamp)
+ .value("referer", referer)
+ .end()
+ .end();
+ }
+
+ private static void addPoToken(@Nonnull final JsonBuilder builder,
+ @Nonnull final String poToken) {
+ builder.object(SERVICE_INTEGRITY_DIMENSIONS)
+ .value(PO_TOKEN, poToken)
+ .end();
+ }
+}
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java
index 988a1bcc23..9a9f769ab2 100644
--- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java
+++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java
@@ -27,18 +27,12 @@
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.CPN;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.RACY_CHECK_OK;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.VIDEO_ID;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.createTvHtml5EmbedPlayerBody;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.generateContentPlaybackNonce;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.generateTParameter;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getImagesFromThumbnailsArray;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonAndroidPostResponse;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonIosPostResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareAndroidMobileJsonBuilder;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareIosMobileJsonBuilder;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import com.grack.nanojson.JsonArray;
@@ -66,9 +60,12 @@
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
import org.schabi.newpipe.extractor.localization.TimeAgoPatternsManager;
import org.schabi.newpipe.extractor.services.youtube.ItagItem;
+import org.schabi.newpipe.extractor.services.youtube.PoTokenProvider;
+import org.schabi.newpipe.extractor.services.youtube.PoTokenResult;
import org.schabi.newpipe.extractor.services.youtube.YoutubeJavaScriptPlayerManager;
import org.schabi.newpipe.extractor.services.youtube.YoutubeMetaInfoHelper;
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
+import org.schabi.newpipe.extractor.services.youtube.YoutubeStreamHelper;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory;
import org.schabi.newpipe.extractor.stream.AudioStream;
import org.schabi.newpipe.extractor.stream.DeliveryMethod;
@@ -104,6 +101,13 @@
public class YoutubeStreamExtractor extends StreamExtractor {
+ public static final String PLAYER_CAPTIONS_TRACKLIST_RENDERER =
+ "playerCaptionsTracklistRenderer";
+ public static final String CAPTIONS = "captions";
+ @Nullable
+ private static PoTokenProvider poTokenProvider;
+ private static boolean forceFetchIosClient;
+
private JsonObject playerResponse;
private JsonObject nextResponse;
@@ -112,7 +116,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Nullable
private JsonObject androidStreamingData;
@Nullable
- private JsonObject tvHtml5SimplyEmbedStreamingData;
+ private JsonObject html5StreamingData;
private JsonObject videoPrimaryInfoRenderer;
private JsonObject videoSecondaryInfoRenderer;
@@ -127,7 +131,14 @@ public class YoutubeStreamExtractor extends StreamExtractor {
// three different strings are used.
private String iosCpn;
private String androidCpn;
- private String tvHtml5SimplyEmbedCpn;
+ private String html5Cpn;
+
+ @Nullable
+ private String iosStreamingUrlsPoToken;
+ @Nullable
+ private String androidStreamingUrlsPoToken;
+ @Nullable
+ private String html5StreamingUrlsPoToken;
public YoutubeStreamExtractor(final StreamingService service, final LinkHandler linkHandler) {
super(service, linkHandler);
@@ -321,7 +332,7 @@ public long getLength() throws ParsingException {
return Long.parseLong(duration);
} catch (final Exception e) {
return getDurationFromFirstAdaptiveFormat(Arrays.asList(
- iosStreamingData, androidStreamingData, tvHtml5SimplyEmbedStreamingData));
+ androidStreamingData, html5StreamingData));
}
}
@@ -579,11 +590,13 @@ public long getUploaderSubscriberCount() throws ParsingException {
public String getDashMpdUrl() throws ParsingException {
assertPageFetched();
- // There is no DASH manifest available in the iOS clients and the DASH manifest of the
- // Android client doesn't contain all available streams (mainly the WEBM ones)
+ // There is no DASH manifest available with the iOS clients
return getManifestUrl(
"dash",
- Arrays.asList(androidStreamingData, tvHtml5SimplyEmbedStreamingData));
+ Arrays.asList(
+ new Pair<>(androidStreamingData, androidStreamingUrlsPoToken),
+ new Pair<>(html5StreamingData, html5StreamingUrlsPoToken)),
+ "mpd_version=7");
}
@Nonnull
@@ -592,25 +605,42 @@ public String getHlsUrl() throws ParsingException {
assertPageFetched();
// Return HLS manifest of the iOS client first because on livestreams, the HLS manifest
- // returned has separated audio and video streams
+ // returned has separated audio and video streams and poTokens requirement do not seem to
+ // impact HLS formats
// Also, on videos, non-iOS clients don't have an HLS manifest URL in their player response
+ // as we don't use a Safari macOS user agent
return getManifestUrl(
"hls",
Arrays.asList(
- iosStreamingData, androidStreamingData, tvHtml5SimplyEmbedStreamingData));
+ new Pair<>(iosStreamingData, iosStreamingUrlsPoToken),
+ new Pair<>(androidStreamingData, androidStreamingUrlsPoToken),
+ new Pair<>(html5StreamingData, html5StreamingUrlsPoToken)),
+ "");
}
@Nonnull
- private static String getManifestUrl(@Nonnull final String manifestType,
- @Nonnull final List streamingDataObjects) {
+ private static String getManifestUrl(
+ @Nonnull final String manifestType,
+ @Nonnull final List> streamingDataObjects,
+ @Nonnull final String partToAppendToManifestUrlEnd) {
final String manifestKey = manifestType + "ManifestUrl";
- return streamingDataObjects.stream()
- .filter(Objects::nonNull)
- .map(streamingDataObject -> streamingDataObject.getString(manifestKey))
- .filter(Objects::nonNull)
- .findFirst()
- .orElse("");
+ for (final Pair streamingDataObj : streamingDataObjects) {
+ final String manifestUrl = streamingDataObj.getFirst().getString(manifestKey);
+ if (isNullOrEmpty(manifestUrl)) {
+ continue;
+ }
+
+ // If poToken is not null, add it to manifest URL
+ if (streamingDataObj.getSecond() == null) {
+ return manifestUrl + "?" + partToAppendToManifestUrlEnd;
+ } else {
+ return manifestUrl + "?pot=" + streamingDataObj.getSecond() + "&="
+ + partToAppendToManifestUrlEnd;
+ }
+ }
+
+ return "";
}
@Override
@@ -766,7 +796,6 @@ public String getErrorMessage() {
private static final String FORMATS = "formats";
private static final String ADAPTIVE_FORMATS = "adaptiveFormats";
private static final String STREAMING_DATA = "streamingData";
- private static final String PLAYER = "player";
private static final String NEXT = "next";
private static final String SIGNATURE_CIPHER = "signatureCipher";
private static final String CIPHER = "cipher";
@@ -779,98 +808,78 @@ public void onFetchPage(@Nonnull final Downloader downloader)
final Localization localization = getExtractorLocalization();
final ContentCountry contentCountry = getExtractorContentCountry();
- final JsonObject webPlayerResponse = YoutubeParsingHelper.getWebPlayerResponse(
- localization, contentCountry, videoId);
+ final PoTokenProvider providerInstance = poTokenProvider;
+ final boolean noPoTokenProviderSet = providerInstance == null;
- if (isPlayerResponseNotValid(webPlayerResponse, videoId)) {
- // Check the playability status, as private and deleted videos and invalid video IDs do
- // not return the ID provided in the player response
- // When the requested video is playable and a different video ID is returned, it has
- // the OK playability status, meaning the ExtractionException after this check will be
- // thrown
- checkPlayabilityStatus(
- webPlayerResponse, webPlayerResponse.getObject("playabilityStatus"));
- throw new ExtractionException("Initial WEB player response is not valid");
- }
+ final PoTokenResult webPoTokenResult = noPoTokenProviderSet ? null
+ : providerInstance.getWebClientPoToken(videoId);
- // Save the webPlayerResponse into playerResponse in the case the video cannot be played,
- // so some metadata can be retrieved
- playerResponse = webPlayerResponse;
+ fetchHtml5Client(localization, contentCountry, videoId, webPoTokenResult);
// Use the player response from the player endpoint of the desktop internal API because
// there can be restrictions on videos in the embedded player.
// E.g. if a video is age-restricted, the embedded player's playabilityStatus says that
// the video cannot be played outside of YouTube, but does not show the original message.
- final JsonObject playabilityStatus = webPlayerResponse.getObject("playabilityStatus");
+ final JsonObject playabilityStatus = playerResponse.getObject("playabilityStatus");
final boolean isAgeRestricted = "login_required".equalsIgnoreCase(
playabilityStatus.getString("status"))
&& playabilityStatus.getString("reason", "")
.contains("age");
- setStreamType();
-
if (isAgeRestricted) {
- fetchTvHtml5EmbedJsonPlayer(contentCountry, localization, videoId);
-
- // If no streams can be fetched in the TVHTML5 simply embed client, the video should be
- // age-restricted, therefore throw an AgeRestrictedContentException explicitly.
- if (tvHtml5SimplyEmbedStreamingData == null) {
- throw new AgeRestrictedContentException(
- "This age-restricted video cannot be watched.");
- }
+ final PoTokenResult webEmbedPoTokenResult = noPoTokenProviderSet ? null
+ : providerInstance.getWebEmbedClientPoToken(videoId);
- // Refresh the stream type because the stream type may be not properly known for
- // age-restricted videos
- setStreamType();
- } else {
- checkPlayabilityStatus(webPlayerResponse, playabilityStatus);
-
- // Fetching successfully the iOS player is mandatory to get streams
- fetchIosMobileJsonPlayer(contentCountry, localization, videoId);
-
- try {
- fetchAndroidMobileJsonPlayer(contentCountry, localization, videoId);
- } catch (final Exception ignored) {
- // Ignore exceptions related to ANDROID client fetch or parsing, as it is not
- // compulsory to play contents
- }
+ fetchHtml5EmbedClient(localization, contentCountry, videoId, webEmbedPoTokenResult);
}
+ setStreamType();
+
// The microformat JSON object of the content is only returned on the WEB client,
// so we need to store it instead of getting it directly from the playerResponse
- playerMicroFormatRenderer = webPlayerResponse.getObject("microformat")
+ playerMicroFormatRenderer = playerResponse.getObject("microformat")
.getObject("playerMicroformatRenderer");
- final byte[] body = JsonWriter.string(
+ final PoTokenResult androidPoTokenResult = noPoTokenProviderSet ? null
+ : providerInstance.getAndroidClientPoToken(videoId);
+
+ fetchAndroidClient(localization, contentCountry, videoId, androidPoTokenResult);
+
+ if (forceFetchIosClient) {
+ final PoTokenResult iosPoTokenResult = noPoTokenProviderSet ? null
+ : providerInstance.getIosClientPoToken(videoId);
+ fetchIosClient(localization, contentCountry, videoId, iosPoTokenResult);
+ }
+
+ final byte[] nextBody = JsonWriter.string(
prepareDesktopJsonBuilder(localization, contentCountry)
.value(VIDEO_ID, videoId)
.value(CONTENT_CHECK_OK, true)
.value(RACY_CHECK_OK, true)
.done())
.getBytes(StandardCharsets.UTF_8);
- nextResponse = getJsonPostResponse(NEXT, body, localization);
+ nextResponse = getJsonPostResponse(NEXT, nextBody, localization);
}
- private void checkPlayabilityStatus(final JsonObject youtubePlayerResponse,
- @Nonnull final JsonObject playabilityStatus)
+ private static void checkPlayabilityStatus(@Nonnull final JsonObject playabilityStatus)
throws ParsingException {
- String status = playabilityStatus.getString("status");
+ final String status = playabilityStatus.getString("status");
if (status == null || status.equalsIgnoreCase("ok")) {
return;
}
- // If status exist, and is not "OK", throw the specific exception based on error message
- // or a ContentNotAvailableException with the reason text if it's an unknown reason.
- final JsonObject newPlayabilityStatus =
- youtubePlayerResponse.getObject("playabilityStatus");
- status = newPlayabilityStatus.getString("status");
- final String reason = newPlayabilityStatus.getString("reason");
-
- if (status.equalsIgnoreCase("login_required") && reason == null) {
- final String message = newPlayabilityStatus.getArray("messages").getString(0);
- if (message != null && message.contains("private")) {
- throw new PrivateContentException("This video is private.");
+ final String reason = playabilityStatus.getString("reason");
+
+ if (status.equalsIgnoreCase("login_required")) {
+ if (reason == null) {
+ final String message = playabilityStatus.getArray("messages").getString(0);
+ if (message != null && message.contains("private")) {
+ throw new PrivateContentException("This video is private");
+ }
+ } else if (reason.contains("age")) {
+ throw new AgeRestrictedContentException(
+ "This age-restricted video cannot be watched anonymously");
}
}
@@ -888,7 +897,7 @@ private void checkPlayabilityStatus(final JsonObject youtubePlayerResponse,
}
if (reason.contains("unavailable")) {
- final String detailedErrorMessage = getTextFromObject(newPlayabilityStatus
+ final String detailedErrorMessage = getTextFromObject(playabilityStatus
.getObject("errorScreen")
.getObject("playerErrorMessageRenderer")
.getObject("subreason"));
@@ -905,115 +914,152 @@ private void checkPlayabilityStatus(final JsonObject youtubePlayerResponse,
throw new ContentNotAvailableException("Got error: \"" + reason + "\"");
}
- /**
- * Fetch the Android Mobile API and assign the streaming data to the androidStreamingData JSON
- * object.
- */
- private void fetchAndroidMobileJsonPlayer(@Nonnull final ContentCountry contentCountry,
- @Nonnull final Localization localization,
- @Nonnull final String videoId)
+ private void fetchHtml5Client(@Nonnull final Localization localization,
+ @Nonnull final ContentCountry contentCountry,
+ @Nonnull final String videoId,
+ @Nullable final PoTokenResult webPoTokenResult)
throws IOException, ExtractionException {
- androidCpn = generateContentPlaybackNonce();
- final byte[] mobileBody = JsonWriter.string(
- prepareAndroidMobileJsonBuilder(localization, contentCountry)
- .object("playerRequest")
- .value(VIDEO_ID, videoId)
- .end()
- .value("disablePlayerResponse", false)
- .value(VIDEO_ID, videoId)
- .value(CPN, androidCpn)
- .value(CONTENT_CHECK_OK, true)
- .value(RACY_CHECK_OK, true)
- .done())
- .getBytes(StandardCharsets.UTF_8);
+ html5Cpn = generateContentPlaybackNonce();
- final JsonObject androidPlayerResponse = getJsonAndroidPostResponse(
- "reel/reel_item_watch",
- mobileBody,
- localization,
- "&t=" + generateTParameter() + "&id=" + videoId + "&$fields=playerResponse");
+ final JsonObject webPlayerResponse;
+ if (webPoTokenResult == null) {
+ webPlayerResponse = YoutubeStreamHelper.getWebMetadataPlayerResponse(
+ localization, contentCountry, videoId);
- final JsonObject playerResponseObject = androidPlayerResponse.getObject("playerResponse");
- if (isPlayerResponseNotValid(playerResponseObject, videoId)) {
- return;
- }
+ throwExceptionIfPlayerResponseNotValid(webPlayerResponse, videoId);
+
+ // Save the webPlayerResponse into playerResponse in the case the video cannot be
+ // played, so some metadata can be retrieved
+ playerResponse = webPlayerResponse;
- final JsonObject streamingData = playerResponseObject.getObject(STREAMING_DATA);
- if (!isNullOrEmpty(streamingData)) {
- androidStreamingData = streamingData;
- if (isNullOrEmpty(playerCaptionsTracklistRenderer)) {
- playerCaptionsTracklistRenderer = playerResponseObject.getObject("captions")
- .getObject("playerCaptionsTracklistRenderer");
+ final JsonObject tvHtml5PlayerResponse = YoutubeStreamHelper.getTvHtml5PlayerResponse(
+ localization, contentCountry, videoId, html5Cpn);
+
+ if (isPlayerResponseNotValid(tvHtml5PlayerResponse, videoId)) {
+ throw new ExtractionException("TVHTML5 player response is not valid");
}
+
+ html5StreamingData = tvHtml5PlayerResponse.getObject(STREAMING_DATA);
+ playerCaptionsTracklistRenderer = tvHtml5PlayerResponse.getObject(CAPTIONS)
+ .getObject(PLAYER_CAPTIONS_TRACKLIST_RENDERER);
+ } else {
+ webPlayerResponse = YoutubeStreamHelper.getWebFullPlayerResponse(
+ localization, contentCountry, videoId, html5Cpn, webPoTokenResult,
+ YoutubeJavaScriptPlayerManager.getSignatureTimestamp(videoId));
+
+ // Save the webPlayerResponse into playerResponse in the case the video cannot be
+ // played, so some metadata can be retrieved
+ playerResponse = webPlayerResponse;
+
+ throwExceptionIfPlayerResponseNotValid(webPlayerResponse, videoId);
+
+ html5StreamingData = webPlayerResponse.getObject(STREAMING_DATA);
+ playerCaptionsTracklistRenderer = webPlayerResponse.getObject(CAPTIONS)
+ .getObject(PLAYER_CAPTIONS_TRACKLIST_RENDERER);
+ html5StreamingUrlsPoToken = webPoTokenResult.streamingDataPoToken;
}
}
- /**
- * Fetch the iOS Mobile API and assign the streaming data to the iosStreamingData JSON
- * object.
- */
- private void fetchIosMobileJsonPlayer(@Nonnull final ContentCountry contentCountry,
- @Nonnull final Localization localization,
- @Nonnull final String videoId)
+ private static void throwExceptionIfPlayerResponseNotValid(
+ @Nonnull final JsonObject webPlayerResponse,
+ @Nonnull final String videoId) throws ExtractionException {
+ if (isPlayerResponseNotValid(webPlayerResponse, videoId)) {
+ // Check the playability status, as private and deleted videos and invalid video
+ // IDs do not return the ID provided in the player response
+ // When the requested video is playable and a different video ID is returned, it
+ // has the OK playability status, meaning the ExtractionException after this check
+ // will be thrown
+ checkPlayabilityStatus(webPlayerResponse.getObject("playabilityStatus"));
+ throw new ExtractionException("WEB player response is not valid");
+ }
+ }
+
+ private void fetchHtml5EmbedClient(@Nonnull final Localization localization,
+ @Nonnull final ContentCountry contentCountry,
+ @Nonnull final String videoId,
+ @Nullable final PoTokenResult webEmbedPoTokenResult)
throws IOException, ExtractionException {
- iosCpn = generateContentPlaybackNonce();
- final byte[] mobileBody = JsonWriter.string(
- prepareIosMobileJsonBuilder(localization, contentCountry)
- .value(VIDEO_ID, videoId)
- .value(CPN, iosCpn)
- .value(CONTENT_CHECK_OK, true)
- .value(RACY_CHECK_OK, true)
- .done())
- .getBytes(StandardCharsets.UTF_8);
+ html5Cpn = generateContentPlaybackNonce();
+
+ final JsonObject webEmbeddedPlayerResponse =
+ YoutubeStreamHelper.getWebEmbeddedPlayerResponse(localization, contentCountry,
+ videoId, html5Cpn, webEmbedPoTokenResult,
+ YoutubeJavaScriptPlayerManager.getSignatureTimestamp(videoId));
- final JsonObject iosPlayerResponse = getJsonIosPostResponse(PLAYER,
- mobileBody, localization, "&t=" + generateTParameter()
- + "&id=" + videoId);
+ // Save the webEmbeddedPlayerResponse into playerResponse in the case the video cannot be
+ // played, so some metadata can be retrieved
+ playerResponse = webEmbeddedPlayerResponse;
+
+ if (webEmbedPoTokenResult != null) {
+ html5StreamingUrlsPoToken = webEmbedPoTokenResult.streamingDataPoToken;
+ }
- if (isPlayerResponseNotValid(iosPlayerResponse, videoId)) {
- throw new ExtractionException("IOS player response is not valid");
+ // Check if the playability status in the player response, if the age-restriction could not
+ // be bypassed, an exception will be thrown
+ checkPlayabilityStatus(webEmbeddedPlayerResponse.getObject("playabilityStatus"));
+ if (isPlayerResponseNotValid(webEmbeddedPlayerResponse, videoId)) {
+ throw new ExtractionException("WEB_EMBEDDED_PLAYER player response is not valid");
}
- final JsonObject streamingData = iosPlayerResponse.getObject(STREAMING_DATA);
- if (!isNullOrEmpty(streamingData)) {
- iosStreamingData = streamingData;
- playerCaptionsTracklistRenderer = iosPlayerResponse.getObject("captions")
- .getObject("playerCaptionsTracklistRenderer");
+ if (isNullOrEmpty(playerCaptionsTracklistRenderer)) {
+ playerCaptionsTracklistRenderer = webEmbeddedPlayerResponse.getObject(CAPTIONS)
+ .getObject(PLAYER_CAPTIONS_TRACKLIST_RENDERER);
}
}
- /**
- * Download the {@code TVHTML5_SIMPLY_EMBEDDED_PLAYER} JSON player as an embed client to bypass
- * some age-restrictions and assign the streaming data to the {@code html5StreamingData} JSON
- * object.
- *
- * @param contentCountry the content country to use
- * @param localization the localization to use
- * @param videoId the video id
- */
- private void fetchTvHtml5EmbedJsonPlayer(@Nonnull final ContentCountry contentCountry,
- @Nonnull final Localization localization,
- @Nonnull final String videoId)
- throws IOException, ExtractionException {
- tvHtml5SimplyEmbedCpn = generateContentPlaybackNonce();
+ private void fetchAndroidClient(@Nonnull final Localization localization,
+ @Nonnull final ContentCountry contentCountry,
+ @Nonnull final String videoId,
+ @Nullable final PoTokenResult androidPoTokenResult) {
+ try {
+ androidCpn = generateContentPlaybackNonce();
+
+ final JsonObject androidPlayerResponse;
+ if (androidPoTokenResult == null) {
+ androidPlayerResponse = YoutubeStreamHelper.getAndroidReelPlayerResponse(
+ contentCountry, localization, videoId, androidCpn);
+ } else {
+ androidPlayerResponse = YoutubeStreamHelper.getAndroidPlayerResponse(
+ contentCountry, localization, videoId, androidCpn,
+ androidPoTokenResult);
+ }
- final JsonObject tvHtml5EmbedPlayerResponse = getJsonPostResponse(PLAYER,
- createTvHtml5EmbedPlayerBody(localization,
- contentCountry,
- videoId,
- YoutubeJavaScriptPlayerManager.getSignatureTimestamp(videoId),
- tvHtml5SimplyEmbedCpn), localization);
+ if (!isPlayerResponseNotValid(androidPlayerResponse, videoId)) {
+ androidStreamingData = androidPlayerResponse.getObject(STREAMING_DATA);
- if (isPlayerResponseNotValid(tvHtml5EmbedPlayerResponse, videoId)) {
- throw new ExtractionException("TVHTML5 embed player response is not valid");
+ if (isNullOrEmpty(playerCaptionsTracklistRenderer)) {
+ playerCaptionsTracklistRenderer =
+ androidPlayerResponse.getObject(CAPTIONS)
+ .getObject(PLAYER_CAPTIONS_TRACKLIST_RENDERER);
+ }
+ }
+ } catch (final Exception ignored) {
+ // Ignore exceptions related to ANDROID client fetch or parsing, as it is not
+ // compulsory to play contents
}
+ }
- final JsonObject streamingData = tvHtml5EmbedPlayerResponse.getObject(STREAMING_DATA);
- if (!isNullOrEmpty(streamingData)) {
- playerResponse = tvHtml5EmbedPlayerResponse;
- tvHtml5SimplyEmbedStreamingData = streamingData;
- playerCaptionsTracklistRenderer = playerResponse.getObject("captions")
- .getObject("playerCaptionsTracklistRenderer");
+ private void fetchIosClient(@Nonnull final Localization localization,
+ @Nonnull final ContentCountry contentCountry,
+ @Nonnull final String videoId,
+ @Nullable final PoTokenResult iosPoTokenResult) {
+ try {
+ iosCpn = generateContentPlaybackNonce();
+
+ final JsonObject iosPlayerResponse = YoutubeStreamHelper.getIosPlayerResponse(
+ contentCountry, localization, videoId, iosCpn, iosPoTokenResult);
+
+ if (!isPlayerResponseNotValid(iosPlayerResponse, videoId)) {
+ iosStreamingData = iosPlayerResponse.getObject(STREAMING_DATA);
+
+ if (isNullOrEmpty(playerCaptionsTracklistRenderer)) {
+ playerCaptionsTracklistRenderer = iosPlayerResponse.getObject(CAPTIONS)
+ .getObject(PLAYER_CAPTIONS_TRACKLIST_RENDERER);
+ }
+ }
+ } catch (final Exception ignored) {
+ // Ignore exceptions related to IOS client fetch or parsing, as it is not
+ // compulsory to play contents
}
}
@@ -1106,22 +1152,28 @@ private List getItags(
java.util.stream.Stream.of(
/*
- Use the iosStreamingData object first because there is no n param and no
- signatureCiphers in streaming URLs of the iOS client
+ Use the html5StreamingData object first because YouTube should have less
+ control on HTML5 clients, especially for poTokens
- The androidStreamingData is used as second way as it isn't used on livestreams,
- it doesn't return all available streams, and the Android client extraction is
- more likely to break
+ The androidStreamingData is used as second way as the Android client extraction
+ is more likely to break
- As age-restricted videos are not common, use tvHtml5SimplyEmbedStreamingData
- last, which will be the only one not empty for age-restricted content
+ As iOS streaming data is affected by poTokens and not passing them should lead
+ to 403 responses, it should be used in the last resort
*/
- new Pair<>(iosStreamingData, iosCpn),
- new Pair<>(androidStreamingData, androidCpn),
- new Pair<>(tvHtml5SimplyEmbedStreamingData, tvHtml5SimplyEmbedCpn)
- )
- .flatMap(pair -> getStreamsFromStreamingDataKey(videoId, pair.getFirst(),
- streamingDataKey, itagTypeWanted, pair.getSecond()))
+ new Pair<>(html5StreamingData,
+ new Pair<>(html5Cpn, html5StreamingUrlsPoToken)),
+ new Pair<>(androidStreamingData,
+ new Pair<>(androidCpn, androidStreamingUrlsPoToken)),
+ new Pair<>(iosStreamingData,
+ new Pair<>(iosCpn, iosStreamingUrlsPoToken)))
+ .flatMap(pair -> getStreamsFromStreamingDataKey(
+ videoId,
+ pair.getFirst(),
+ streamingDataKey,
+ itagTypeWanted,
+ pair.getSecond().getFirst(),
+ pair.getSecond().getSecond()))
.map(streamBuilderHelper)
.forEachOrdered(stream -> {
if (!Stream.containSimilarStream(stream, streamList)) {
@@ -1255,7 +1307,8 @@ private java.util.stream.Stream getStreamsFromStreamingDataKey(
final JsonObject streamingData,
final String streamingDataKey,
@Nonnull final ItagItem.ItagType itagTypeWanted,
- @Nonnull final String contentPlaybackNonce) {
+ @Nonnull final String contentPlaybackNonce,
+ @Nullable final String poToken) {
if (streamingData == null || !streamingData.has(streamingDataKey)) {
return java.util.stream.Stream.empty();
}
@@ -1268,7 +1321,7 @@ private java.util.stream.Stream getStreamsFromStreamingDataKey(
final ItagItem itagItem = ItagItem.getItag(formatData.getInt("itag"));
if (itagItem.itagType == itagTypeWanted) {
return buildAndAddItagInfoToList(videoId, formatData, itagItem,
- itagItem.itagType, contentPlaybackNonce);
+ itagItem.itagType, contentPlaybackNonce, poToken);
}
} catch (final ExtractionException ignored) {
// If the itag is not supported, the n parameter of HTML5 clients cannot be
@@ -1284,7 +1337,8 @@ private ItagInfo buildAndAddItagInfoToList(
@Nonnull final JsonObject formatData,
@Nonnull final ItagItem itagItem,
@Nonnull final ItagItem.ItagType itagType,
- @Nonnull final String contentPlaybackNonce) throws ExtractionException {
+ @Nonnull final String contentPlaybackNonce,
+ @Nullable final String poToken) throws ExtractionException {
String streamUrl;
if (formatData.has("url")) {
streamUrl = formatData.getString("url");
@@ -1298,9 +1352,6 @@ private ItagInfo buildAndAddItagInfoToList(
streamUrl = cipher.get("url") + "&" + cipher.get("sp") + "=" + signature;
}
- // Add the content playback nonce to the stream URL
- streamUrl += "&" + CPN + "=" + contentPlaybackNonce;
-
// Decode the n parameter if it is present
// If it cannot be decoded, the stream cannot be used as streaming URLs return HTTP 403
// responses if it has not the right value
@@ -1310,6 +1361,14 @@ private ItagInfo buildAndAddItagInfoToList(
streamUrl = YoutubeJavaScriptPlayerManager.getUrlWithThrottlingParameterDeobfuscated(
videoId, streamUrl);
+ // Add the content playback nonce to the stream URL
+ streamUrl += "&" + CPN + "=" + contentPlaybackNonce;
+
+ // Add the poToken, if there is one
+ if (poToken != null) {
+ streamUrl += "&pot=" + poToken;
+ }
+
final JsonObject initRange = formatData.getObject("initRange");
final JsonObject indexRange = formatData.getObject("indexRange");
final String mimeType = formatData.getString("mimeType", "");
@@ -1587,4 +1646,40 @@ public List getMetaInfo() throws ParsingException {
.getObject("results")
.getArray("contents"));
}
+
+ /**
+ * Sets the {@link PoTokenProvider} instance to be used for fetching poTokens.
+ *
+ *
+ * This method allows setting an implementation of {@link PoTokenProvider} which will be used
+ * to obtain poTokens required for YouTube player requests and streaming URLs. These tokens
+ * are used by YouTube to verify the integrity of the user's device or browser and are
+ * necessary for playback for several clients.
+ *
+ *
+ * @param poTokenProvider the {@link PoTokenProvider} instance to set
+ */
+ public static void setPoTokenProvider(@Nullable final PoTokenProvider poTokenProvider) {
+ YoutubeStreamExtractor.poTokenProvider = poTokenProvider;
+ }
+
+ /**
+ * Sets whether to force fetch the iOS player response on livestreams.
+ *
+ *
+ * This method allows setting a flag to force the fetching of the iOS player response, which
+ * can be useful in scenarios where streams from the iOS player response is preferred.
+ *
+ *
+ *
+ * Note that at the time of writing, YouTube is rolling out a poToken requirement on this
+ * client. Formats from HLS manifests do not seem to be affected.
+ *
+ *
+ * @param forceFetchIosClient a boolean flag indicating whether to force fetch the iOS
+ * player response
+ */
+ public static void setForceFetchIosClient(final boolean forceFetchIosClient) {
+ YoutubeStreamExtractor.forceFetchIosClient = forceFetchIosClient;
+ }
}