Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[YouTube] Update iOS client and add visitor data to InnerTube requests #1262

Merged
merged 5 commits into from
Jan 21, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@
import org.schabi.newpipe.extractor.stream.AudioTrackType;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.ProtoBuilder;
import org.schabi.newpipe.extractor.utils.RandomStringFromAlphabetGenerator;
import org.schabi.newpipe.extractor.utils.Utils;

Expand Down Expand Up @@ -174,7 +175,7 @@ private YoutubeParsingHelper() {
* Store page of the YouTube app</a>, in the {@code What’s New} section.
* </p>
*/
private static final String IOS_YOUTUBE_CLIENT_VERSION = "19.28.1";
private static final String IOS_YOUTUBE_CLIENT_VERSION = "19.45.4";

/**
* The hardcoded client version used for InnerTube requests with the TV HTML5 embed client.
Expand Down Expand Up @@ -222,28 +223,28 @@ private YoutubeParsingHelper() {
private static final String IOS_DEVICE_MODEL = "iPhone16,2";
Copy link
Contributor

@gechoto gechoto Jan 21, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The device used here is an iPhone 15 Pro Max but the comments are incorrectly update to "iPhone 16".

Either change/revert the comments to iPhone 15 Pro Max or update the device model to "iPhone17,2" to match the iPhone 16 Pro Max.

The mapping can be found here: https://gist.github.com/adamawolf/3048717
or: https://theapplewiki.com/wiki/Firmware/iPhone/18.x#iPhone_16_Pro_Max


/**
* Spoofing an iPhone 15 Pro Max running iOS 17.5.1 with the hardcoded version of the iOS app.
* Spoofing an iPhone 15 Pro Max running iOS 18.1.0 with the hardcoded version of the iOS app.
* To be used for the {@code "osVersion"} field in JSON POST requests.
* <p>
* The value of this field seems to use the following structure:
* "iOS major version.minor version.patch version.build version", where
* "patch version" is equal to 0 if it isn't set
* The build version corresponding to the iOS version used can be found on
* <a href="https://theapplewiki.com/wiki/Firmware/iPhone/17.x#iPhone_15_Pro_Max">
* https://theapplewiki.com/wiki/Firmware/iPhone/17.x#iPhone_15_Pro_Max</a>
* <a href="https://theapplewiki.com/wiki/Firmware/iPhone/18.x#iPhone_15_Pro_Max">
* https://theapplewiki.com/wiki/Firmware/iPhone/18.x#iPhone_15_Pro_Max</a>
* </p>
*
* @see #IOS_USER_AGENT_VERSION
*/
private static final String IOS_OS_VERSION = "17.5.1.21F90";
private static final String IOS_OS_VERSION = "18.1.0.22B83";

/**
* Spoofing an iPhone 15 running iOS 17.5.1 with the hardcoded version of the iOS app. To be
* Spoofing an iPhone 15 Pro Max running iOS 18.1.0 with the hardcoded version of the iOS app. To be
* used in the user agent for requests.
*
* @see #IOS_OS_VERSION
*/
private static final String IOS_USER_AGENT_VERSION = "17_5_1";
private static final String IOS_USER_AGENT_VERSION = "18_1_0";

private static Random numberGenerator = new Random();

Expand Down Expand Up @@ -303,6 +304,23 @@ public static boolean isY2ubeURL(@Nonnull final URL url) {
return url.getHost().equalsIgnoreCase("y2u.be");
}

public static String randomVisitorData(final ContentCountry country) {
final ProtoBuilder pbE2 = new ProtoBuilder();
pbE2.string(2, "");
pbE2.varint(4, numberGenerator.nextInt(255) + 1);

final ProtoBuilder pbE = new ProtoBuilder();
pbE.string(1, country.getCountryCode());
pbE.bytes(2, pbE2.toBytes());

final ProtoBuilder pb = new ProtoBuilder();
pb.string(1, RandomStringFromAlphabetGenerator.generate(
CONTENT_PLAYBACK_NONCE_ALPHABET, 11, numberGenerator));
pb.varint(5, System.currentTimeMillis() / 1000 - numberGenerator.nextInt(600000));
pb.bytes(6, pbE.toBytes());
return pb.toUrlencodedBase64();
}

/**
* Parses the duration string of the video expecting ":" or "." as separators
*
Expand Down Expand Up @@ -1166,8 +1184,13 @@ public static JsonBuilder<JsonObject> prepareDesktopJsonBuilder(
@Nonnull final ContentCountry contentCountry,
@Nullable final String visitorData)
throws IOException, ExtractionException {
String vData = visitorData;
if (vData == null) {
vData = randomVisitorData(contentCountry);
}

// @formatter:off
final JsonBuilder<JsonObject> builder = JsonObject.builder()
return JsonObject.builder()
.object("context")
.object("client")
.value("hl", localization.getLocalizationCode())
Expand All @@ -1176,13 +1199,9 @@ public static JsonBuilder<JsonObject> prepareDesktopJsonBuilder(
.value("clientVersion", getClientVersion())
.value("originalUrl", "https://www.youtube.com")
.value("platform", "DESKTOP")
.value("utcOffsetMinutes", 0);

if (visitorData != null) {
builder.value("visitorData", visitorData);
}

return builder.end()
.value("utcOffsetMinutes", 0)
.value("visitorData", vData)
.end()
.object("request")
.array("internalExperimentFlags")
.end()
Expand Down Expand Up @@ -1256,6 +1275,7 @@ public static JsonBuilder<JsonObject> prepareIosMobileJsonBuilder(
.value("platform", "MOBILE")
.value("osName", "iOS")
.value("osVersion", IOS_OS_VERSION)
.value("visitorData", randomVisitorData(contentCountry))
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can this be the same as vData?

Maybe add a short comment into the code that documents this

.value("hl", localization.getLocalizationCode())
.value("gl", contentCountry.getCountryCode())
.value("utcOffsetMinutes", 0)
Expand Down Expand Up @@ -1392,7 +1412,7 @@ public static String getAndroidUserAgent(@Nullable final Localization localizati
*/
@Nonnull
public static String getIosUserAgent(@Nullable final Localization localization) {
// Spoofing an iPhone 15 running iOS 17.5.1 with the hardcoded version of the iOS app
// Spoofing an iPhone 15 Pro Max running iOS 18.1.0 with the hardcoded version of the iOS app
return "com.google.ios.youtube/" + IOS_YOUTUBE_CLIENT_VERSION
+ "(" + IOS_DEVICE_MODEL + "; U; CPU iOS "
+ IOS_USER_AGENT_VERSION + " like Mac OS X; "
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
package org.schabi.newpipe.extractor.utils;

import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.util.Base64;

public class ProtoBuilder {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Some documentation would be nice, e.g. what this class does

ByteArrayOutputStream byteBuffer;

public ProtoBuilder() {
this.byteBuffer = new ByteArrayOutputStream();
}

public byte[] toBytes() {
return byteBuffer.toByteArray();
}

public String toUrlencodedBase64() {
final String b64 = Base64.getUrlEncoder().encodeToString(toBytes());
return URLEncoder.encode(b64, StandardCharsets.UTF_8);
}

private void writeVarint(final long val) {
try {
if (val == 0) {
byteBuffer.write(new byte[]{(byte) 0});
} else {
long v = val;
while (v != 0) {
byte b = (byte) (v & 0x7f);
v >>= 7;

if (v != 0) {
b |= (byte) 0x80;
}
byteBuffer.write(new byte[]{b});
}
}
} catch (final IOException e) {
throw new RuntimeException(e);
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Side note: It may be better to use UncheckedIOException here since it's more specific
This might be also relevant for the rest of the file ;)

}
}

private void field(final int field, final byte wire) {
final long fbits = ((long) field) << 3;
final long wbits = ((long) wire) & 0x07;
final long val = fbits | wbits;
writeVarint(val);
}

public void varint(final int field, final long val) {
field(field, (byte) 0);
writeVarint(val);
}

public void string(final int field, final String string) {
final byte[] strBts = string.getBytes(StandardCharsets.UTF_8);
bytes(field, strBts);
}

public void bytes(final int field, final byte[] bytes) {
field(field, (byte) 2);
writeVarint(bytes.length);
try {
byteBuffer.write(bytes);
} catch (final IOException e) {
throw new RuntimeException(e);
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
package org.schabi.newpipe.extractor.utils;

import org.junit.jupiter.api.Test;

import static org.junit.jupiter.api.Assertions.assertEquals;

public class ProtoBuilderTest {
@Test
public void testProtoBuilder() {
final ProtoBuilder pb = new ProtoBuilder();
pb.varint(1, 128);
pb.varint(2, 1234567890);
pb.varint(3, 1234567890123456789L);
pb.string(4, "Hello");
pb.bytes(5, new byte[]{1, 2, 3});
assertEquals("CIABENKF2MwEGJWCpu_HnoSRESIFSGVsbG8qAwECAw%3D%3D", pb.toUrlencodedBase64());
}
}