Skip to content

Commit

Permalink
Merge pull request #1262 from Theta-Dev/fix/ios-client-vdata
Browse files Browse the repository at this point in the history
[YouTube] update iOS client, add visitor data to requests
  • Loading branch information
Stypox authored Jan 21, 2025
2 parents 8e92227 + 936bf2d commit 4c72032
Show file tree
Hide file tree
Showing 3 changed files with 126 additions and 16 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@
import org.schabi.newpipe.extractor.stream.AudioTrackType;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.ProtoBuilder;
import org.schabi.newpipe.extractor.utils.RandomStringFromAlphabetGenerator;
import org.schabi.newpipe.extractor.utils.Utils;

Expand Down Expand Up @@ -174,7 +175,7 @@ private YoutubeParsingHelper() {
* Store page of the YouTube app</a>, in the {@code What’s New} section.
* </p>
*/
private static final String IOS_YOUTUBE_CLIENT_VERSION = "19.28.1";
private static final String IOS_YOUTUBE_CLIENT_VERSION = "19.45.4";

/**
* The hardcoded client version used for InnerTube requests with the TV HTML5 embed client.
Expand Down Expand Up @@ -222,28 +223,28 @@ private YoutubeParsingHelper() {
private static final String IOS_DEVICE_MODEL = "iPhone16,2";

/**
* Spoofing an iPhone 15 Pro Max running iOS 17.5.1 with the hardcoded version of the iOS app.
* Spoofing an iPhone 15 Pro Max running iOS 18.1.0 with the hardcoded version of the iOS app.
* To be used for the {@code "osVersion"} field in JSON POST requests.
* <p>
* The value of this field seems to use the following structure:
* "iOS major version.minor version.patch version.build version", where
* "patch version" is equal to 0 if it isn't set
* The build version corresponding to the iOS version used can be found on
* <a href="https://theapplewiki.com/wiki/Firmware/iPhone/17.x#iPhone_15_Pro_Max">
* https://theapplewiki.com/wiki/Firmware/iPhone/17.x#iPhone_15_Pro_Max</a>
* <a href="https://theapplewiki.com/wiki/Firmware/iPhone/18.x#iPhone_15_Pro_Max">
* https://theapplewiki.com/wiki/Firmware/iPhone/18.x#iPhone_15_Pro_Max</a>
* </p>
*
* @see #IOS_USER_AGENT_VERSION
*/
private static final String IOS_OS_VERSION = "17.5.1.21F90";
private static final String IOS_OS_VERSION = "18.1.0.22B83";

/**
* Spoofing an iPhone 15 running iOS 17.5.1 with the hardcoded version of the iOS app. To be
* Spoofing an iPhone 15 Pro Max running iOS 18.1.0 with the hardcoded version of the iOS app. To be
* used in the user agent for requests.
*
* @see #IOS_OS_VERSION
*/
private static final String IOS_USER_AGENT_VERSION = "17_5_1";
private static final String IOS_USER_AGENT_VERSION = "18_1_0";

private static Random numberGenerator = new Random();

Expand Down Expand Up @@ -303,6 +304,23 @@ public static boolean isY2ubeURL(@Nonnull final URL url) {
return url.getHost().equalsIgnoreCase("y2u.be");
}

public static String randomVisitorData(final ContentCountry country) {
final ProtoBuilder pbE2 = new ProtoBuilder();
pbE2.string(2, "");
pbE2.varint(4, numberGenerator.nextInt(255) + 1);

final ProtoBuilder pbE = new ProtoBuilder();
pbE.string(1, country.getCountryCode());
pbE.bytes(2, pbE2.toBytes());

final ProtoBuilder pb = new ProtoBuilder();
pb.string(1, RandomStringFromAlphabetGenerator.generate(
CONTENT_PLAYBACK_NONCE_ALPHABET, 11, numberGenerator));
pb.varint(5, System.currentTimeMillis() / 1000 - numberGenerator.nextInt(600000));
pb.bytes(6, pbE.toBytes());
return pb.toUrlencodedBase64();
}

/**
* Parses the duration string of the video expecting ":" or "." as separators
*
Expand Down Expand Up @@ -1166,8 +1184,13 @@ public static JsonBuilder<JsonObject> prepareDesktopJsonBuilder(
@Nonnull final ContentCountry contentCountry,
@Nullable final String visitorData)
throws IOException, ExtractionException {
String vData = visitorData;
if (vData == null) {
vData = randomVisitorData(contentCountry);
}

// @formatter:off
final JsonBuilder<JsonObject> builder = JsonObject.builder()
return JsonObject.builder()
.object("context")
.object("client")
.value("hl", localization.getLocalizationCode())
Expand All @@ -1176,13 +1199,9 @@ public static JsonBuilder<JsonObject> prepareDesktopJsonBuilder(
.value("clientVersion", getClientVersion())
.value("originalUrl", "https://www.youtube.com")
.value("platform", "DESKTOP")
.value("utcOffsetMinutes", 0);

if (visitorData != null) {
builder.value("visitorData", visitorData);
}

return builder.end()
.value("utcOffsetMinutes", 0)
.value("visitorData", vData)
.end()
.object("request")
.array("internalExperimentFlags")
.end()
Expand Down Expand Up @@ -1256,6 +1275,7 @@ public static JsonBuilder<JsonObject> prepareIosMobileJsonBuilder(
.value("platform", "MOBILE")
.value("osName", "iOS")
.value("osVersion", IOS_OS_VERSION)
.value("visitorData", randomVisitorData(contentCountry))
.value("hl", localization.getLocalizationCode())
.value("gl", contentCountry.getCountryCode())
.value("utcOffsetMinutes", 0)
Expand Down Expand Up @@ -1392,7 +1412,7 @@ public static String getAndroidUserAgent(@Nullable final Localization localizati
*/
@Nonnull
public static String getIosUserAgent(@Nullable final Localization localization) {
// Spoofing an iPhone 15 running iOS 17.5.1 with the hardcoded version of the iOS app
// Spoofing an iPhone 15 Pro Max running iOS 18.1.0 with the hardcoded version of the iOS app
return "com.google.ios.youtube/" + IOS_YOUTUBE_CLIENT_VERSION
+ "(" + IOS_DEVICE_MODEL + "; U; CPU iOS "
+ IOS_USER_AGENT_VERSION + " like Mac OS X; "
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
package org.schabi.newpipe.extractor.utils;

import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.util.Base64;

public class ProtoBuilder {
ByteArrayOutputStream byteBuffer;

public ProtoBuilder() {
this.byteBuffer = new ByteArrayOutputStream();
}

public byte[] toBytes() {
return byteBuffer.toByteArray();
}

public String toUrlencodedBase64() {
final String b64 = Base64.getUrlEncoder().encodeToString(toBytes());
return URLEncoder.encode(b64, StandardCharsets.UTF_8);
}

private void writeVarint(final long val) {
try {
if (val == 0) {
byteBuffer.write(new byte[]{(byte) 0});
} else {
long v = val;
while (v != 0) {
byte b = (byte) (v & 0x7f);
v >>= 7;

if (v != 0) {
b |= (byte) 0x80;
}
byteBuffer.write(new byte[]{b});
}
}
} catch (final IOException e) {
throw new RuntimeException(e);
}
}

private void field(final int field, final byte wire) {
final long fbits = ((long) field) << 3;
final long wbits = ((long) wire) & 0x07;
final long val = fbits | wbits;
writeVarint(val);
}

public void varint(final int field, final long val) {
field(field, (byte) 0);
writeVarint(val);
}

public void string(final int field, final String string) {
final byte[] strBts = string.getBytes(StandardCharsets.UTF_8);
bytes(field, strBts);
}

public void bytes(final int field, final byte[] bytes) {
field(field, (byte) 2);
writeVarint(bytes.length);
try {
byteBuffer.write(bytes);
} catch (final IOException e) {
throw new RuntimeException(e);
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
package org.schabi.newpipe.extractor.utils;

import org.junit.jupiter.api.Test;

import static org.junit.jupiter.api.Assertions.assertEquals;

public class ProtoBuilderTest {
@Test
public void testProtoBuilder() {
final ProtoBuilder pb = new ProtoBuilder();
pb.varint(1, 128);
pb.varint(2, 1234567890);
pb.varint(3, 1234567890123456789L);
pb.string(4, "Hello");
pb.bytes(5, new byte[]{1, 2, 3});
assertEquals("CIABENKF2MwEGJWCpu_HnoSRESIFSGVsbG8qAwECAw%3D%3D", pb.toUrlencodedBase64());
}
}

0 comments on commit 4c72032

Please sign in to comment.