-
Notifications
You must be signed in to change notification settings - Fork 440
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
[YouTube] Update iOS client and add visitor data to InnerTube requests #1262
Changes from 4 commits
40059ed
65d888f
45645b0
5b31ff2
936bf2d
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -48,6 +48,7 @@ | |
import org.schabi.newpipe.extractor.stream.AudioTrackType; | ||
import org.schabi.newpipe.extractor.utils.JsonUtils; | ||
import org.schabi.newpipe.extractor.utils.Parser; | ||
import org.schabi.newpipe.extractor.utils.ProtoBuilder; | ||
import org.schabi.newpipe.extractor.utils.RandomStringFromAlphabetGenerator; | ||
import org.schabi.newpipe.extractor.utils.Utils; | ||
|
||
|
@@ -174,7 +175,7 @@ private YoutubeParsingHelper() { | |
* Store page of the YouTube app</a>, in the {@code What’s New} section. | ||
* </p> | ||
*/ | ||
private static final String IOS_YOUTUBE_CLIENT_VERSION = "19.28.1"; | ||
private static final String IOS_YOUTUBE_CLIENT_VERSION = "19.45.4"; | ||
|
||
/** | ||
* The hardcoded client version used for InnerTube requests with the TV HTML5 embed client. | ||
|
@@ -222,28 +223,28 @@ private YoutubeParsingHelper() { | |
private static final String IOS_DEVICE_MODEL = "iPhone16,2"; | ||
|
||
/** | ||
* Spoofing an iPhone 15 Pro Max running iOS 17.5.1 with the hardcoded version of the iOS app. | ||
* Spoofing an iPhone 15 Pro Max running iOS 18.1.0 with the hardcoded version of the iOS app. | ||
* To be used for the {@code "osVersion"} field in JSON POST requests. | ||
* <p> | ||
* The value of this field seems to use the following structure: | ||
* "iOS major version.minor version.patch version.build version", where | ||
* "patch version" is equal to 0 if it isn't set | ||
* The build version corresponding to the iOS version used can be found on | ||
* <a href="https://theapplewiki.com/wiki/Firmware/iPhone/17.x#iPhone_15_Pro_Max"> | ||
* https://theapplewiki.com/wiki/Firmware/iPhone/17.x#iPhone_15_Pro_Max</a> | ||
* <a href="https://theapplewiki.com/wiki/Firmware/iPhone/18.x#iPhone_15_Pro_Max"> | ||
* https://theapplewiki.com/wiki/Firmware/iPhone/18.x#iPhone_15_Pro_Max</a> | ||
* </p> | ||
* | ||
* @see #IOS_USER_AGENT_VERSION | ||
*/ | ||
private static final String IOS_OS_VERSION = "17.5.1.21F90"; | ||
private static final String IOS_OS_VERSION = "18.1.0.22B83"; | ||
|
||
/** | ||
* Spoofing an iPhone 15 running iOS 17.5.1 with the hardcoded version of the iOS app. To be | ||
* Spoofing an iPhone 16 Pro Max running iOS 18.1.0 with the hardcoded version of the iOS app. To be | ||
Theta-Dev marked this conversation as resolved.
Show resolved
Hide resolved
|
||
* used in the user agent for requests. | ||
* | ||
* @see #IOS_OS_VERSION | ||
*/ | ||
private static final String IOS_USER_AGENT_VERSION = "17_5_1"; | ||
private static final String IOS_USER_AGENT_VERSION = "18_1_0"; | ||
|
||
private static Random numberGenerator = new Random(); | ||
|
||
|
@@ -303,6 +304,23 @@ public static boolean isY2ubeURL(@Nonnull final URL url) { | |
return url.getHost().equalsIgnoreCase("y2u.be"); | ||
} | ||
|
||
public static String randomVisitorData(final ContentCountry country) { | ||
final ProtoBuilder pbE2 = new ProtoBuilder(); | ||
pbE2.string(2, ""); | ||
pbE2.varint(4, numberGenerator.nextInt(255) + 1); | ||
|
||
final ProtoBuilder pbE = new ProtoBuilder(); | ||
pbE.string(1, country.getCountryCode()); | ||
pbE.bytes(2, pbE2.toBytes()); | ||
|
||
final ProtoBuilder pb = new ProtoBuilder(); | ||
pb.string(1, RandomStringFromAlphabetGenerator.generate( | ||
CONTENT_PLAYBACK_NONCE_ALPHABET, 11, numberGenerator)); | ||
pb.varint(5, System.currentTimeMillis() / 1000 - numberGenerator.nextInt(600000)); | ||
pb.bytes(6, pbE.toBytes()); | ||
return pb.toUrlencodedBase64(); | ||
} | ||
|
||
/** | ||
* Parses the duration string of the video expecting ":" or "." as separators | ||
* | ||
|
@@ -1166,8 +1184,13 @@ public static JsonBuilder<JsonObject> prepareDesktopJsonBuilder( | |
@Nonnull final ContentCountry contentCountry, | ||
@Nullable final String visitorData) | ||
throws IOException, ExtractionException { | ||
String vData = visitorData; | ||
if (vData == null) { | ||
vData = randomVisitorData(contentCountry); | ||
} | ||
|
||
// @formatter:off | ||
final JsonBuilder<JsonObject> builder = JsonObject.builder() | ||
return JsonObject.builder() | ||
.object("context") | ||
.object("client") | ||
.value("hl", localization.getLocalizationCode()) | ||
|
@@ -1176,13 +1199,9 @@ public static JsonBuilder<JsonObject> prepareDesktopJsonBuilder( | |
.value("clientVersion", getClientVersion()) | ||
.value("originalUrl", "https://www.youtube.com") | ||
.value("platform", "DESKTOP") | ||
.value("utcOffsetMinutes", 0); | ||
|
||
if (visitorData != null) { | ||
builder.value("visitorData", visitorData); | ||
} | ||
|
||
return builder.end() | ||
.value("utcOffsetMinutes", 0) | ||
.value("visitorData", vData) | ||
.end() | ||
.object("request") | ||
.array("internalExperimentFlags") | ||
.end() | ||
|
@@ -1256,6 +1275,7 @@ public static JsonBuilder<JsonObject> prepareIosMobileJsonBuilder( | |
.value("platform", "MOBILE") | ||
.value("osName", "iOS") | ||
.value("osVersion", IOS_OS_VERSION) | ||
.value("visitorData", randomVisitorData(contentCountry)) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Can this be the same as Maybe add a short comment into the code that documents this |
||
.value("hl", localization.getLocalizationCode()) | ||
.value("gl", contentCountry.getCountryCode()) | ||
.value("utcOffsetMinutes", 0) | ||
|
@@ -1392,7 +1412,7 @@ public static String getAndroidUserAgent(@Nullable final Localization localizati | |
*/ | ||
@Nonnull | ||
public static String getIosUserAgent(@Nullable final Localization localization) { | ||
// Spoofing an iPhone 15 running iOS 17.5.1 with the hardcoded version of the iOS app | ||
// Spoofing an iPhone 15 Pro Max running iOS 18.1.0 with the hardcoded version of the iOS app | ||
return "com.google.ios.youtube/" + IOS_YOUTUBE_CLIENT_VERSION | ||
+ "(" + IOS_DEVICE_MODEL + "; U; CPU iOS " | ||
+ IOS_USER_AGENT_VERSION + " like Mac OS X; " | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,72 @@ | ||
package org.schabi.newpipe.extractor.utils; | ||
|
||
import java.io.ByteArrayOutputStream; | ||
import java.io.IOException; | ||
import java.net.URLEncoder; | ||
import java.nio.charset.StandardCharsets; | ||
import java.util.Base64; | ||
|
||
public class ProtoBuilder { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Some documentation would be nice, e.g. what this class does |
||
ByteArrayOutputStream byteBuffer; | ||
|
||
public ProtoBuilder() { | ||
this.byteBuffer = new ByteArrayOutputStream(); | ||
} | ||
|
||
public byte[] toBytes() { | ||
return byteBuffer.toByteArray(); | ||
} | ||
|
||
public String toUrlencodedBase64() { | ||
final String b64 = Base64.getUrlEncoder().encodeToString(toBytes()); | ||
return URLEncoder.encode(b64, StandardCharsets.UTF_8); | ||
} | ||
|
||
private void writeVarint(final long val) { | ||
try { | ||
if (val == 0) { | ||
byteBuffer.write(new byte[]{(byte) 0}); | ||
} else { | ||
long v = val; | ||
while (v != 0) { | ||
byte b = (byte) (v & 0x7f); | ||
v >>= 7; | ||
|
||
if (v != 0) { | ||
b |= (byte) 0x80; | ||
} | ||
byteBuffer.write(new byte[]{b}); | ||
} | ||
} | ||
} catch (final IOException e) { | ||
throw new RuntimeException(e); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Side note: It may be better to use |
||
} | ||
} | ||
|
||
private void field(final int field, final byte wire) { | ||
final long fbits = ((long) field) << 3; | ||
final long wbits = ((long) wire) & 0x07; | ||
final long val = fbits | wbits; | ||
writeVarint(val); | ||
} | ||
|
||
public void varint(final int field, final long val) { | ||
field(field, (byte) 0); | ||
writeVarint(val); | ||
} | ||
|
||
public void string(final int field, final String string) { | ||
final byte[] strBts = string.getBytes(StandardCharsets.UTF_8); | ||
bytes(field, strBts); | ||
} | ||
|
||
public void bytes(final int field, final byte[] bytes) { | ||
field(field, (byte) 2); | ||
writeVarint(bytes.length); | ||
try { | ||
byteBuffer.write(bytes); | ||
} catch (final IOException e) { | ||
throw new RuntimeException(e); | ||
} | ||
} | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,18 @@ | ||
package org.schabi.newpipe.extractor.utils; | ||
|
||
import org.junit.jupiter.api.Test; | ||
|
||
import static org.junit.jupiter.api.Assertions.assertEquals; | ||
|
||
public class ProtoBuilderTest { | ||
@Test | ||
public void testProtoBuilder() { | ||
final ProtoBuilder pb = new ProtoBuilder(); | ||
pb.varint(1, 128); | ||
pb.varint(2, 1234567890); | ||
pb.varint(3, 1234567890123456789L); | ||
pb.string(4, "Hello"); | ||
pb.bytes(5, new byte[]{1, 2, 3}); | ||
assertEquals("CIABENKF2MwEGJWCpu_HnoSRESIFSGVsbG8qAwECAw%3D%3D", pb.toUrlencodedBase64()); | ||
} | ||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The device used here is an iPhone 15 Pro Max but the comments are incorrectly update to "iPhone 16".
Either change/revert the comments to iPhone 15 Pro Max or update the device model to "iPhone17,2" to match the iPhone 16 Pro Max.
The mapping can be found here: https://gist.github.com/adamawolf/3048717
or: https://theapplewiki.com/wiki/Firmware/iPhone/18.x#iPhone_16_Pro_Max