diff --git a/SABR_COMPATIBILITY_PROFILES.md b/SABR_COMPATIBILITY_PROFILES.md new file mode 100644 index 00000000..4842e745 --- /dev/null +++ b/SABR_COMPATIBILITY_PROFILES.md @@ -0,0 +1,160 @@ +# SABR compatibility profiles + +SABR compatibility profiles describe volatile WEB and MWEB protocol layouts. They are data, not +programs. PipePipeExtractor owns the native engine, validates every profile value, and exposes only +predefined sources, normalized response targets, predicates, and actions. + +Profiles cannot perform network requests, read arbitrary process state, inspect media payloads, +access cookies or PO tokens directly, allocate unbounded state, define functions, or execute code. +The Host still owns transport, URL validation, request limits, tokens, cookies, UMP framing, media +assembly, retries, generations, seek, cache, and download behavior. + +## Document + +The UTF-8 JSON envelope is strict: malformed UTF-8 plus unknown and missing properties are +rejected. The maximum size is 1 MiB, nesting depth is 16, and structural token count is 8192. + +```json +{ + "format": 1, + "revision": 12, + "validFromMs": 1784678400000, + "validUntilMs": 1792454400000, + "minimumExtractorRevision": 1, + "capabilities": [ + "request-template-v1", + "response-schema-v1", + "recovery-rules-v1" + ], + "clients": { + "MWEB": { + "mediaParts": {"header": 20, "payload": 21, "end": 22}, + "initialRequest": [ + {"field": 1, "wireType": "BYTES", "source": "CLIENT_ABR_STATE", + "required": true}, + {"field": 2, "wireType": "BYTES", "source": "SELECTED_FORMATS", + "required": false}, + {"field": 3, "wireType": "BYTES", "source": "BUFFERED_RANGES", + "required": false}, + {"field": 4, "wireType": "VARINT", "source": "PLAYER_TIME_MS", + "required": false}, + {"field": 5, "wireType": "BYTES", "source": "USTREAMER_CONFIG", + "required": true}, + {"field": 16, "wireType": "BYTES", "source": "PREFERRED_AUDIO_FORMATS", + "required": false}, + {"field": 17, "wireType": "BYTES", "source": "PREFERRED_VIDEO_FORMATS", + "required": false}, + {"field": 19, "wireType": "BYTES", "source": "CLIENT_CONTEXT", + "required": true} + ], + "followingRequest": [ + {"field": 1, "wireType": "BYTES", "source": "CLIENT_ABR_STATE", + "required": true}, + {"field": 2, "wireType": "BYTES", "source": "SELECTED_FORMATS", + "required": false}, + {"field": 3, "wireType": "BYTES", "source": "BUFFERED_RANGES", + "required": false}, + {"field": 4, "wireType": "VARINT", "source": "PLAYER_TIME_MS", + "required": false}, + {"field": 5, "wireType": "BYTES", "source": "USTREAMER_CONFIG", + "required": true}, + {"field": 16, "wireType": "BYTES", "source": "PREFERRED_AUDIO_FORMATS", + "required": false}, + {"field": 17, "wireType": "BYTES", "source": "PREFERRED_VIDEO_FORMATS", + "required": false}, + {"field": 19, "wireType": "BYTES", "source": "CLIENT_CONTEXT", + "required": true} + ], + "responseMappings": [ + {"partType": 35, "target": "NEXT_REQUEST.BACKOFF_MS", "path": [4], + "wireType": "VARINT", "required": false}, + {"partType": 20, "target": "MEDIA_HEADER.SEQUENCE", "path": [9], + "wireType": "VARINT", "required": true} + ], + "recovery": { + "maximumOmissions": 3, + "maximumElapsedMs": 15000, + "forwardThresholdMs": 30000, + "retryDelayMs": 0 + }, + "rules": [ + {"whenAll": ["HAS_ERROR"], "actions": ["FAIL_SESSION"]}, + {"whenAll": ["RELOAD_REQUESTED"], "actions": ["TRY_RELOAD"]}, + {"whenAll": ["HAS_PROTECTION_BOUNDARY", "FETCH_SEGMENT_MODE"], + "actions": ["REQUIRE_PO_TOKEN", "RETRY"]}, + {"whenAll": ["HAS_PROTECTION_BOUNDARY", "PUMP_MODE"], + "actions": ["REFRESH_PO_TOKEN", "CONTINUE"]}, + {"whenAll": ["HAS_REDIRECT"], "actions": ["APPLY_REDIRECT", "CONTINUE"]}, + {"whenAll": [], "actions": ["CONTINUE"]} + ] + } + }, + "keyId": "current", + "signature": "base64-ed25519-signature" +} +``` + +The Ed25519 signature covers the deterministic binary serialization produced by +`SabrCompatibilityProfile.serialize()`, not the JSON whitespace or property order. A profile is +accepted only for its validity interval, supported format and capabilities, compatible Extractor +revision, embedded signing key, and positive monotonic revision. + +The canonical payload writes the public client, wire-type, source, target, predicate, and action +identifiers as UTF-8 strings. It never depends on Java enum ordinals, so adding or reordering an +internal enum constant cannot silently change an existing signature payload. + +## Request templates + +Every field has a protobuf number, a matching `VARINT` or `BYTES` wire type, an enum source, and a +required flag. Initial and following templates are independent and contain at most 64 fields. +Supported sources are: + +- `PLAYER_TIME_MS` +- `BUFFERED_RANGES` +- `PLAYBACK_COOKIE` +- `PO_TOKEN` +- `SELECTED_FORMATS` +- `CLIENT_CONTEXT` +- `CLIENT_ABR_STATE` +- `USTREAMER_CONFIG` +- `PREFERRED_AUDIO_FORMATS` +- `PREFERRED_VIDEO_FORMATS` + +The profile receives the encoded value selected by the Host; it never receives an accessor for the +underlying secret or mutable stream state. The generated request remains subject to the Host's +256 KiB limit. + +## Response mappings + +A mapping identifies a UMP part type, a protobuf field path of at most eight elements, an expected +wire type, and one normalized target. When its UMP part is present, a missing required value fails +the profile and triggers the circuit breaker. Optional missing values leave the native value +unchanged. + +The native media-header baseline ignores known field numbers whose wire type no longer matches the +bundled schema before applying profile mappings. This lets a profile deliberately reuse an old +field number with a new protobuf wire type without failing in the baseline decoder first. + +Targets cover next-request timing and playback state, redirects, errors, reload and protection +signals, plus all normalized media-header properties. Media payload and media-end parts cannot be +inspected by mappings. Media header, payload, and end UMP types are selected once per session and +the payload continues through the native streaming collector without JSON or policy evaluation. + +## Recovery and behavior + +Recovery values are bounded by native limits: at most 16 omissions, 120 seconds elapsed, 300 +seconds forward threshold, and 5 seconds retry delay. Rules contain only known predicates and +actions, have no loops, and must end in exactly one terminal action. A final unconditional +`CONTINUE` rule is mandatory. + +The Host validates every decision again before applying it. Redirects, token refreshes, retries, +backoff, reloads, generations, request count, and all other hard limits remain native invariants. + +## Compatibility boundary + +A profile can adapt known request fields, response paths, UMP media part types, recovery thresholds, +and combinations of predefined actions. A new transport, cryptographic primitive, token flow, +media framing model, or Host capability is structural and requires an Extractor release. + +A document may carry only WEB or only MWEB. Sessions for a client absent from that valid document +stay on the bundled native policy without disabling the profile for the client it does cover. diff --git a/SABR_JAVASCRIPT_POLICY.md b/SABR_JAVASCRIPT_POLICY.md deleted file mode 100644 index 0ef37a88..00000000 --- a/SABR_JAVASCRIPT_POLICY.md +++ /dev/null @@ -1,124 +0,0 @@ -# SABR JavaScript policy contract - -A policy payload contains signed JavaScript source. It must define one global function: - -```js -function createSabrPolicy(sabr) { - return { - describe: function () {}, - initialRequest: function (event) {}, - followUpRequest: function (event) {}, - response: function (event) {}, - demandRoute: function (event) {}, - demandResponse: function (event) {}, - mediaHeader: function (event) {} - }; -} -``` - -Each playback or download session gets a new policy object. Properties stored on that object are -session state and survive calls. PipePipe Client uses QuickJS with no Java object bindings and with -bounded native memory, stack, input, and output sizes. The current binding does not expose -QuickJS's interrupt handler, so execution time is not bounded by the runtime; signed policies must -be tested for termination before release. - -`sabr.base64.decode(string)` returns an unsigned byte array and `sabr.base64.encode(bytes)` returns a -string. `sabr.proto.decode(bytes)` returns protobuf fields shaped as `{n, w, v}` for varints or -`{n, w, b}` for byte/fixed fields. `sabr.proto.encode(fields)` performs the inverse operation. A -policy can include any additional ordinary JavaScript helpers it needs. - -`describe()` returns media envelope part types: - -```js -{demand: true, - media: {headerType: 20, mediaType: 21, endType: 22, headerDecoder: "builtin"}} -``` - -`headerDecoder: "builtin"` keeps the Host's existing protobuf media-header decoder when that -schema has not changed, avoiding JavaScript on the per-segment hot path. Omit it when the policy -needs `mediaHeader(event)` to implement a changed header schema. - -Set `demand: true` only when the policy implements both demand methods below. If it is absent or -false, the Host uses the bundled demand behavior, preserving compatibility with older signed -policies. - -Request methods receive `requestNumber`, recovery counters, player/buffer metrics, and -`fallbackBody`, the Base64-encoded builtin request. They return `{body: "..."}`. The script may use, -patch, or completely replace the builtin protobuf. - -`response(event)` receives the recovery state, mode, media segment count, a bounded array of raw -non-media UMP parts (`{type, data}`), and a `builtin` diagnostic interpretation. It returns an -ordered `actions` array and optional `backoffMs`, `redirectUrl`, `errorDetails`, and recovery -`state`. Actions are validated by the Host before execution; the only available capabilities are -the values in `SabrSessionPolicy.ActionType`. - -To replace response schemas, return `APPLY_RESPONSE_STATE` and a normalized `statePatch` object. -The patch may contain `nextRequest`, `live`, `formats`, `contexts`, and `contextPolicy`. This lets a -policy update playback cookies and pacing, live heads, format initialization metadata, and SABR -contexts without constructing Java protocol objects. Array counts and byte values are bounded by -the Host. `APPLY_BUILTIN_RESPONSE_STATE` remains available for policies that only replace control -decisions and still use the bundled Java response decoder. - -```js -{ - actions: ['APPLY_RESPONSE_STATE', 'CONTINUE'], - statePatch: { - nextRequest: { - targetAudioReadaheadMs: 15000, - targetVideoReadaheadMs: 15000, - playbackCookie: 'base64...' - }, - live: [{headSequenceNumber: 42, headTimeMs: 210000, postLiveDvr: false}], - formats: [{itag: 251, endSegmentNumber: 120, durationUnits: 600, - durationTimescale: 1}], - contexts: [{type: 7, scope: 1, value: 'base64...', sendByDefault: true, - writePolicy: 1}], - contextPolicy: {start: [7], stop: [], discard: []} - } -} -``` - -`mediaHeader({data})` receives a Base64 protobuf payload and returns a normalized media descriptor. -`headerId` and `itag` are required. Optional properties match the getters on `SabrMediaHeader`. -Media payload bytes never enter JavaScript. - -`demandRoute(event)` chooses how to request a segment synchronously demanded by a reader. The event -contains `targetItag`, `targetSequenceNumber`, `targetStartMs`, `bufferedEdgeMs`, `createdAtMs`, -`nowMs`, `elapsedMs`, `responsesWithoutDemandedSegment`, and `recoveryCount`. It returns one route: -`STREAM`, `REWIND`, `FORWARD`, `RECOVER_REWIND`, `RECOVER_FORWARD`, or `RECOVER_MISSING`. -Recovery routes reset the corresponding request state before retrying; the Host records each -recovery and rejects inconsistent counters. - -`demandResponse(event)` runs after a media-bearing SABR response that did not return the exact -demanded segment. Control-only responses remain owned by `response(event)` and do not consume the -demand omission budget. The event receives the same fields plus `segmentCount`, `targetTrackSegmentCount`, -`returnedSegmentsTruncated`, and up to 64 payload-free returned media identities shaped as -`{itag, sequenceNumber, startMs, durationMs}`. It returns `{outcome, retryDelayMs}`. Outcomes are -`CONTINUE`, `FAIL_REPEATED_TARGET_OMISSION`, and `FAIL_NO_TARGET_MEDIA`; retry delay is bounded to -0..5000 ms and must be zero for a terminal outcome. Server backoff intervals in which no request -was sent are not counted as omissions. - -Both demand methods execute in the same policy object as request/response handling, so policy -session state, protocol revision, failover, signature validation, and diagnostics remain atomic. - -The canonical signed payload is produced with `new SabrScriptPolicy(revision, validFromMs, -validUntilMs, source).serialize()`. Sign those exact bytes with Ed25519, then publish a single UTF-8 -JSON document using `SabrScriptPolicyDocument.encode(policy, signature)`: - -```json -{ - "format": 1, - "revision": 1001, - "validFromMs": 1784390400000, - "validUntilMs": 1792166400000, - "source": "function createSabrPolicy(sabr) { /* ... */ }", - "signature": "base64-encoded Ed25519 signature" -} -``` - -The client reconstructs the canonical payload from the JSON fields before signature verification, -so whitespace and object-key order in the delivery document are not security-sensitive. `format`, -`revision`, `validFromMs`, and `validUntilMs` must be exact JSON integers: fractional values, -scientific notation, strings, and values outside the signed 64-bit range are rejected rather than -coerced. Document versioning only describes the source envelope; protocol behavior remains -JavaScript rather than a second serialized DSL. diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/NewPipe.java b/extractor/src/main/java/org/schabi/newpipe/extractor/NewPipe.java index 6e1efd59..5bf5ef34 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/NewPipe.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/NewPipe.java @@ -173,6 +173,7 @@ public static String getYoutubePlayerClient() { public static void setYoutubePlayerClient(final String youtubePlayerClient) { if ("mweb".equals(youtubePlayerClient) + || "web".equals(youtubePlayerClient) || "web_safari".equals(youtubePlayerClient) || "android_vr".equals(youtubePlayerClient) || "tv_downgraded".equals(youtubePlayerClient)) { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/BuiltinSabrSessionPolicy.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/BuiltinSabrSessionPolicy.java index 96ffbae7..7c7aac00 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/BuiltinSabrSessionPolicy.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/BuiltinSabrSessionPolicy.java @@ -4,7 +4,7 @@ import java.util.ArrayList; import java.util.List; -/** Bundled protocol behavior used when no verified JavaScript policy is active. */ +/** Bundled protocol behavior used when no verified compatibility profile is active. */ public final class BuiltinSabrSessionPolicy implements SabrSessionPolicy { @Nonnull @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/CompatibilitySabrSessionPolicy.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/CompatibilitySabrSessionPolicy.java new file mode 100644 index 00000000..399222f0 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/CompatibilitySabrSessionPolicy.java @@ -0,0 +1,105 @@ +package org.schabi.newpipe.extractor.services.youtube.sabr; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +/** Selects the signed WEB or MWEB compatibility variant once for a SABR session. */ +public final class CompatibilitySabrSessionPolicy implements SabrSessionPolicy { + @Nonnull private final SabrCompatibilityProfile profile; + @Nonnull private final SabrMediaProtocol mediaProtocol = new SabrMediaProtocol() { + @Override + public int getHeaderPartType() { + return selectedMediaProtocol().getHeaderPartType(); + } + + @Override + public int getMediaPartType() { + return selectedMediaProtocol().getMediaPartType(); + } + + @Override + public int getEndPartType() { + return selectedMediaProtocol().getEndPartType(); + } + + @Nonnull + @Override + public SabrMediaHeader decodeHeader(@Nonnull final byte[] payload) + throws SabrProtocolException { + return selectedMediaProtocol().decodeHeader(payload); + } + }; + @Nullable private SabrSessionPolicy selected; + + public CompatibilitySabrSessionPolicy(@Nonnull final SabrCompatibilityProfile profile) { + this.profile = profile; + } + + @Nonnull + @Override + public synchronized Result evaluate(@Nonnull final State state, @Nonnull final Event event) + throws SabrProtocolException { + return select(event).evaluate(state, event); + } + + @Nonnull + @Override + public synchronized DemandRoute evaluateDemandRoute(@Nonnull final DemandRouteEvent event) + throws SabrProtocolException { + return requireSelected().evaluateDemandRoute(event); + } + + @Nonnull + @Override + public synchronized DemandResponseDecision evaluateDemandResponse( + @Nonnull final DemandResponseEvent event) throws SabrProtocolException { + return requireSelected().evaluateDemandResponse(event); + } + + @Nonnull + @Override + public SabrMediaProtocol getMediaProtocol() { + return mediaProtocol; + } + + @Nonnull + private SabrSessionPolicy select(@Nonnull final Event event) + throws SabrProtocolException { + if (selected != null) { + return selected; + } + if (!(event instanceof RequestEvent)) { + throw new SabrProtocolException("SABR profile client is not selected"); + } + final RequestEvent request = (RequestEvent) event; + final YoutubeSabrClientProfile client = request.getProfileRequestData() + .getClientProfile(); + final SabrCompatibilityProfileClient configured = profile.getClient(client); + if (configured == null) { + selected = new BuiltinSabrSessionPolicy(); + return selected; + } + selected = new ProfiledSabrSessionPolicy(configured); + return selected; + } + + @Nonnull + private SabrSessionPolicy requireSelected() throws SabrProtocolException { + if (selected == null) { + throw new SabrProtocolException("SABR profile client is not selected"); + } + return selected; + } + + @Nonnull + private synchronized SabrMediaProtocol selectedMediaProtocol() { + return selected == null ? SabrMediaProtocol.builtin() : selected.getMediaProtocol(); + } + + @Override + public synchronized void close() { + if (selected != null) { + selected.close(); + } + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/FallbackSabrSessionPolicy.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/FallbackSabrSessionPolicy.java new file mode 100644 index 00000000..5d651b53 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/FallbackSabrSessionPolicy.java @@ -0,0 +1,118 @@ +package org.schabi.newpipe.extractor.services.youtube.sabr; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.util.concurrent.atomic.AtomicBoolean; + +/** Session-local circuit breaker which falls back atomically to bundled behavior. */ +public final class FallbackSabrSessionPolicy implements SabrSessionPolicy { + public interface FailureListener { + void onProfileDisabled(@Nonnull Throwable failure); + } + + @Nonnull private final SabrSessionPolicy profile; + @Nonnull private final SabrSessionPolicy fallback; + @Nullable private final FailureListener listener; + @Nonnull private final AtomicBoolean disabled = new AtomicBoolean(); + + public FallbackSabrSessionPolicy(@Nonnull final SabrSessionPolicy profile, + @Nullable final FailureListener listener) { + this(profile, new BuiltinSabrSessionPolicy(), listener); + } + + public FallbackSabrSessionPolicy(@Nonnull final SabrSessionPolicy profile, + @Nonnull final SabrSessionPolicy fallback, + @Nullable final FailureListener listener) { + this.profile = profile; + this.fallback = fallback; + this.listener = listener; + } + + @Nonnull + @Override + public Result evaluate(@Nonnull final State state, @Nonnull final Event event) + throws SabrProtocolException { + if (disabled.get()) return fallback.evaluate(state, event); + try { + return profile.evaluate(state, event); + } catch (final SabrProtocolException | RuntimeException failure) { + disable(failure); + return fallback.evaluate(state, event); + } + } + + @Nonnull + @Override + public DemandRoute evaluateDemandRoute(@Nonnull final DemandRouteEvent event) + throws SabrProtocolException { + if (disabled.get()) return fallback.evaluateDemandRoute(event); + try { + return profile.evaluateDemandRoute(event); + } catch (final SabrProtocolException | RuntimeException failure) { + disable(failure); + return fallback.evaluateDemandRoute(event); + } + } + + @Nonnull + @Override + public DemandResponseDecision evaluateDemandResponse( + @Nonnull final DemandResponseEvent event) throws SabrProtocolException { + if (disabled.get()) return fallback.evaluateDemandResponse(event); + try { + return profile.evaluateDemandResponse(event); + } catch (final SabrProtocolException | RuntimeException failure) { + disable(failure); + return fallback.evaluateDemandResponse(event); + } + } + + @Nonnull + @Override + public SabrMediaProtocol getMediaProtocol() { + if (disabled.get()) { + return fallback.getMediaProtocol(); + } + final SabrMediaProtocol selected = profile.getMediaProtocol(); + final int headerPartType = selected.getHeaderPartType(); + final int mediaPartType = selected.getMediaPartType(); + final int endPartType = selected.getEndPartType(); + return new SabrMediaProtocol() { + @Override public int getHeaderPartType() { return headerPartType; } + @Override public int getMediaPartType() { return mediaPartType; } + @Override public int getEndPartType() { return endPartType; } + + @Nonnull + @Override + public SabrMediaHeader decodeHeader(@Nonnull final byte[] payload) + throws SabrProtocolException { + try { + return selected.decodeHeader(payload); + } catch (final SabrProtocolException | RuntimeException failure) { + disable(failure); + throw new SabrRecoverableException( + "SABR compatibility profile media decoding failed", failure); + } + } + }; + } + + public boolean isDisabled() { + return disabled.get(); + } + + private void disable(@Nonnull final Throwable failure) { + if (disabled.compareAndSet(false, true) && listener != null) { + listener.onProfileDisabled(failure); + } + } + + @Override + public void close() { + try { + profile.close(); + } finally { + fallback.close(); + } + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/ProfiledSabrMediaProtocol.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/ProfiledSabrMediaProtocol.java new file mode 100644 index 00000000..2c705154 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/ProfiledSabrMediaProtocol.java @@ -0,0 +1,25 @@ +package org.schabi.newpipe.extractor.services.youtube.sabr; + +import javax.annotation.Nonnull; + +/** Profile-selected UMP envelope with native bounded header decoding. */ +final class ProfiledSabrMediaProtocol implements SabrMediaProtocol { + @Nonnull private final SabrCompatibilityProfileClient.MediaParts mediaParts; + @Nonnull private final SabrProfileMediaHeaderMapper headerMapper; + + ProfiledSabrMediaProtocol(@Nonnull final SabrCompatibilityProfileClient client) { + mediaParts = client.getMediaParts(); + headerMapper = new SabrProfileMediaHeaderMapper(client.getResponseMappings()); + } + + @Override public int getHeaderPartType() { return mediaParts.getHeader(); } + @Override public int getMediaPartType() { return mediaParts.getPayload(); } + @Override public int getEndPartType() { return mediaParts.getEnd(); } + + @Nonnull + @Override + public SabrMediaHeader decodeHeader(@Nonnull final byte[] payload) + throws SabrProtocolException { + return headerMapper.decode(payload); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/ProfiledSabrSessionPolicy.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/ProfiledSabrSessionPolicy.java new file mode 100644 index 00000000..cb9f9fcc --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/ProfiledSabrSessionPolicy.java @@ -0,0 +1,177 @@ +package org.schabi.newpipe.extractor.services.youtube.sabr; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.util.ArrayList; +import java.util.List; + +/** Native evaluator for one immutable declarative compatibility client. */ +public final class ProfiledSabrSessionPolicy implements SabrSessionPolicy { + @Nonnull private final SabrCompatibilityProfileClient client; + @Nonnull private final SabrProfileResponseMapper responseMapper; + @Nonnull private final SabrMediaProtocol mediaProtocol; + + public ProfiledSabrSessionPolicy(@Nonnull final SabrCompatibilityProfileClient client) { + this.client = client; + responseMapper = new SabrProfileResponseMapper(client.getResponseMappings()); + mediaProtocol = new ProfiledSabrMediaProtocol(client); + } + + @Nonnull + @Override + public Result evaluate(@Nonnull final State state, @Nonnull final Event event) + throws SabrProtocolException { + if (event instanceof RequestEvent) { + final List template = state.getRequestNumber() == 0 + ? client.getInitialRequest() : client.getFollowingRequest(); + return Result.request(state, state.getRequestNumber() == 0 + ? ActionType.SEND_INITIAL_REQUEST + : ActionType.SEND_FOLLOW_UP_REQUEST, + ((RequestEvent) event).buildProfileRequest(template)); + } + final ControlResponseEvent control = (ControlResponseEvent) event; + final SabrProfileMappedResponse response = responseMapper.map(control.getResponse()); + final SabrProfileRule rule = selectRule(control, response); + final List actions = new ArrayList<>(); + actions.add(new Action(ActionType.APPLY_RESPONSE_STATE)); + State next = state; + boolean profileRedirect = false; + for (final SabrProfileRule.Action configured : rule.getActions()) { + final ActionType action = actionType(configured); + if (action == ActionType.APPLY_REDIRECT) { + final String redirectUrl = response.getRedirectUrl(); + if (redirectUrl == null || redirectUrl.isEmpty()) { + throw new SabrProtocolException("SABR profile applied an absent redirect"); + } + profileRedirect = true; + next = new State(state.getRequestNumber(), state.getRedirectCount() + 1, + state.getPoTokenRefreshes(), state.getReloads()); + } + if (!isTerminal(action)) { + actions.add(new Action(action)); + } + } + if (control.getMode() == ControlMode.PUMP && control.getSegmentCount() > 0) { + next = new State(next.getRequestNumber(), 0, 0, next.getReloads()); + actions.add(new Action(ActionType.RESET_RECOVERY_BUDGETS)); + } + final int backoff = Math.max(0, response.getBackoffMs()); + if (backoff > 0) { + actions.add(new Action(control.shouldHonorBackoff() + ? ActionType.SLEEP_BACKOFF : ActionType.DEFER_BACKOFF)); + } else if (!control.shouldHonorBackoff()) { + actions.add(new Action(ActionType.CLEAR_DEMAND_BACKOFF)); + } + for (final SabrProfileRule.Action configured : rule.getActions()) { + final ActionType action = actionType(configured); + if (isTerminal(action)) { + actions.add(new Action(action)); + } + } + final String error = actions.get(actions.size() - 1).getType() + == ActionType.FAIL_SABR_ERROR + ? response.getErrorDetails() == null + ? "SABR compatibility profile terminated the session" + : response.getErrorDetails() + : null; + return Result.control(next, actions, new ControlDecision(backoff, + profileRedirect ? response.getRedirectUrl() : null, error), + response.getStatePatch()); + } + + @Nonnull + private SabrProfileRule selectRule(@Nonnull final ControlResponseEvent control, + @Nonnull final SabrProfileMappedResponse response) + throws SabrProtocolException { + for (final SabrProfileRule rule : client.getRules()) { + boolean matches = true; + for (final SabrProfileRule.Predicate predicate : rule.getPredicates()) { + if (!matches(predicate, control, response)) { + matches = false; + break; + } + } + if (matches) return rule; + } + throw new SabrProtocolException("SABR profile has no matching behavior rule"); + } + + private static boolean matches(@Nonnull final SabrProfileRule.Predicate predicate, + @Nonnull final ControlResponseEvent control, + @Nonnull final SabrProfileMappedResponse response) { + switch (predicate) { + case HAS_MEDIA: return control.getResponse().hasMedia(); + case HAS_REDIRECT: + return response.getRedirectUrl() != null && !response.getRedirectUrl().isEmpty(); + case HAS_PROTECTION_BOUNDARY: return response.getProtectionStatus() >= 2; + case DEMANDED_SEGMENT_MISSING: + return control.getMode() == ControlMode.FETCH_SEGMENT + && control.getSegmentCount() == 0; + case BACKOFF_PRESENT: return response.getBackoffMs() > 0; + case HAS_ERROR: return response.getErrorDetails() != null; + case RELOAD_REQUESTED: return response.isReloadRequested(); + case PUMP_MODE: return control.getMode() == ControlMode.PUMP; + case FETCH_SEGMENT_MODE: return control.getMode() == ControlMode.FETCH_SEGMENT; + default: return false; + } + } + + @Nonnull + private static ActionType actionType(@Nonnull final SabrProfileRule.Action action) { + switch (action) { + case CONTINUE: return ActionType.CONTINUE; + case RETRY: return ActionType.RETRY; + case APPLY_REDIRECT: return ActionType.APPLY_REDIRECT; + case REFRESH_PO_TOKEN: return ActionType.REFRESH_PO_TOKEN; + case REQUIRE_PO_TOKEN: return ActionType.REQUIRE_PO_TOKEN; + case FAIL_SESSION: return ActionType.FAIL_SABR_ERROR; + case TRY_RELOAD: return ActionType.TRY_RELOAD; + default: throw new IllegalArgumentException("Unsupported SABR profile action"); + } + } + + private static boolean isTerminal(@Nonnull final ActionType action) { + return action == ActionType.CONTINUE || action == ActionType.RETRY + || action == ActionType.FAIL_SABR_ERROR || action == ActionType.TRY_RELOAD; + } + + @Nonnull + @Override + public DemandRoute evaluateDemandRoute(@Nonnull final DemandRouteEvent event) { + final SabrProfileRecovery recovery = client.getRecovery(); + final DemandState demand = event.getState(); + final boolean recovering = demand.getResponsesWithoutDemandedSegment() + > demand.getRecoveryCount(); + if (event.getTargetStartMs() < event.getBufferedEdgeMs()) { + return recovering ? DemandRoute.RECOVER_REWIND : DemandRoute.REWIND; + } + if (event.getTargetStartMs() + > event.getBufferedEdgeMs() + recovery.getForwardThresholdMs()) { + return recovering ? DemandRoute.RECOVER_FORWARD : DemandRoute.FORWARD; + } + return recovering ? DemandRoute.RECOVER_MISSING : DemandRoute.STREAM; + } + + @Nonnull + @Override + public DemandResponseDecision evaluateDemandResponse( + @Nonnull final DemandResponseEvent event) { + final SabrProfileRecovery recovery = client.getRecovery(); + final DemandState demand = event.getState(); + if (demand.getResponsesWithoutDemandedSegment() >= recovery.getMaximumOmissions()) { + return new DemandResponseDecision(DemandOutcome.FAIL_REPEATED_TARGET_OMISSION, 0); + } + if (demand.getElapsedMs() >= recovery.getMaximumElapsedMs()) { + return new DemandResponseDecision(event.getTargetTrackSegmentCount() > 0 + ? DemandOutcome.FAIL_REPEATED_TARGET_OMISSION + : DemandOutcome.FAIL_NO_TARGET_MEDIA, 0); + } + return new DemandResponseDecision(DemandOutcome.CONTINUE, recovery.getRetryDelayMs()); + } + + @Nonnull + @Override + public SabrMediaProtocol getMediaProtocol() { + return mediaProtocol; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrCompatibilityProfile.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrCompatibilityProfile.java new file mode 100644 index 00000000..3cc0ba06 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrCompatibilityProfile.java @@ -0,0 +1,171 @@ +package org.schabi.newpipe.extractor.services.youtube.sabr; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.EnumMap; +import java.util.EnumSet; +import java.util.List; +import java.util.Map; + +/** Signed, immutable data contract for volatile WEB and MWEB SABR details. */ +public final class SabrCompatibilityProfile { + static final int MAX_PROTOBUF_FIELD_NUMBER = (1 << 29) - 1; + static final int MAX_UMP_PART_TYPE = 65_535; + public static final int FORMAT_VERSION = 1; + public static final int EXTRACTOR_REVISION = 1; + private static final int MAGIC = 0x53435031; + + public enum Capability { + REQUEST_TEMPLATE_V1("request-template-v1"), + RESPONSE_SCHEMA_V1("response-schema-v1"), + RECOVERY_RULES_V1("recovery-rules-v1"); + + @Nonnull private final String id; + + Capability(@Nonnull final String id) { + this.id = id; + } + + @Nonnull + public String getId() { + return id; + } + + @Nonnull + static Capability fromId(@Nonnull final String id) { + for (final Capability capability : values()) { + if (capability.id.equals(id)) { + return capability; + } + } + throw new IllegalArgumentException("Unsupported SABR profile capability: " + id); + } + } + + private final long revision; + private final long validFromMs; + private final long validUntilMs; + private final int minimumExtractorRevision; + @Nonnull private final EnumSet capabilities; + @Nonnull private final Map clients; + @Nonnull private final byte[] canonicalPayload; + + public SabrCompatibilityProfile( + final long revision, + final long validFromMs, + final long validUntilMs, + final int minimumExtractorRevision, + @Nonnull final EnumSet capabilities, + @Nonnull final Map clients) { + if (revision <= 0 || validFromMs < 0 || validUntilMs <= validFromMs + || minimumExtractorRevision < 1 + || minimumExtractorRevision > EXTRACTOR_REVISION + || !capabilities.containsAll(EnumSet.allOf(Capability.class)) + || clients.isEmpty() || clients.size() > 2) { + throw new IllegalArgumentException("Invalid SABR compatibility profile metadata"); + } + final EnumMap checked = + new EnumMap<>(YoutubeSabrClientProfile.class); + for (final Map.Entry entry + : clients.entrySet()) { + if (entry.getKey() == null || entry.getValue() == null + || entry.getKey() != entry.getValue().getClient()) { + throw new IllegalArgumentException("Invalid SABR compatibility client map"); + } + checked.put(entry.getKey(), entry.getValue()); + } + this.revision = revision; + this.validFromMs = validFromMs; + this.validUntilMs = validUntilMs; + this.minimumExtractorRevision = minimumExtractorRevision; + this.capabilities = EnumSet.copyOf(capabilities); + this.clients = Collections.unmodifiableMap(checked); + canonicalPayload = encodeCanonical(); + } + + public long getRevision() { + return revision; + } + + public long getValidFromMs() { + return validFromMs; + } + + public long getValidUntilMs() { + return validUntilMs; + } + + public int getMinimumExtractorRevision() { + return minimumExtractorRevision; + } + + @Nonnull + public EnumSet getCapabilities() { + return EnumSet.copyOf(capabilities); + } + + @Nonnull + public Map getClients() { + return clients; + } + + @Nullable + public SabrCompatibilityProfileClient getClient( + @Nonnull final YoutubeSabrClientProfile client) { + return clients.get(client); + } + + public boolean isValidAt(final long nowMs) { + return nowMs >= validFromMs && nowMs < validUntilMs; + } + + @Nonnull + public byte[] serialize() { + return canonicalPayload.clone(); + } + + @Nonnull + private byte[] encodeCanonical() { + try { + final ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + final DataOutputStream output = new DataOutputStream(bytes); + output.writeInt(MAGIC); + output.writeByte(FORMAT_VERSION); + output.writeLong(revision); + output.writeLong(validFromMs); + output.writeLong(validUntilMs); + output.writeInt(minimumExtractorRevision); + final List sortedCapabilities = new ArrayList<>(capabilities); + sortedCapabilities.sort(Comparator.comparing(Capability::getId)); + output.writeByte(sortedCapabilities.size()); + for (final Capability capability : sortedCapabilities) { + writeString(output, capability.getId()); + } + final List sortedClients = + new ArrayList<>(clients.values()); + sortedClients.sort(Comparator.comparing(value -> value.getClient().getClientName())); + output.writeByte(sortedClients.size()); + for (final SabrCompatibilityProfileClient client : sortedClients) { + client.writeCanonical(output); + } + output.flush(); + return bytes.toByteArray(); + } catch (final IOException impossible) { + throw new IllegalStateException(impossible); + } + } + + static void writeString(@Nonnull final DataOutputStream output, + @Nonnull final String value) throws IOException { + final byte[] encoded = value.getBytes(StandardCharsets.UTF_8); + output.writeInt(encoded.length); + output.write(encoded); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrCompatibilityProfileClient.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrCompatibilityProfileClient.java new file mode 100644 index 00000000..03589819 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrCompatibilityProfileClient.java @@ -0,0 +1,146 @@ +package org.schabi.newpipe.extractor.services.youtube.sabr; + +import javax.annotation.Nonnull; +import java.io.DataOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +/** Immutable WEB or MWEB compatibility variant compiled from a signed document. */ +public final class SabrCompatibilityProfileClient { + public static final class MediaParts { + private final int header; + private final int payload; + private final int end; + + public MediaParts(final int header, final int payload, final int end) { + if (!validPart(header) || !validPart(payload) || !validPart(end) + || header == payload || header == end || payload == end) { + throw new IllegalArgumentException("Invalid SABR media part types"); + } + this.header = header; + this.payload = payload; + this.end = end; + } + + private static boolean validPart(final int value) { + return value >= 0 && value <= SabrCompatibilityProfile.MAX_UMP_PART_TYPE + && (value < 10 || value > 12) && (value < 30 || value > 67); + } + + public int getHeader() { + return header; + } + + public int getPayload() { + return payload; + } + + public int getEnd() { + return end; + } + + void writeCanonical(@Nonnull final DataOutputStream output) throws IOException { + output.writeInt(header); + output.writeInt(payload); + output.writeInt(end); + } + } + + @Nonnull private final YoutubeSabrClientProfile client; + @Nonnull private final MediaParts mediaParts; + @Nonnull private final List initialRequest; + @Nonnull private final List followingRequest; + @Nonnull private final List responseMappings; + @Nonnull private final SabrProfileRecovery recovery; + @Nonnull private final List rules; + + public SabrCompatibilityProfileClient( + @Nonnull final YoutubeSabrClientProfile client, + @Nonnull final MediaParts mediaParts, + @Nonnull final List initialRequest, + @Nonnull final List followingRequest, + @Nonnull final List responseMappings, + @Nonnull final SabrProfileRecovery recovery, + @Nonnull final List rules) { + if (client != YoutubeSabrClientProfile.WEB && client != YoutubeSabrClientProfile.MWEB) { + throw new IllegalArgumentException("Compatibility profiles support WEB and MWEB only"); + } + SabrCompatibilityProfileValidator.validate(initialRequest, followingRequest, + responseMappings, rules, mediaParts); + this.client = client; + this.mediaParts = Objects.requireNonNull(mediaParts); + this.initialRequest = immutableCopy(initialRequest); + this.followingRequest = immutableCopy(followingRequest); + this.responseMappings = immutableCopy(responseMappings); + this.recovery = Objects.requireNonNull(recovery); + this.rules = immutableCopy(rules); + } + + @Nonnull + private static List immutableCopy(@Nonnull final List values) { + return Collections.unmodifiableList(new ArrayList<>(values)); + } + + @Nonnull + public YoutubeSabrClientProfile getClient() { + return client; + } + + @Nonnull + public MediaParts getMediaParts() { + return mediaParts; + } + + @Nonnull + public List getInitialRequest() { + return initialRequest; + } + + @Nonnull + public List getFollowingRequest() { + return followingRequest; + } + + @Nonnull + public List getResponseMappings() { + return responseMappings; + } + + @Nonnull + public SabrProfileRecovery getRecovery() { + return recovery; + } + + @Nonnull + public List getRules() { + return rules; + } + + void writeCanonical(@Nonnull final DataOutputStream output) throws IOException { + SabrCompatibilityProfile.writeString(output, client.getClientName()); + mediaParts.writeCanonical(output); + writeRequestFields(output, initialRequest); + writeRequestFields(output, followingRequest); + output.writeInt(responseMappings.size()); + for (final SabrProfileResponseMapping mapping : responseMappings) { + mapping.writeCanonical(output); + } + recovery.writeCanonical(output); + output.writeInt(rules.size()); + for (final SabrProfileRule rule : rules) { + rule.writeCanonical(output); + } + } + + private static void writeRequestFields(@Nonnull final DataOutputStream output, + @Nonnull final List fields) + throws IOException { + output.writeInt(fields.size()); + for (final SabrProfileRequestField field : fields) { + field.writeCanonical(output); + } + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrCompatibilityProfileDocument.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrCompatibilityProfileDocument.java new file mode 100644 index 00000000..bf980918 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrCompatibilityProfileDocument.java @@ -0,0 +1,85 @@ +package org.schabi.newpipe.extractor.services.youtube.sabr; + +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; +import com.grack.nanojson.JsonParserException; + +import javax.annotation.Nonnull; +import java.nio.ByteBuffer; +import java.nio.charset.CharacterCodingException; +import java.nio.charset.CodingErrorAction; +import java.nio.charset.StandardCharsets; +import java.util.Base64; + +/** Strict JSON delivery envelope for signed declarative compatibility profiles. */ +public final class SabrCompatibilityProfileDocument { + public static final int MAX_DOCUMENT_BYTES = 1024 * 1024; + private static final int ED25519_SIGNATURE_BYTES = 64; + + private SabrCompatibilityProfileDocument() { + } + + @Nonnull + public static Parsed decode(@Nonnull final byte[] encoded) { + if (encoded.length == 0 || encoded.length > MAX_DOCUMENT_BYTES) { + throw new IllegalArgumentException("Invalid SABR compatibility profile size"); + } + try { + final String json = decodeUtf8(encoded); + SabrProfileJson.validateStructure(json); + final JsonObject root = JsonParser.object().from(json); + SabrProfileJson.requireKeys(root, "format", "revision", "validFromMs", + "validUntilMs", "minimumExtractorRevision", "capabilities", "clients", + "keyId", "signature"); + if (SabrProfileJson.requireLong(root, "format") + != SabrCompatibilityProfile.FORMAT_VERSION) { + throw new IllegalArgumentException("Unsupported SABR compatibility profile"); + } + final SabrCompatibilityProfile profile = SabrCompatibilityProfileParser.parse(root); + final String keyId = SabrProfileJson.requireString(root, "keyId"); + if (!keyId.matches("[A-Za-z0-9._-]{1,64}")) { + throw new IllegalArgumentException("Invalid SABR profile key id"); + } + final byte[] signature = Base64.getDecoder().decode( + SabrProfileJson.requireString(root, "signature")); + if (signature.length != ED25519_SIGNATURE_BYTES) { + throw new IllegalArgumentException("Invalid SABR profile signature size"); + } + return new Parsed(profile, keyId, signature); + } catch (final IllegalArgumentException error) { + throw error; + } catch (final JsonParserException | RuntimeException error) { + throw new IllegalArgumentException("Malformed SABR compatibility profile", error); + } + } + + @Nonnull + private static String decodeUtf8(@Nonnull final byte[] encoded) { + try { + return StandardCharsets.UTF_8.newDecoder() + .onMalformedInput(CodingErrorAction.REPORT) + .onUnmappableCharacter(CodingErrorAction.REPORT) + .decode(ByteBuffer.wrap(encoded)).toString(); + } catch (final CharacterCodingException error) { + throw new IllegalArgumentException("Malformed SABR profile UTF-8", error); + } + } + + public static final class Parsed { + @Nonnull private final SabrCompatibilityProfile profile; + @Nonnull private final String keyId; + @Nonnull private final byte[] signature; + + Parsed(@Nonnull final SabrCompatibilityProfile profile, + @Nonnull final String keyId, + @Nonnull final byte[] signature) { + this.profile = profile; + this.keyId = keyId; + this.signature = signature.clone(); + } + + @Nonnull public SabrCompatibilityProfile getProfile() { return profile; } + @Nonnull public String getKeyId() { return keyId; } + @Nonnull public byte[] getSignature() { return signature.clone(); } + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrCompatibilityProfileManager.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrCompatibilityProfileManager.java new file mode 100644 index 00000000..dd72da9e --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrCompatibilityProfileManager.java @@ -0,0 +1,162 @@ +package org.schabi.newpipe.extractor.services.youtube.sabr; + +import org.bouncycastle.crypto.params.Ed25519PublicKeyParameters; +import org.bouncycastle.crypto.signers.Ed25519Signer; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; + +/** Thread-safe key rotation, verification and monotonic profile activation boundary. */ +public final class SabrCompatibilityProfileManager { + private static final int ED25519_PUBLIC_KEY_BYTES = 32; + @Nonnull private final Map publicKeys; + @Nullable private volatile SabrCompatibilityProfile active; + @Nullable private volatile SabrCompatibilityProfile previous; + private long highestRevision; + + public SabrCompatibilityProfileManager(@Nonnull final Map publicKeys, + final long minimumRevision) { + if (publicKeys.isEmpty() || publicKeys.size() > 8 || minimumRevision < 0) { + throw new IllegalArgumentException("Invalid SABR profile verifier"); + } + final Map checked = new LinkedHashMap<>(); + for (final Map.Entry entry : publicKeys.entrySet()) { + if (entry.getKey() == null || !entry.getKey().matches("[A-Za-z0-9._-]{1,64}") + || entry.getValue() == null + || entry.getValue().length != ED25519_PUBLIC_KEY_BYTES) { + throw new IllegalArgumentException("Invalid SABR profile public key"); + } + checked.put(entry.getKey(), entry.getValue().clone()); + } + this.publicKeys = Collections.unmodifiableMap(checked); + highestRevision = minimumRevision; + } + + @Nonnull + public synchronized SabrCompatibilityProfile verifyDocument( + @Nonnull final byte[] document, final long nowMs) { + final SabrCompatibilityProfile profile = verifySignature(document, nowMs); + if (profile.getRevision() < highestRevision + || profile.getRevision() == highestRevision + && (active == null || active.getRevision() != highestRevision)) { + throw new IllegalArgumentException("SABR profile rollback or validity check failed"); + } + return profile; + } + + @Nonnull + private SabrCompatibilityProfile verifySignature( + @Nonnull final byte[] document, final long nowMs) { + final SabrCompatibilityProfileDocument.Parsed parsed = + SabrCompatibilityProfileDocument.decode(document); + final byte[] key = publicKeys.get(parsed.getKeyId()); + if (key == null) { + throw new IllegalArgumentException("Unknown SABR profile signing key"); + } + final SabrCompatibilityProfile profile = parsed.getProfile(); + if (!profile.isValidAt(nowMs)) { + throw new IllegalArgumentException("SABR profile rollback or validity check failed"); + } + final byte[] payload = profile.serialize(); + final Ed25519Signer verifier = new Ed25519Signer(); + verifier.init(false, new Ed25519PublicKeyParameters(key)); + verifier.update(payload, 0, payload.length); + if (!verifier.verifySignature(parsed.getSignature())) { + throw new IllegalArgumentException("Invalid SABR compatibility profile signature"); + } + if (active != null && active.getRevision() == profile.getRevision() + && !Arrays.equals(active.serialize(), profile.serialize())) { + throw new IllegalArgumentException("Conflicting SABR profile revision"); + } + return profile; + } + + /** Restores the normal active generation at or above the monotonic revision floor. */ + @Nonnull + public synchronized SabrCompatibilityProfile restoreDocument( + @Nonnull final byte[] document, final long nowMs) { + final SabrCompatibilityProfile restored = verifySignature(document, nowMs); + if (restored.getRevision() < highestRevision) { + throw new IllegalArgumentException("SABR profile rollback rejected"); + } + active = restored; + previous = null; + highestRevision = restored.getRevision(); + return restored; + } + + /** Restores the signed previous generation only behind a verified active generation. */ + @Nonnull + public synchronized SabrCompatibilityProfile restorePreviousDocument( + @Nonnull final byte[] document, final long nowMs) { + final SabrCompatibilityProfile restored = verifySignature(document, nowMs); + if (active == null || restored.getRevision() >= active.getRevision()) { + throw new IllegalArgumentException("Invalid previous SABR profile generation"); + } + previous = restored; + return restored; + } + + /** Restores a circuit-breaker fallback while retaining a higher revision floor. */ + @Nonnull + public synchronized SabrCompatibilityProfile restoreFallbackDocument( + @Nonnull final byte[] document, final long nowMs) { + final SabrCompatibilityProfile restored = verifySignature(document, nowMs); + if (active != null || restored.getRevision() >= highestRevision) { + throw new IllegalArgumentException("Invalid SABR profile fallback generation"); + } + active = restored; + previous = null; + return restored; + } + + public synchronized void activate(@Nonnull final SabrCompatibilityProfile verified) { + if (verified.getRevision() < highestRevision) { + throw new IllegalArgumentException("SABR profile rollback rejected"); + } + if (active != null && active.getRevision() == verified.getRevision() + && !Arrays.equals(active.serialize(), verified.serialize())) { + throw new IllegalArgumentException("Conflicting SABR profile revision"); + } + if (active != null && active.getRevision() < verified.getRevision()) { + previous = active; + } + active = verified; + highestRevision = verified.getRevision(); + } + + @Nonnull + public synchronized SabrCompatibilityProfile installDocument( + @Nonnull final byte[] document, final long nowMs) { + final SabrCompatibilityProfile verified = verifyDocument(document, nowMs); + activate(verified); + return verified; + } + + @Nullable + public synchronized SabrCompatibilityProfile current(final long nowMs) { + if (active != null && active.isValidAt(nowMs)) { + return active; + } + active = null; + previous = null; + return null; + } + + public synchronized boolean deactivate(@Nonnull final SabrCompatibilityProfile expected) { + if (active == expected) { + active = previous; + previous = null; + return true; + } + return false; + } + + public synchronized long getHighestRevision() { + return highestRevision; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrCompatibilityProfileParser.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrCompatibilityProfileParser.java new file mode 100644 index 00000000..14a9318f --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrCompatibilityProfileParser.java @@ -0,0 +1,178 @@ +package org.schabi.newpipe.extractor.services.youtube.sabr; + +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; + +import javax.annotation.Nonnull; +import java.util.ArrayList; +import java.util.EnumMap; +import java.util.EnumSet; +import java.util.List; +import java.util.Map; + +/** Converts the strict JSON shape into immutable native profile objects. */ +final class SabrCompatibilityProfileParser { + private SabrCompatibilityProfileParser() { + } + + @Nonnull + static SabrCompatibilityProfile parse(@Nonnull final JsonObject root) { + return new SabrCompatibilityProfile(SabrProfileJson.requireLong(root, "revision"), + SabrProfileJson.requireLong(root, "validFromMs"), + SabrProfileJson.requireLong(root, "validUntilMs"), + SabrProfileJson.requireInt(root, "minimumExtractorRevision"), + parseCapabilities(SabrProfileJson.requireArray(root, "capabilities")), + parseClients(SabrProfileJson.requireObject(root, "clients"))); + } + + @Nonnull + private static EnumSet parseCapabilities( + @Nonnull final JsonArray array) { + if (array.isEmpty() + || array.size() > SabrCompatibilityProfile.Capability.values().length) { + throw new IllegalArgumentException("Invalid SABR profile capabilities"); + } + final EnumSet capabilities = + EnumSet.noneOf(SabrCompatibilityProfile.Capability.class); + for (int index = 0; index < array.size(); index++) { + final Object value = array.get(index); + if (!(value instanceof String) + || !capabilities.add(SabrCompatibilityProfile.Capability.fromId( + (String) value))) { + throw new IllegalArgumentException("Duplicate SABR profile capability"); + } + } + return capabilities; + } + + @Nonnull + private static Map parseClients( + @Nonnull final JsonObject object) { + if (object.isEmpty() || object.size() > 2) { + throw new IllegalArgumentException("Invalid SABR compatibility clients"); + } + final EnumMap result = + new EnumMap<>(YoutubeSabrClientProfile.class); + for (final String name : object.keySet()) { + final YoutubeSabrClientProfile client = enumValue( + YoutubeSabrClientProfile.class, name, "compatibility client"); + if (client != YoutubeSabrClientProfile.WEB + && client != YoutubeSabrClientProfile.MWEB) { + throw new IllegalArgumentException( + "Unsupported SABR compatibility client: " + name); + } + result.put(client, parseClient(client, SabrProfileJson.requireObject(object, name))); + } + return result; + } + + @Nonnull + private static SabrCompatibilityProfileClient parseClient( + @Nonnull final YoutubeSabrClientProfile client, + @Nonnull final JsonObject object) { + SabrProfileJson.requireKeys(object, "mediaParts", "initialRequest", "followingRequest", + "responseMappings", "recovery", "rules"); + final JsonObject media = SabrProfileJson.requireObject(object, "mediaParts"); + SabrProfileJson.requireKeys(media, "header", "payload", "end"); + return new SabrCompatibilityProfileClient(client, + new SabrCompatibilityProfileClient.MediaParts( + SabrProfileJson.requireInt(media, "header"), + SabrProfileJson.requireInt(media, "payload"), + SabrProfileJson.requireInt(media, "end")), + parseRequest(SabrProfileJson.requireArray(object, "initialRequest")), + parseRequest(SabrProfileJson.requireArray(object, "followingRequest")), + parseMappings(SabrProfileJson.requireArray(object, "responseMappings")), + parseRecovery(SabrProfileJson.requireObject(object, "recovery")), + parseRules(SabrProfileJson.requireArray(object, "rules"))); + } + + @Nonnull + private static List parseRequest(@Nonnull final JsonArray array) { + final List result = new ArrayList<>(); + for (int index = 0; index < array.size(); index++) { + final JsonObject item = SabrProfileJson.requireObject(array, index); + SabrProfileJson.requireKeys(item, "field", "wireType", "source", "required"); + result.add(new SabrProfileRequestField(SabrProfileJson.requireInt(item, "field"), + enumValue(SabrProfileRequestField.WireType.class, + SabrProfileJson.requireString(item, "wireType"), "request wire type"), + enumValue(SabrProfileRequestField.Source.class, + SabrProfileJson.requireString(item, "source"), "request source"), + SabrProfileJson.requireBoolean(item, "required"))); + } + return result; + } + + @Nonnull + private static List parseMappings(@Nonnull final JsonArray array) { + final List result = new ArrayList<>(); + for (int index = 0; index < array.size(); index++) { + final JsonObject item = SabrProfileJson.requireObject(array, index); + SabrProfileJson.requireKeys(item, "partType", "target", "path", "wireType", + "required"); + result.add(new SabrProfileResponseMapping( + SabrProfileJson.requireInt(item, "partType"), + SabrProfileResponseMapping.Target.fromId( + SabrProfileJson.requireString(item, "target")), + parsePath(SabrProfileJson.requireArray(item, "path")), + enumValue(SabrProfileResponseMapping.WireType.class, + SabrProfileJson.requireString(item, "wireType"), "response wire type"), + SabrProfileJson.requireBoolean(item, "required"))); + } + return result; + } + + private static int[] parsePath(@Nonnull final JsonArray array) { + final int[] path = new int[array.size()]; + for (int index = 0; index < path.length; index++) { + path[index] = SabrProfileJson.exactInt(array.get(index), "response path"); + } + return path; + } + + private static SabrProfileRecovery parseRecovery(@Nonnull final JsonObject object) { + SabrProfileJson.requireKeys(object, "maximumOmissions", "maximumElapsedMs", + "forwardThresholdMs", "retryDelayMs"); + return new SabrProfileRecovery(SabrProfileJson.requireInt(object, "maximumOmissions"), + SabrProfileJson.requireLong(object, "maximumElapsedMs"), + SabrProfileJson.requireLong(object, "forwardThresholdMs"), + SabrProfileJson.requireInt(object, "retryDelayMs")); + } + + private static List parseRules(@Nonnull final JsonArray array) { + final List result = new ArrayList<>(); + for (int index = 0; index < array.size(); index++) { + final JsonObject item = SabrProfileJson.requireObject(array, index); + SabrProfileJson.requireKeys(item, "whenAll", "actions"); + result.add(new SabrProfileRule(parseEnums( + SabrProfileJson.requireArray(item, "whenAll"), + SabrProfileRule.Predicate.class, "behavior predicate"), + parseEnums(SabrProfileJson.requireArray(item, "actions"), + SabrProfileRule.Action.class, "behavior action"))); + } + return result; + } + + private static > List parseEnums( + @Nonnull final JsonArray array, @Nonnull final Class type, + @Nonnull final String label) { + final List result = new ArrayList<>(); + for (int index = 0; index < array.size(); index++) { + if (!(array.get(index) instanceof String)) { + throw new IllegalArgumentException("Invalid SABR " + label); + } + result.add(enumValue(type, (String) array.get(index), label)); + } + return result; + } + + private static > T enumValue(@Nonnull final Class type, + @Nonnull final String value, + @Nonnull final String label) { + try { + return Enum.valueOf(type, value); + } catch (final IllegalArgumentException error) { + throw new IllegalArgumentException("Unsupported SABR " + label + ": " + value, + error); + } + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrCompatibilityProfileValidator.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrCompatibilityProfileValidator.java new file mode 100644 index 00000000..84316109 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrCompatibilityProfileValidator.java @@ -0,0 +1,101 @@ +package org.schabi.newpipe.extractor.services.youtube.sabr; + +import javax.annotation.Nonnull; +import java.util.EnumSet; +import java.util.HashSet; +import java.util.List; + +final class SabrCompatibilityProfileValidator { + private static final int MAX_REQUEST_FIELDS = 64; + private static final int MAX_RESPONSE_MAPPINGS = 128; + private static final int MAX_RULES = 32; + + private SabrCompatibilityProfileValidator() { + } + + static void validate(@Nonnull final List initialRequest, + @Nonnull final List followingRequest, + @Nonnull final List responseMappings, + @Nonnull final List rules, + @Nonnull final SabrCompatibilityProfileClient.MediaParts mediaParts) { + validateSize(initialRequest, MAX_REQUEST_FIELDS, "initial request"); + validateSize(followingRequest, MAX_REQUEST_FIELDS, "following request"); + validateSize(responseMappings, MAX_RESPONSE_MAPPINGS, "response mappings"); + validateSize(rules, MAX_RULES, "behavior rules"); + if (initialRequest.isEmpty() || followingRequest.isEmpty() || rules.isEmpty() + || !rules.get(rules.size() - 1).getPredicates().isEmpty()) { + throw new IllegalArgumentException("Incomplete SABR compatibility client"); + } + validateRequestTemplate(initialRequest); + validateRequestTemplate(followingRequest); + validateMappings(responseMappings, mediaParts); + validateRules(rules); + } + + private static void validateRules(@Nonnull final List rules) { + for (final SabrProfileRule rule : rules) { + final List predicates = rule.getPredicates(); + final List actions = rule.getActions(); + if (actions.contains(SabrProfileRule.Action.APPLY_REDIRECT) + && !predicates.contains(SabrProfileRule.Predicate.HAS_REDIRECT) + || actions.contains(SabrProfileRule.Action.REFRESH_PO_TOKEN) + && !predicates.contains(SabrProfileRule.Predicate.PUMP_MODE) + || actions.contains(SabrProfileRule.Action.REQUIRE_PO_TOKEN) + && !predicates.contains(SabrProfileRule.Predicate.FETCH_SEGMENT_MODE) + || actions.contains(SabrProfileRule.Action.TRY_RELOAD) + && !predicates.contains(SabrProfileRule.Predicate.RELOAD_REQUESTED)) { + throw new IllegalArgumentException("Unsafe SABR behavior rule"); + } + } + final SabrProfileRule fallback = rules.get(rules.size() - 1); + if (fallback.getActions().size() != 1 + || fallback.getActions().get(0) != SabrProfileRule.Action.CONTINUE) { + throw new IllegalArgumentException("SABR default behavior must continue"); + } + } + + private static void validateRequestTemplate( + @Nonnull final List fields) { + final EnumSet sources = EnumSet.noneOf( + SabrProfileRequestField.Source.class); + final HashSet fieldNumbers = new HashSet<>(); + for (final SabrProfileRequestField field : fields) { + if (field == null || !sources.add(field.getSource()) + || !fieldNumbers.add(field.getField())) { + throw new IllegalArgumentException("Duplicate SABR request source or field"); + } + } + if (!sources.contains(SabrProfileRequestField.Source.CLIENT_ABR_STATE) + || !sources.contains(SabrProfileRequestField.Source.USTREAMER_CONFIG) + || !sources.contains(SabrProfileRequestField.Source.CLIENT_CONTEXT)) { + throw new IllegalArgumentException("SABR request template misses required sources"); + } + } + + private static void validateMappings( + @Nonnull final List mappings, + @Nonnull final SabrCompatibilityProfileClient.MediaParts mediaParts) { + final EnumSet targets = EnumSet.noneOf( + SabrProfileResponseMapping.Target.class); + for (final SabrProfileResponseMapping mapping : mappings) { + if (mapping == null || !targets.add(mapping.getTarget())) { + throw new IllegalArgumentException("Duplicate SABR response target"); + } + if (mapping.getPartType() == mediaParts.getPayload() + || mapping.getPartType() == mediaParts.getEnd()) { + throw new IllegalArgumentException("SABR profiles cannot inspect media payloads"); + } + if (mapping.getTarget().name().startsWith("MEDIA_HEADER_") + && mapping.getPartType() != mediaParts.getHeader()) { + throw new IllegalArgumentException("Media mapping uses non-header UMP part"); + } + } + } + + private static void validateSize(@Nonnull final List values, final int maximum, + @Nonnull final String name) { + if (values.size() > maximum) { + throw new IllegalArgumentException("Too many SABR " + name); + } + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrMediaHeader.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrMediaHeader.java index 968b3e37..e4f93991 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrMediaHeader.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrMediaHeader.java @@ -81,6 +81,19 @@ public static SabrMediaHeader normalized(final int headerId, @Nonnull static SabrMediaHeader decode(@Nonnull final byte[] data) throws SabrProtocolException { + return decode(data, false); + } + + @Nonnull + static SabrMediaHeader decodeLenient(@Nonnull final byte[] data) + throws SabrProtocolException { + return decode(data, true); + } + + @Nonnull + private static SabrMediaHeader decode(@Nonnull final byte[] data, + final boolean ignoreWireTypeChanges) + throws SabrProtocolException { int headerId = -1; String videoId = null; int itag = -1; @@ -100,6 +113,9 @@ static SabrMediaHeader decode(@Nonnull final byte[] data) throws SabrProtocolExc long sequenceLastModified = -1; for (final SabrProto.Field field : SabrProto.readFields(data)) { + if (ignoreWireTypeChanges && !hasBuiltinWireType(field)) { + continue; + } switch (field.getNumber()) { case 1: headerId = (int) field.getVarint(); @@ -181,6 +197,31 @@ static SabrMediaHeader decode(@Nonnull final byte[] data) throws SabrProtocolExc sequenceLastModified); } + private static boolean hasBuiltinWireType(@Nonnull final SabrProto.Field field) { + switch (field.getNumber()) { + case 2: + case 5: + case 13: + case 15: + return field.getWireType() == SabrProto.WIRE_LENGTH_DELIMITED; + case 1: + case 3: + case 4: + case 6: + case 7: + case 8: + case 9: + case 10: + case 11: + case 12: + case 14: + case 16: + return field.getWireType() == SabrProto.WIRE_VARINT; + default: + return true; + } + } + @Nonnull private static FormatId decodeFormatId(@Nonnull final byte[] data) throws SabrProtocolException { int itag = -1; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrProfileJson.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrProfileJson.java new file mode 100644 index 00000000..c72b46ee --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrProfileJson.java @@ -0,0 +1,216 @@ +package org.schabi.newpipe.extractor.services.youtube.sabr; + +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; + +import javax.annotation.Nonnull; +import java.math.BigInteger; +import java.util.ArrayDeque; +import java.util.Deque; +import java.util.HashSet; +import java.util.Set; + +/** Strict bounded JSON accessors shared by compatibility profile parsers. */ +final class SabrProfileJson { + private static final int MAX_JSON_DEPTH = 16; + private static final int MAX_STRUCTURE_TOKENS = 8192; + + private SabrProfileJson() { + } + + static void validateStructure(@Nonnull final String encoded) { + int depth = 0; + int structureTokens = 0; + final Deque> objectKeys = new ArrayDeque<>(); + for (int index = 0; index < encoded.length(); index++) { + final char value = encoded.charAt(index); + if (value == '"') { + final int end = findStringEnd(encoded, index + 1); + int following = end + 1; + while (following < encoded.length() + && Character.isWhitespace(encoded.charAt(following))) { + following++; + } + if (following < encoded.length() && encoded.charAt(following) == ':' + && !objectKeys.isEmpty()) { + final String key = unescape(encoded, index + 1, end); + if (!objectKeys.peek().add(key)) { + throw new IllegalArgumentException( + "Duplicate SABR compatibility profile field: " + key); + } + } + index = end; + } else if (value == '{' || value == '[') { + if (++depth > MAX_JSON_DEPTH) { + throw new IllegalArgumentException("SABR profile JSON is too deeply nested"); + } + if (value == '{') { + objectKeys.push(new HashSet<>()); + } + } else if (value == ':' || value == ',') { + if (++structureTokens > MAX_STRUCTURE_TOKENS) { + throw new IllegalArgumentException( + "SABR profile JSON has too many entries"); + } + } else if (value == '}' || value == ']') { + if (--depth < 0) { + throw new IllegalArgumentException("Malformed SABR compatibility profile"); + } + if (value == '}') { + if (objectKeys.isEmpty()) { + throw new IllegalArgumentException( + "Malformed SABR compatibility profile"); + } + objectKeys.pop(); + } + } + } + if (depth != 0 || !objectKeys.isEmpty()) { + throw new IllegalArgumentException("Malformed SABR compatibility profile"); + } + } + + private static int findStringEnd(@Nonnull final String value, final int start) { + boolean escaped = false; + for (int index = start; index < value.length(); index++) { + final char current = value.charAt(index); + if (escaped) { + escaped = false; + } else if (current == '\\') { + escaped = true; + } else if (current == '"') { + return index; + } + } + throw new IllegalArgumentException("Malformed SABR compatibility profile string"); + } + + @Nonnull + private static String unescape(@Nonnull final String value, final int start, final int end) { + final StringBuilder result = new StringBuilder(end - start); + for (int index = start; index < end; index++) { + final char current = value.charAt(index); + if (current != '\\') { + result.append(current); + continue; + } + if (++index >= end) { + throw new IllegalArgumentException("Malformed SABR profile field escape"); + } + final char escaped = value.charAt(index); + switch (escaped) { + case '"': case '\\': case '/': result.append(escaped); break; + case 'b': result.append('\b'); break; + case 'f': result.append('\f'); break; + case 'n': result.append('\n'); break; + case 'r': result.append('\r'); break; + case 't': result.append('\t'); break; + case 'u': + if (index + 4 >= end) { + throw new IllegalArgumentException("Malformed SABR profile Unicode escape"); + } + try { + result.append((char) Integer.parseInt( + value.substring(index + 1, index + 5), 16)); + } catch (final NumberFormatException error) { + throw new IllegalArgumentException( + "Malformed SABR profile Unicode escape", error); + } + index += 4; + break; + default: + throw new IllegalArgumentException("Malformed SABR profile field escape"); + } + } + return result.toString(); + } + + static void requireKeys(@Nonnull final JsonObject object, + @Nonnull final String... keys) { + final Set expected = new HashSet<>(); + java.util.Collections.addAll(expected, keys); + if (object.size() != expected.size() || !object.keySet().equals(expected)) { + throw new IllegalArgumentException("Unexpected SABR compatibility profile fields"); + } + } + + @Nonnull + static JsonObject requireObject(@Nonnull final JsonObject parent, + @Nonnull final String field) { + final Object value = parent.get(field); + if (!(value instanceof JsonObject)) { + throw new IllegalArgumentException("SABR profile field is not an object: " + field); + } + return (JsonObject) value; + } + + @Nonnull + static JsonObject requireObject(@Nonnull final JsonArray parent, final int index) { + final Object value = parent.get(index); + if (!(value instanceof JsonObject)) { + throw new IllegalArgumentException("SABR profile array item is not an object"); + } + return (JsonObject) value; + } + + @Nonnull + static JsonArray requireArray(@Nonnull final JsonObject parent, + @Nonnull final String field) { + final Object value = parent.get(field); + if (!(value instanceof JsonArray)) { + throw new IllegalArgumentException("SABR profile field is not an array: " + field); + } + return (JsonArray) value; + } + + @Nonnull + static String requireString(@Nonnull final JsonObject parent, + @Nonnull final String field) { + final Object value = parent.get(field); + if (!(value instanceof String)) { + throw new IllegalArgumentException("SABR profile field is not a string: " + field); + } + return (String) value; + } + + static boolean requireBoolean(@Nonnull final JsonObject parent, + @Nonnull final String field) { + final Object value = parent.get(field); + if (!(value instanceof Boolean)) { + throw new IllegalArgumentException("SABR profile field is not a boolean: " + field); + } + return (Boolean) value; + } + + static int requireInt(@Nonnull final JsonObject parent, @Nonnull final String field) { + return exactInt(parent.get(field), field); + } + + static int exactInt(final Object value, @Nonnull final String field) { + final long exact = exactLong(value, field); + if (exact < Integer.MIN_VALUE || exact > Integer.MAX_VALUE) { + throw new IllegalArgumentException("SABR profile integer is out of range: " + field); + } + return (int) exact; + } + + static long requireLong(@Nonnull final JsonObject parent, @Nonnull final String field) { + return exactLong(parent.get(field), field); + } + + private static long exactLong(final Object value, @Nonnull final String field) { + if (value instanceof Integer || value instanceof Long) { + return ((Number) value).longValue(); + } + if (value instanceof BigInteger) { + try { + return ((BigInteger) value).longValueExact(); + } catch (final ArithmeticException error) { + throw new IllegalArgumentException( + "SABR profile integer is out of range: " + field, error); + } + } + throw new IllegalArgumentException( + "SABR profile field is not an exact integer: " + field); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrProfileMappedResponse.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrProfileMappedResponse.java new file mode 100644 index 00000000..e2512988 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrProfileMappedResponse.java @@ -0,0 +1,185 @@ +package org.schabi.newpipe.extractor.services.youtube.sabr; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.util.List; + +/** Per-response normalized control values produced without exposing media payloads. */ +final class SabrProfileMappedResponse { + @Nonnull private final SabrResponseStatePatch statePatch; + @Nullable private final String redirectUrl; + @Nullable private final String errorDetails; + private final int protectionStatus; + private final int backoffMs; + private final boolean reloadRequested; + + private SabrProfileMappedResponse(@Nonnull final Builder builder, + @Nonnull final SabrDecodedResponse response) { + final SabrNextRequestPolicy builtin = response.getNextRequestPolicy(); + final SabrNextRequestPolicy nextRequest = builder.hasNextRequestOverride + ? SabrNextRequestPolicy.normalized( + builder.targetAudioReadaheadMs(builtin), + builder.targetVideoReadaheadMs(builtin), + builder.maxTimeSinceLastRequestMs(builtin), + builder.backoffMs(builtin), + builder.minAudioReadaheadMs(builtin), + builder.minVideoReadaheadMs(builtin), + builder.playbackCookie(builtin), builder.videoId(builtin)) : builtin; + final SabrResponseStatePatch.Builder patch = SabrResponseStatePatch.builder() + .setNextRequestPolicy(nextRequest) + .setContextSendingPolicy(response.getSabrContextSendingPolicy()); + for (final SabrLiveMetadata value : response.getLiveMetadata()) { + patch.addLiveMetadata(value); + } + for (final SabrFormatInitializationMetadata value + : response.getFormatInitializationMetadata()) { + patch.addFormatMetadata(value); + } + for (final SabrMediaHeader value : response.getMediaHeaders()) { + patch.addMediaHeader(value); + } + for (final SabrContextUpdate value : response.getSabrContextUpdates()) { + patch.addContextUpdate(value); + } + statePatch = patch.build(); + redirectUrl = builder.redirectUrl != null ? builder.redirectUrl : response.getRedirectUrl(); + final SabrError builtinError = response.getSabrErrorDetails(); + final String builtinErrorType = builtinError == null ? null : builtinError.getType(); + final int builtinErrorCode = builtinError == null ? 0 : builtinError.getCode(); + final String errorType = builder.errorType != null ? builder.errorType : builtinErrorType; + final int errorCode = builder.errorCode != null ? builder.errorCode : builtinErrorCode; + errorDetails = errorType == null && errorCode == 0 ? null + : "type=" + (errorType == null ? "null" : errorType) + ", code=" + errorCode; + protectionStatus = builder.protectionStatus != null + ? builder.protectionStatus : response.getStreamProtectionStatus(); + backoffMs = builder.backoffMs != null ? builder.backoffMs + : Math.max(0, response.getBackoffTimeMs()); + reloadRequested = builder.reloadRequested != null + ? builder.reloadRequested : response.isReloadRequested(); + } + + @Nonnull SabrResponseStatePatch getStatePatch() { return statePatch; } + @Nullable String getRedirectUrl() { return redirectUrl; } + @Nullable String getErrorDetails() { return errorDetails; } + int getProtectionStatus() { return protectionStatus; } + int getBackoffMs() { return backoffMs; } + boolean isReloadRequested() { return reloadRequested; } + + static final class Builder { + @Nullable private Integer targetAudioReadaheadMs; + @Nullable private Integer targetVideoReadaheadMs; + @Nullable private Integer maxTimeSinceLastRequestMs; + @Nullable private Integer backoffMs; + @Nullable private Integer minAudioReadaheadMs; + @Nullable private Integer minVideoReadaheadMs; + @Nullable private byte[] playbackCookie; + @Nullable private String videoId; + @Nullable private String redirectUrl; + @Nullable private String errorType; + @Nullable private Integer errorCode; + @Nullable private Integer protectionStatus; + @Nullable private Boolean reloadRequested; + private boolean hasNextRequestOverride; + + void apply(@Nonnull final SabrProfileResponseMapping.Target target, + @Nonnull final SabrProfileWireValue value) throws SabrProtocolException { + switch (target) { + case NEXT_REQUEST_TARGET_AUDIO_READAHEAD_MS: + targetAudioReadaheadMs = checkedInt(value.asLong(), target); + hasNextRequestOverride = true; + break; + case NEXT_REQUEST_TARGET_VIDEO_READAHEAD_MS: + targetVideoReadaheadMs = checkedInt(value.asLong(), target); + hasNextRequestOverride = true; + break; + case NEXT_REQUEST_MAX_TIME_SINCE_LAST_REQUEST_MS: + maxTimeSinceLastRequestMs = checkedInt(value.asLong(), target); + hasNextRequestOverride = true; + break; + case NEXT_REQUEST_BACKOFF_MS: + backoffMs = checkedInt(value.asLong(), target); + hasNextRequestOverride = true; + break; + case NEXT_REQUEST_MIN_AUDIO_READAHEAD_MS: + minAudioReadaheadMs = checkedInt(value.asLong(), target); + hasNextRequestOverride = true; + break; + case NEXT_REQUEST_MIN_VIDEO_READAHEAD_MS: + minVideoReadaheadMs = checkedInt(value.asLong(), target); + hasNextRequestOverride = true; + break; + case NEXT_REQUEST_PLAYBACK_COOKIE: + playbackCookie = value.asBytes(); + hasNextRequestOverride = true; + break; + case NEXT_REQUEST_VIDEO_ID: + videoId = value.asString(); + hasNextRequestOverride = true; + break; + case REDIRECT_URL: + redirectUrl = value.asString(); + break; + case ERROR_TYPE: + errorType = value.asString(); + break; + case ERROR_CODE: + errorCode = checkedInt(value.asLong(), target); + break; + case RELOAD_REQUESTED: + reloadRequested = value.asBoolean(); + break; + case PROTECTION_STATUS: + protectionStatus = checkedInt(value.asLong(), target); + break; + default: + break; + } + } + + @Nonnull + SabrProfileMappedResponse build(@Nonnull final SabrDecodedResponse response) { + return new SabrProfileMappedResponse(this, response); + } + + private static int checkedInt(final long value, + @Nonnull final SabrProfileResponseMapping.Target target) + throws SabrProtocolException { + if (value < Integer.MIN_VALUE || value > Integer.MAX_VALUE) { + throw new SabrProtocolException("SABR profile integer overflow: " + target); + } + return (int) value; + } + + private int targetAudioReadaheadMs(@Nullable final SabrNextRequestPolicy builtin) { + return targetAudioReadaheadMs != null ? targetAudioReadaheadMs + : builtin == null ? -1 : builtin.getTargetAudioReadaheadMs(); + } + private int targetVideoReadaheadMs(@Nullable final SabrNextRequestPolicy builtin) { + return targetVideoReadaheadMs != null ? targetVideoReadaheadMs + : builtin == null ? -1 : builtin.getTargetVideoReadaheadMs(); + } + private int maxTimeSinceLastRequestMs(@Nullable final SabrNextRequestPolicy builtin) { + return maxTimeSinceLastRequestMs != null ? maxTimeSinceLastRequestMs + : builtin == null ? -1 : builtin.getMaxTimeSinceLastRequestMs(); + } + private int backoffMs(@Nullable final SabrNextRequestPolicy builtin) { + return backoffMs != null ? backoffMs + : builtin == null ? -1 : builtin.getBackoffTimeMs(); + } + private int minAudioReadaheadMs(@Nullable final SabrNextRequestPolicy builtin) { + return minAudioReadaheadMs != null ? minAudioReadaheadMs + : builtin == null ? -1 : builtin.getMinAudioReadaheadMs(); + } + private int minVideoReadaheadMs(@Nullable final SabrNextRequestPolicy builtin) { + return minVideoReadaheadMs != null ? minVideoReadaheadMs + : builtin == null ? -1 : builtin.getMinVideoReadaheadMs(); + } + @Nullable private byte[] playbackCookie(@Nullable final SabrNextRequestPolicy builtin) { + return playbackCookie != null ? playbackCookie + : builtin == null ? null : builtin.getPlaybackCookie(); + } + @Nullable private String videoId(@Nullable final SabrNextRequestPolicy builtin) { + return videoId != null ? videoId : builtin == null ? null : builtin.getVideoId(); + } + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrProfileMediaHeaderMapper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrProfileMediaHeaderMapper.java new file mode 100644 index 00000000..4aa5fbd0 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrProfileMediaHeaderMapper.java @@ -0,0 +1,130 @@ +package org.schabi.newpipe.extractor.services.youtube.sabr; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.util.List; + +/** Overlays profile-declared protobuf paths on the stable native media-header model. */ +final class SabrProfileMediaHeaderMapper { + @Nonnull private final List mappings; + + SabrProfileMediaHeaderMapper(@Nonnull final List mappings) { + this.mappings = mappings; + } + + @Nonnull + SabrMediaHeader decode(@Nonnull final byte[] payload) throws SabrProtocolException { + final SabrMediaHeader builtin = SabrMediaHeader.decodeLenient(payload); + final Values values = new Values(builtin); + for (final SabrProfileResponseMapping mapping : mappings) { + if (!mapping.getTarget().name().startsWith("MEDIA_HEADER_")) { + continue; + } + final SabrProfileWireValue value = SabrProfileWireValue.find(payload, mapping); + if (value == null) { + if (mapping.isRequired()) { + throw new SabrProtocolException("Required SABR media-header mapping is absent: " + + mapping); + } + } else { + values.apply(mapping.getTarget(), value); + } + } + return values.build(); + } + + private static final class Values { + private int headerId; + @Nullable private String videoId; + private int itag; + private long lastModified; + @Nullable private String xtags; + private long startRange; + private int compression; + private boolean init; + private int sequence; + private long bitrateBps; + private long startMs; + private long durationMs; + private long contentLength; + private long timeRangeStart; + private long timeRangeDuration; + private int timeRangeTimescale; + private long sequenceLastModified; + + Values(@Nonnull final SabrMediaHeader header) { + headerId = header.getHeaderId(); + videoId = header.getVideoId(); + itag = header.getItag(); + lastModified = header.getLastModified(); + xtags = header.getXtags(); + startRange = header.getStartRange(); + compression = header.getCompressionAlgorithm(); + init = header.isInitSegment(); + sequence = header.getSequenceNumber(); + bitrateBps = header.getBitrateBps(); + startMs = header.getStartMs(); + durationMs = header.getDurationMs(); + contentLength = header.getContentLength(); + timeRangeStart = header.getTimeRangeStartTicks(); + timeRangeDuration = header.getTimeRangeDurationTicks(); + timeRangeTimescale = header.getTimeRangeTimescale(); + sequenceLastModified = header.getSequenceLastModified(); + } + + void apply(@Nonnull final SabrProfileResponseMapping.Target target, + @Nonnull final SabrProfileWireValue value) throws SabrProtocolException { + switch (target) { + case MEDIA_HEADER_ID: headerId = integer(value, target); break; + case MEDIA_HEADER_VIDEO_ID: videoId = value.asString(); break; + case MEDIA_HEADER_ITAG: itag = integer(value, target); break; + case MEDIA_HEADER_LAST_MODIFIED: lastModified = value.asLong(); break; + case MEDIA_HEADER_XTAGS: xtags = value.asString(); break; + case MEDIA_HEADER_START_RANGE: startRange = value.asLong(); break; + case MEDIA_HEADER_COMPRESSION: compression = integer(value, target); break; + case MEDIA_HEADER_IS_INIT: init = value.asBoolean(); break; + case MEDIA_HEADER_SEQUENCE: sequence = integer(value, target); break; + case MEDIA_HEADER_BITRATE_BPS: bitrateBps = value.asLong(); break; + case MEDIA_HEADER_START_MS: startMs = value.asLong(); break; + case MEDIA_HEADER_DURATION_MS: durationMs = value.asLong(); break; + case MEDIA_HEADER_CONTENT_LENGTH: contentLength = value.asLong(); break; + case MEDIA_HEADER_TIME_RANGE_START: timeRangeStart = value.asLong(); break; + case MEDIA_HEADER_TIME_RANGE_DURATION: timeRangeDuration = value.asLong(); break; + case MEDIA_HEADER_TIME_RANGE_TIMESCALE: + timeRangeTimescale = integer(value, target); + break; + case MEDIA_HEADER_SEQUENCE_LAST_MODIFIED: + sequenceLastModified = value.asLong(); + break; + default: + break; + } + } + + @Nonnull + SabrMediaHeader build() { + if (timeRangeTimescale > 0) { + if (startMs < 0 && timeRangeStart >= 0) { + startMs = timeRangeStart * 1000L / timeRangeTimescale; + } + if (durationMs < 0 && timeRangeDuration >= 0) { + durationMs = timeRangeDuration * 1000L / timeRangeTimescale; + } + } + return SabrMediaHeader.normalized(headerId, videoId, itag, lastModified, xtags, + startRange, compression, init, sequence, bitrateBps, startMs, durationMs, + contentLength, timeRangeStart, timeRangeDuration, timeRangeTimescale, + sequenceLastModified); + } + + private static int integer(@Nonnull final SabrProfileWireValue value, + @Nonnull final SabrProfileResponseMapping.Target target) + throws SabrProtocolException { + final long number = value.asLong(); + if (number < Integer.MIN_VALUE || number > Integer.MAX_VALUE) { + throw new SabrProtocolException("SABR media-header integer overflow: " + target); + } + return (int) number; + } + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrProfileRecovery.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrProfileRecovery.java new file mode 100644 index 00000000..bf265103 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrProfileRecovery.java @@ -0,0 +1,58 @@ +package org.schabi.newpipe.extractor.services.youtube.sabr; + +import javax.annotation.Nonnull; +import java.io.DataOutputStream; +import java.io.IOException; + +/** Bounded recovery thresholds which may change without changing the SABR engine. */ +public final class SabrProfileRecovery { + public static final int MAXIMUM_OMISSIONS_LIMIT = 16; + public static final long MAXIMUM_ELAPSED_MS_LIMIT = 120_000; + public static final long FORWARD_THRESHOLD_MS_LIMIT = 300_000; + public static final int RETRY_DELAY_MS_LIMIT = 5_000; + + private final int maximumOmissions; + private final long maximumElapsedMs; + private final long forwardThresholdMs; + private final int retryDelayMs; + + public SabrProfileRecovery(final int maximumOmissions, + final long maximumElapsedMs, + final long forwardThresholdMs, + final int retryDelayMs) { + if (maximumOmissions < 1 || maximumOmissions > MAXIMUM_OMISSIONS_LIMIT + || maximumElapsedMs < 1 || maximumElapsedMs > MAXIMUM_ELAPSED_MS_LIMIT + || forwardThresholdMs < 0 + || forwardThresholdMs > FORWARD_THRESHOLD_MS_LIMIT + || retryDelayMs < 0 || retryDelayMs > RETRY_DELAY_MS_LIMIT) { + throw new IllegalArgumentException("Invalid SABR recovery thresholds"); + } + this.maximumOmissions = maximumOmissions; + this.maximumElapsedMs = maximumElapsedMs; + this.forwardThresholdMs = forwardThresholdMs; + this.retryDelayMs = retryDelayMs; + } + + public int getMaximumOmissions() { + return maximumOmissions; + } + + public long getMaximumElapsedMs() { + return maximumElapsedMs; + } + + public long getForwardThresholdMs() { + return forwardThresholdMs; + } + + public int getRetryDelayMs() { + return retryDelayMs; + } + + void writeCanonical(@Nonnull final DataOutputStream output) throws IOException { + output.writeInt(maximumOmissions); + output.writeLong(maximumElapsedMs); + output.writeLong(forwardThresholdMs); + output.writeInt(retryDelayMs); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrProfileRequestBuilder.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrProfileRequestBuilder.java new file mode 100644 index 00000000..65305397 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrProfileRequestBuilder.java @@ -0,0 +1,199 @@ +package org.schabi.newpipe.extractor.services.youtube.sabr; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** Builds bounded protobuf requests from Host-owned typed values. */ +final class SabrProfileRequestBuilder { + private SabrProfileRequestBuilder() { + } + + @Nonnull + static byte[] build(@Nonnull final YoutubeSabrInfo info, + @Nonnull final YoutubeSabrFormat audioFormat, + @Nonnull final YoutubeSabrFormat videoFormat, + @Nonnull final YoutubeSabrStreamState streamState, + final boolean initial, + @Nonnull final List template) + throws SabrProtocolException { + synchronized (streamState) { + final SabrProto.Writer request = new SabrProto.Writer(); + for (final SabrProfileRequestField field : template) { + final List values = resolve(info, audioFormat, videoFormat, streamState, + initial, field.getSource()); + if (values.isEmpty() && field.isRequired()) { + throw new SabrProtocolException("Required SABR profile source is empty: " + + field.getSource()); + } + for (final Object value : values) { + if (field.getWireType() == SabrProfileRequestField.WireType.VARINT) { + request.writeUInt64(field.getField(), (Long) value); + } else { + request.writeBytes(field.getField(), (byte[]) value); + } + } + } + return request.toByteArray(); + } + } + + @Nonnull + private static List resolve( + @Nonnull final YoutubeSabrInfo info, + @Nonnull final YoutubeSabrFormat audioFormat, + @Nonnull final YoutubeSabrFormat videoFormat, + @Nonnull final YoutubeSabrStreamState streamState, + final boolean initial, + @Nonnull final SabrProfileRequestField.Source source) + throws SabrProtocolException { + final long playerTimeMs = streamState.getRequestPlayerTimeMs(); + final List bufferedRanges = streamState.getBufferedRanges(); + final boolean includePlaybackState = !initial || playerTimeMs > 0 + || !bufferedRanges.isEmpty(); + switch (source) { + case PLAYER_TIME_MS: + return includePlaybackState && streamState.shouldWriteTopLevelPlayerTimeMs() + ? singletonLong(playerTimeMs) : Collections.emptyList(); + case BUFFERED_RANGES: + if (!includePlaybackState) return Collections.emptyList(); + final List ranges = new ArrayList<>(); + for (final SabrBufferedRange range : bufferedRanges) { + ranges.add(range.toProto(streamState.shouldWriteBufferedRangeTimeRange())); + } + return ranges; + case PLAYBACK_COOKIE: + return singletonBytes(streamState.getRawPlaybackCookie()); + case PO_TOKEN: + return singletonBytes(streamState.getRawPoToken()); + case SELECTED_FORMATS: + return selectedFormats(audioFormat, videoFormat, streamState, + includePlaybackState); + case CLIENT_CONTEXT: + return singletonBytes(YoutubeSabrRequestBuilder.buildStreamerContext( + info, streamState)); + case CLIENT_ABR_STATE: + return singletonBytes(YoutubeSabrRequestBuilder.buildClientAbrState( + audioFormat, videoFormat, playerTimeMs, includePlaybackState, + streamState.getEnabledTrackTypesBitfield(), streamState)); + case USTREAMER_CONFIG: + final String config = info.getVideoPlaybackUstreamerConfig(); + if (config == null || config.isEmpty()) return Collections.emptyList(); + return singletonBytes(YoutubeSabrRequestBuilder.decodeBase64(config)); + case PREFERRED_AUDIO_FORMATS: + return preferredFormats(info, audioFormat, videoFormat, streamState, true); + case PREFERRED_VIDEO_FORMATS: + return preferredFormats(info, audioFormat, videoFormat, streamState, false); + default: + throw new SabrProtocolException("Unsupported SABR profile request source: " + + source); + } + } + + @Nonnull + private static List selectedFormats(@Nonnull final YoutubeSabrFormat audioFormat, + @Nonnull final YoutubeSabrFormat videoFormat, + @Nonnull final YoutubeSabrStreamState streamState, + final boolean includePlaybackState) { + if (!includePlaybackState) return Collections.emptyList(); + final List values = new ArrayList<>(); + if (streamState.shouldSelectVideoFormatBeforeAudio()) { + addSelected(values, videoFormat, streamState.shouldSelectVideoFormat(), streamState); + } + addSelected(values, audioFormat, streamState.shouldSelectAudioFormat(), streamState); + if (!streamState.shouldSelectVideoFormatBeforeAudio()) { + addSelected(values, videoFormat, streamState.shouldSelectVideoFormat(), streamState); + } + return values; + } + + private static void addSelected(@Nonnull final List values, + @Nonnull final YoutubeSabrFormat format, + final boolean selected, + @Nonnull final YoutubeSabrStreamState streamState) { + if (selected && streamState.isInitialized(format)) { + values.add(SabrProto.formatId(format)); + } + } + + @Nonnull + private static List preferredFormats( + @Nonnull final YoutubeSabrInfo info, + @Nonnull final YoutubeSabrFormat audioFormat, + @Nonnull final YoutubeSabrFormat videoFormat, + @Nonnull final YoutubeSabrStreamState streamState, + final boolean audio) { + final List values = new ArrayList<>(); + if (audio && !streamState.shouldSelectAudioFormat() + || !audio && !streamState.shouldSelectVideoFormat()) { + return values; + } + if (!streamState.shouldWriteAllPreferredFormats()) { + values.add(SabrProto.formatId(audio ? audioFormat : videoFormat)); + return values; + } + if (streamState.shouldWriteOfficialWebPreferredFormats()) { + for (final YoutubeSabrFormat format : officialPreferredFormats(info)) { + if (audio == format.isAudio()) values.add(SabrProto.formatId(format)); + } + return values; + } + for (final YoutubeSabrFormat format : info.getFormats()) { + if (audio == format.isAudio() && (audio || format.isVideo())) { + values.add(SabrProto.formatId(format)); + } + } + return values; + } + + @Nonnull + private static List officialPreferredFormats( + @Nonnull final YoutubeSabrInfo info) { + final List values = new ArrayList<>(); + addAudio(values, info, 251, 12); + addAudio(values, info, 251, 14); + addAudio(values, info, 251, 0); + addAudio(values, info, 250, 14); + addAudio(values, info, 250, 0); + addAudio(values, info, 250, 12); + for (final int itag : new int[]{248, 247, 244, 243, 242, 278}) { + addVideo(values, info, itag); + } + return values; + } + + private static void addAudio(@Nonnull final List values, + @Nonnull final YoutubeSabrInfo info, + final int itag, final int xtagsLength) { + for (final YoutubeSabrFormat format : info.getFormats()) { + final String xtags = format.getXtags(); + if (format.isAudio() && format.getItag() == itag + && (xtags == null ? 0 : xtags.length()) == xtagsLength) { + values.add(format); + } + } + } + + private static void addVideo(@Nonnull final List values, + @Nonnull final YoutubeSabrInfo info, final int itag) { + for (final YoutubeSabrFormat format : info.getFormats()) { + if (format.isVideo() && format.getItag() == itag) { + values.add(format); + return; + } + } + } + + @Nonnull + private static List singletonLong(final long value) { + return Collections.singletonList(value); + } + + @Nonnull + private static List singletonBytes(@Nullable final byte[] value) { + return value == null || value.length == 0 + ? Collections.emptyList() : Collections.singletonList(value); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrProfileRequestData.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrProfileRequestData.java new file mode 100644 index 00000000..2b87a43c --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrProfileRequestData.java @@ -0,0 +1,37 @@ +package org.schabi.newpipe.extractor.services.youtube.sabr; + +import javax.annotation.Nonnull; +import java.util.List; + +/** Host-only request inputs. No secret accessor is exposed to policy implementations. */ +final class SabrProfileRequestData { + @Nonnull private final YoutubeSabrInfo info; + @Nonnull private final YoutubeSabrFormat audioFormat; + @Nonnull private final YoutubeSabrFormat videoFormat; + @Nonnull private final YoutubeSabrStreamState streamState; + private final boolean initial; + + SabrProfileRequestData(@Nonnull final YoutubeSabrInfo info, + @Nonnull final YoutubeSabrFormat audioFormat, + @Nonnull final YoutubeSabrFormat videoFormat, + @Nonnull final YoutubeSabrStreamState streamState, + final boolean initial) { + this.info = info; + this.audioFormat = audioFormat; + this.videoFormat = videoFormat; + this.streamState = streamState; + this.initial = initial; + } + + @Nonnull + byte[] build(@Nonnull final List template) + throws SabrProtocolException { + return SabrProfileRequestBuilder.build(info, audioFormat, videoFormat, + streamState, initial, template); + } + + @Nonnull + YoutubeSabrClientProfile getClientProfile() { + return info.getProfile(); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrProfileRequestField.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrProfileRequestField.java new file mode 100644 index 00000000..f67c967c --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrProfileRequestField.java @@ -0,0 +1,94 @@ +package org.schabi.newpipe.extractor.services.youtube.sabr; + +import javax.annotation.Nonnull; +import java.io.DataOutputStream; +import java.io.IOException; +import java.util.Objects; + +/** One protobuf field whose value is supplied by a Host-owned typed source. */ +public final class SabrProfileRequestField { + public enum WireType { + VARINT(SabrProto.WIRE_VARINT), + BYTES(SabrProto.WIRE_LENGTH_DELIMITED); + + private final int protobufType; + + WireType(final int protobufType) { + this.protobufType = protobufType; + } + + int getProtobufType() { + return protobufType; + } + } + + public enum Source { + PLAYER_TIME_MS(WireType.VARINT), + BUFFERED_RANGES(WireType.BYTES), + PLAYBACK_COOKIE(WireType.BYTES), + PO_TOKEN(WireType.BYTES), + SELECTED_FORMATS(WireType.BYTES), + CLIENT_CONTEXT(WireType.BYTES), + CLIENT_ABR_STATE(WireType.BYTES), + USTREAMER_CONFIG(WireType.BYTES), + PREFERRED_AUDIO_FORMATS(WireType.BYTES), + PREFERRED_VIDEO_FORMATS(WireType.BYTES); + + @Nonnull private final WireType wireType; + + Source(@Nonnull final WireType wireType) { + this.wireType = wireType; + } + + @Nonnull + WireType getWireType() { + return wireType; + } + } + + private final int field; + @Nonnull private final WireType wireType; + @Nonnull private final Source source; + private final boolean required; + + public SabrProfileRequestField(final int field, + @Nonnull final WireType wireType, + @Nonnull final Source source, + final boolean required) { + if (field <= 0 || field > SabrCompatibilityProfile.MAX_PROTOBUF_FIELD_NUMBER) { + throw new IllegalArgumentException("Invalid SABR request field number"); + } + this.wireType = Objects.requireNonNull(wireType); + this.source = Objects.requireNonNull(source); + if (source.getWireType() != wireType) { + throw new IllegalArgumentException("SABR request source/wire type mismatch"); + } + this.field = field; + this.required = required; + } + + public int getField() { + return field; + } + + @Nonnull + public WireType getWireType() { + return wireType; + } + + @Nonnull + public Source getSource() { + return source; + } + + public boolean isRequired() { + return required; + } + + void writeCanonical(@Nonnull final DataOutputStream output) throws IOException { + output.writeInt(field); + SabrCompatibilityProfile.writeString(output, wireType.name()); + SabrCompatibilityProfile.writeString(output, source.name()); + output.writeBoolean(required); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrProfileResponseMapper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrProfileResponseMapper.java new file mode 100644 index 00000000..c77bf2b7 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrProfileResponseMapper.java @@ -0,0 +1,42 @@ +package org.schabi.newpipe.extractor.services.youtube.sabr; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.util.List; + +/** Applies precompiled response paths to small UMP control parts. */ +final class SabrProfileResponseMapper { + @Nonnull private final List mappings; + + SabrProfileResponseMapper(@Nonnull final List mappings) { + this.mappings = mappings; + } + + @Nonnull + SabrProfileMappedResponse map(@Nonnull final SabrDecodedResponse response) + throws SabrProtocolException { + final SabrProfileMappedResponse.Builder builder = new SabrProfileMappedResponse.Builder(); + for (final SabrProfileResponseMapping mapping : mappings) { + if (mapping.getTarget().name().startsWith("MEDIA_HEADER_")) { + continue; + } + boolean partSeen = false; + SabrProfileWireValue selected = null; + for (final UmpPart part : response.getParts()) { + if (part.getType() == mapping.getPartType()) { + partSeen = true; + final SabrProfileWireValue current = SabrProfileWireValue.find( + part.getRawData(), mapping); + if (current != null) selected = current; + } + } + if (selected != null) { + builder.apply(mapping.getTarget(), selected); + } else if (partSeen && mapping.isRequired()) { + throw new SabrProtocolException("Required SABR response mapping is absent: " + + mapping); + } + } + return builder.build(response); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrProfileResponseMapping.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrProfileResponseMapping.java new file mode 100644 index 00000000..63e768c2 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrProfileResponseMapping.java @@ -0,0 +1,167 @@ +package org.schabi.newpipe.extractor.services.youtube.sabr; + +import javax.annotation.Nonnull; +import java.io.DataOutputStream; +import java.io.IOException; +import java.util.Arrays; +import java.util.Objects; + +/** Compiled path from one UMP protobuf part to bounded normalized SABR state. */ +public final class SabrProfileResponseMapping { + private static final int MAX_PATH_DEPTH = 8; + + public enum WireType { + VARINT, + BYTES, + STRING, + BOOL + } + + public enum Target { + NEXT_REQUEST_TARGET_AUDIO_READAHEAD_MS("NEXT_REQUEST.TARGET_AUDIO_READAHEAD_MS"), + NEXT_REQUEST_TARGET_VIDEO_READAHEAD_MS("NEXT_REQUEST.TARGET_VIDEO_READAHEAD_MS"), + NEXT_REQUEST_MAX_TIME_SINCE_LAST_REQUEST_MS( + "NEXT_REQUEST.MAX_TIME_SINCE_LAST_REQUEST_MS"), + NEXT_REQUEST_BACKOFF_MS("NEXT_REQUEST.BACKOFF_MS"), + NEXT_REQUEST_MIN_AUDIO_READAHEAD_MS("NEXT_REQUEST.MIN_AUDIO_READAHEAD_MS"), + NEXT_REQUEST_MIN_VIDEO_READAHEAD_MS("NEXT_REQUEST.MIN_VIDEO_READAHEAD_MS"), + NEXT_REQUEST_PLAYBACK_COOKIE("NEXT_REQUEST.PLAYBACK_COOKIE"), + NEXT_REQUEST_VIDEO_ID("NEXT_REQUEST.VIDEO_ID"), + REDIRECT_URL("REDIRECT.URL"), + ERROR_TYPE("ERROR.TYPE"), + ERROR_CODE("ERROR.CODE"), + RELOAD_REQUESTED("RELOAD.REQUESTED"), + PROTECTION_STATUS("PROTECTION.STATUS"), + MEDIA_HEADER_ID("MEDIA_HEADER.ID"), + MEDIA_HEADER_VIDEO_ID("MEDIA_HEADER.VIDEO_ID"), + MEDIA_HEADER_ITAG("MEDIA_HEADER.ITAG"), + MEDIA_HEADER_LAST_MODIFIED("MEDIA_HEADER.LAST_MODIFIED"), + MEDIA_HEADER_XTAGS("MEDIA_HEADER.XTAGS"), + MEDIA_HEADER_START_RANGE("MEDIA_HEADER.START_RANGE"), + MEDIA_HEADER_COMPRESSION("MEDIA_HEADER.COMPRESSION"), + MEDIA_HEADER_IS_INIT("MEDIA_HEADER.IS_INIT"), + MEDIA_HEADER_SEQUENCE("MEDIA_HEADER.SEQUENCE"), + MEDIA_HEADER_BITRATE_BPS("MEDIA_HEADER.BITRATE_BPS"), + MEDIA_HEADER_START_MS("MEDIA_HEADER.START_MS"), + MEDIA_HEADER_DURATION_MS("MEDIA_HEADER.DURATION_MS"), + MEDIA_HEADER_CONTENT_LENGTH("MEDIA_HEADER.CONTENT_LENGTH"), + MEDIA_HEADER_TIME_RANGE_START("MEDIA_HEADER.TIME_RANGE_START"), + MEDIA_HEADER_TIME_RANGE_DURATION("MEDIA_HEADER.TIME_RANGE_DURATION"), + MEDIA_HEADER_TIME_RANGE_TIMESCALE("MEDIA_HEADER.TIME_RANGE_TIMESCALE"), + MEDIA_HEADER_SEQUENCE_LAST_MODIFIED("MEDIA_HEADER.SEQUENCE_LAST_MODIFIED"); + + @Nonnull private final String id; + + Target(@Nonnull final String id) { + this.id = id; + } + + @Nonnull + public String getId() { + return id; + } + + @Nonnull + static Target fromId(@Nonnull final String id) { + for (final Target target : values()) { + if (target.id.equals(id)) { + return target; + } + } + throw new IllegalArgumentException("Unsupported SABR response target: " + id); + } + } + + private final int partType; + @Nonnull private final Target target; + @Nonnull private final int[] path; + @Nonnull private final WireType wireType; + private final boolean required; + + public SabrProfileResponseMapping(final int partType, + @Nonnull final Target target, + @Nonnull final int[] path, + @Nonnull final WireType wireType, + final boolean required) { + if (partType < 0 || partType > SabrCompatibilityProfile.MAX_UMP_PART_TYPE + || path.length == 0 || path.length > MAX_PATH_DEPTH) { + throw new IllegalArgumentException("Invalid SABR response mapping"); + } + for (final int field : path) { + if (field <= 0 || field > SabrCompatibilityProfile.MAX_PROTOBUF_FIELD_NUMBER) { + throw new IllegalArgumentException("Invalid SABR response field path"); + } + } + this.partType = partType; + this.target = Objects.requireNonNull(target); + this.path = path.clone(); + this.wireType = Objects.requireNonNull(wireType); + if (!accepts(target, wireType)) { + throw new IllegalArgumentException("SABR response target/wire type mismatch"); + } + this.required = required; + } + + private static boolean accepts(@Nonnull final Target target, + @Nonnull final WireType wireType) { + switch (target) { + case NEXT_REQUEST_PLAYBACK_COOKIE: + return wireType == WireType.BYTES; + case NEXT_REQUEST_VIDEO_ID: + case REDIRECT_URL: + case ERROR_TYPE: + case MEDIA_HEADER_VIDEO_ID: + case MEDIA_HEADER_XTAGS: + return wireType == WireType.STRING; + case RELOAD_REQUESTED: + case MEDIA_HEADER_IS_INIT: + return wireType == WireType.BOOL; + default: + return wireType == WireType.VARINT; + } + } + + public int getPartType() { + return partType; + } + + @Nonnull + public Target getTarget() { + return target; + } + + @Nonnull + public int[] getPath() { + return path.clone(); + } + + @Nonnull + int[] getRawPath() { + return path; + } + + @Nonnull + public WireType getWireType() { + return wireType; + } + + public boolean isRequired() { + return required; + } + + void writeCanonical(@Nonnull final DataOutputStream output) throws IOException { + output.writeInt(partType); + SabrCompatibilityProfile.writeString(output, target.getId()); + output.writeByte(path.length); + for (final int field : path) { + output.writeInt(field); + } + SabrCompatibilityProfile.writeString(output, wireType.name()); + output.writeBoolean(required); + } + + @Override + public String toString() { + return partType + ":" + target.getId() + Arrays.toString(path); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrProfileRule.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrProfileRule.java new file mode 100644 index 00000000..3f34a6ff --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrProfileRule.java @@ -0,0 +1,102 @@ +package org.schabi.newpipe.extractor.services.youtube.sabr; + +import javax.annotation.Nonnull; +import java.io.DataOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.EnumSet; +import java.util.List; + +/** One loop-free rule composed only of Host-known predicates and actions. */ +public final class SabrProfileRule { + public enum Predicate { + HAS_MEDIA, + HAS_REDIRECT, + HAS_PROTECTION_BOUNDARY, + DEMANDED_SEGMENT_MISSING, + BACKOFF_PRESENT, + HAS_ERROR, + RELOAD_REQUESTED, + PUMP_MODE, + FETCH_SEGMENT_MODE + } + + public enum Action { + CONTINUE(true), + RETRY(true), + APPLY_REDIRECT(false), + REFRESH_PO_TOKEN(false), + REQUIRE_PO_TOKEN(false), + FAIL_SESSION(true), + TRY_RELOAD(true); + + private final boolean terminal; + + Action(final boolean terminal) { + this.terminal = terminal; + } + + boolean isTerminal() { + return terminal; + } + } + + @Nonnull private final List predicates; + @Nonnull private final List actions; + + public SabrProfileRule(@Nonnull final List predicates, + @Nonnull final List actions) { + if (predicates.size() > Predicate.values().length || actions.isEmpty() + || actions.size() > Action.values().length) { + throw new IllegalArgumentException("Invalid SABR behavior rule size"); + } + final EnumSet uniquePredicates = EnumSet.noneOf(Predicate.class); + final EnumSet uniqueActions = EnumSet.noneOf(Action.class); + int terminals = 0; + for (final Predicate predicate : predicates) { + if (predicate == null || !uniquePredicates.add(predicate)) { + throw new IllegalArgumentException("Duplicate SABR behavior predicate"); + } + } + for (final Action action : actions) { + if (action == null || !uniqueActions.add(action)) { + throw new IllegalArgumentException("Duplicate SABR behavior action"); + } + if (action.isTerminal()) { + terminals++; + } + } + if (terminals != 1 || !actions.get(actions.size() - 1).isTerminal()) { + throw new IllegalArgumentException("SABR behavior rule needs one terminal action"); + } + this.predicates = immutableCopy(predicates); + this.actions = immutableCopy(actions); + } + + @Nonnull + public List getPredicates() { + return predicates; + } + + @Nonnull + public List getActions() { + return actions; + } + + void writeCanonical(@Nonnull final DataOutputStream output) throws IOException { + output.writeByte(predicates.size()); + for (final Predicate predicate : predicates) { + SabrCompatibilityProfile.writeString(output, predicate.name()); + } + output.writeByte(actions.size()); + for (final Action action : actions) { + SabrCompatibilityProfile.writeString(output, action.name()); + } + } + + @Nonnull + private static List immutableCopy(@Nonnull final List values) { + return Collections.unmodifiableList(new ArrayList<>(values)); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrProfileWireValue.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrProfileWireValue.java new file mode 100644 index 00000000..992574cf --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrProfileWireValue.java @@ -0,0 +1,91 @@ +package org.schabi.newpipe.extractor.services.youtube.sabr; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +/** Bounded protobuf path reader used only by compiled profile mappings. */ +final class SabrProfileWireValue { + @Nullable private final Long varint; + @Nullable private final byte[] bytes; + + private SabrProfileWireValue(@Nullable final Long varint, + @Nullable final byte[] bytes) { + this.varint = varint; + this.bytes = bytes; + } + + @Nullable + static SabrProfileWireValue find(@Nonnull final byte[] protobuf, + @Nonnull final SabrProfileResponseMapping mapping) + throws SabrProtocolException { + byte[] current = protobuf; + final int[] path = mapping.getRawPath(); + for (int depth = 0; depth < path.length; depth++) { + SabrProto.Field selected = null; + for (final SabrProto.Field field : SabrProto.readFields(current)) { + if (field.getNumber() == path[depth]) { + selected = field; + } + } + if (selected == null) { + return null; + } + if (depth < path.length - 1) { + if (selected.getWireType() != SabrProto.WIRE_LENGTH_DELIMITED) { + throw new SabrProtocolException("SABR profile path crossed a non-message"); + } + current = selected.getBytes(); + continue; + } + return fromField(selected, mapping.getWireType()); + } + return null; + } + + @Nonnull + private static SabrProfileWireValue fromField( + @Nonnull final SabrProto.Field field, + @Nonnull final SabrProfileResponseMapping.WireType expected) + throws SabrProtocolException { + switch (expected) { + case VARINT: + case BOOL: + if (field.getWireType() != SabrProto.WIRE_VARINT) { + throw new SabrProtocolException("SABR profile expected protobuf varint"); + } + return new SabrProfileWireValue(field.getVarint(), null); + case BYTES: + case STRING: + if (field.getWireType() != SabrProto.WIRE_LENGTH_DELIMITED) { + throw new SabrProtocolException("SABR profile expected protobuf bytes"); + } + return new SabrProfileWireValue(null, field.getBytes()); + default: + throw new SabrProtocolException("Unsupported SABR profile wire type"); + } + } + + long asLong() throws SabrProtocolException { + if (varint == null) { + throw new SabrProtocolException("SABR profile value is not a varint"); + } + return varint; + } + + boolean asBoolean() throws SabrProtocolException { + return asLong() != 0; + } + + @Nonnull + byte[] asBytes() throws SabrProtocolException { + if (bytes == null) { + throw new SabrProtocolException("SABR profile value is not bytes"); + } + return bytes.clone(); + } + + @Nonnull + String asString() throws SabrProtocolException { + return SabrProto.asString(asBytes()); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrScriptPolicy.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrScriptPolicy.java deleted file mode 100644 index 40d69227..00000000 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrScriptPolicy.java +++ /dev/null @@ -1,148 +0,0 @@ -package org.schabi.newpipe.extractor.services.youtube.sabr; - -import org.bouncycastle.crypto.params.Ed25519PublicKeyParameters; -import org.bouncycastle.crypto.signers.Ed25519Signer; - -import javax.annotation.Nonnull; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.DataInputStream; -import java.io.DataOutputStream; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.security.GeneralSecurityException; -import java.security.PublicKey; -import java.security.Signature; - -/** Signed metadata and ordinary JavaScript source. This is a container, not a policy language. */ -public final class SabrScriptPolicy { - private static final int MAGIC = 0x534A5331; - private static final int VERSION = 1; - private static final int MAX_SOURCE_BYTES = 512 * 1024; - private final long revision; - private final long validFromMs; - private final long validUntilMs; - @Nonnull private final String source; - @Nonnull private final byte[] payload; - - public SabrScriptPolicy(final long revision, final long validFromMs, - final long validUntilMs, @Nonnull final String source) { - if (revision < 0 || validFromMs < 0 || validUntilMs <= validFromMs - || source.isEmpty()) { - throw new IllegalArgumentException("Invalid SABR JavaScript policy"); - } - this.revision = revision; - this.validFromMs = validFromMs; - this.validUntilMs = validUntilMs; - this.source = source; - this.payload = encode(revision, validFromMs, validUntilMs, source); - } - - private SabrScriptPolicy(final long revision, final long validFromMs, - final long validUntilMs, @Nonnull final String source, - @Nonnull final byte[] payload) { - this.revision = revision; - this.validFromMs = validFromMs; - this.validUntilMs = validUntilMs; - this.source = source; - this.payload = payload; - } - - public long getRevision() { return revision; } - public long getValidFromMs() { return validFromMs; } - public long getValidUntilMs() { return validUntilMs; } - @Nonnull public String getSource() { return source; } - @Nonnull public byte[] serialize() { return payload.clone(); } - - @Nonnull - public static SabrScriptPolicy parseVerified(@Nonnull final byte[] payload, - @Nonnull final byte[] signature, - @Nonnull final PublicKey key, - final long nowMs, - final long minimumRevision) { - try { - final Signature verifier = Signature.getInstance("Ed25519"); - verifier.initVerify(key); - verifier.update(payload); - if (!verifier.verify(signature)) throw new IllegalArgumentException( - "Invalid SABR JavaScript policy signature"); - } catch (final GeneralSecurityException error) { - throw new IllegalArgumentException("Could not verify SABR JavaScript policy", error); - } - return parse(payload, nowMs, minimumRevision); - } - - @Nonnull - public static SabrScriptPolicy parseVerified(@Nonnull final byte[] payload, - @Nonnull final byte[] signature, - @Nonnull final byte[] rawKey, - final long nowMs, - final long minimumRevision) { - if (rawKey.length != Ed25519PublicKeyParameters.KEY_SIZE) { - throw new IllegalArgumentException("Invalid Ed25519 public key"); - } - final Ed25519Signer verifier = new Ed25519Signer(); - verifier.init(false, new Ed25519PublicKeyParameters(rawKey)); - verifier.update(payload, 0, payload.length); - if (!verifier.verifySignature(signature)) { - throw new IllegalArgumentException("Invalid SABR JavaScript policy signature"); - } - return parse(payload, nowMs, minimumRevision); - } - - @Nonnull - private static SabrScriptPolicy parse(@Nonnull final byte[] payload, final long nowMs, - final long minimumRevision) { - if (payload.length == 0 || payload.length > MAX_SOURCE_BYTES + 64) { - throw new IllegalArgumentException("Invalid SABR JavaScript policy size"); - } - try { - final DataInputStream input = new DataInputStream(new ByteArrayInputStream(payload)); - if (input.readInt() != MAGIC || input.readUnsignedByte() != VERSION) { - throw new IllegalArgumentException("Unsupported SABR JavaScript policy"); - } - final long revision = input.readLong(); - final long from = input.readLong(); - final long until = input.readLong(); - final int size = input.readInt(); - if (revision < minimumRevision || from < 0 || until <= from - || size <= 0 || size > MAX_SOURCE_BYTES) { - throw new IllegalArgumentException("Invalid SABR JavaScript policy metadata"); - } - final byte[] source = new byte[size]; - input.readFully(source); - if (input.available() != 0) throw new IllegalArgumentException( - "Trailing SABR JavaScript policy bytes"); - if (nowMs < from || nowMs >= until) throw new IllegalArgumentException( - "SABR JavaScript policy is not currently valid"); - return new SabrScriptPolicy(revision, from, until, - new String(source, StandardCharsets.UTF_8), payload.clone()); - } catch (final IOException error) { - throw new IllegalArgumentException("Malformed SABR JavaScript policy", error); - } - } - - @Nonnull - private static byte[] encode(final long revision, final long from, final long until, - @Nonnull final String source) { - final byte[] script = source.getBytes(StandardCharsets.UTF_8); - if (script.length == 0 || script.length > MAX_SOURCE_BYTES) { - throw new IllegalArgumentException("Invalid SABR JavaScript source size"); - } - try { - final ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - final DataOutputStream output = new DataOutputStream(bytes); - output.writeInt(MAGIC); - output.writeByte(VERSION); - output.writeLong(revision); - output.writeLong(from); - output.writeLong(until); - output.writeInt(script.length); - output.write(script); - output.flush(); - return bytes.toByteArray(); - } catch (final IOException impossible) { - throw new IllegalStateException(impossible); - } - } -} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrScriptPolicyDocument.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrScriptPolicyDocument.java deleted file mode 100644 index b88ebcfc..00000000 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrScriptPolicyDocument.java +++ /dev/null @@ -1,128 +0,0 @@ -package org.schabi.newpipe.extractor.services.youtube.sabr; - -import com.grack.nanojson.JsonObject; -import com.grack.nanojson.JsonParser; -import com.grack.nanojson.JsonParserException; -import com.grack.nanojson.JsonWriter; -import javax.annotation.Nonnull; -import java.math.BigInteger; -import java.nio.charset.StandardCharsets; -import java.util.Base64; - -/** - * Human-readable delivery document for a signed SABR JavaScript policy. - * - *

The signature covers the canonical payload returned by - * {@link SabrScriptPolicy#serialize()}, not the JSON representation itself. A decoder therefore - * reconstructs that payload from the signed metadata and source before verification.

- */ -public final class SabrScriptPolicyDocument { - private static final int FORMAT_VERSION = 1; - private static final int MAX_DOCUMENT_BYTES = 1024 * 1024; - private static final int MAX_SIGNATURE_BYTES = 1024; - - private SabrScriptPolicyDocument() { - } - - /** Encodes one policy and its detached signature as a single UTF-8 JSON document. */ - @Nonnull - public static byte[] encode(@Nonnull final SabrScriptPolicy policy, - @Nonnull final byte[] signature) { - validateSignature(signature); - final JsonObject document = new JsonObject(); - document.put("format", FORMAT_VERSION); - document.put("revision", policy.getRevision()); - document.put("validFromMs", policy.getValidFromMs()); - document.put("validUntilMs", policy.getValidUntilMs()); - document.put("source", policy.getSource()); - document.put("signature", Base64.getEncoder().encodeToString(signature)); - final byte[] encoded = JsonWriter.string(document).getBytes(StandardCharsets.UTF_8); - if (encoded.length > MAX_DOCUMENT_BYTES) { - throw new IllegalArgumentException("SABR policy document exceeded size limit"); - } - return encoded; - } - - /** Decodes JSON and reconstructs the exact payload which must be signature verified. */ - @Nonnull - public static Parsed decode(@Nonnull final byte[] encoded) { - if (encoded.length == 0 || encoded.length > MAX_DOCUMENT_BYTES) { - throw new IllegalArgumentException("Invalid SABR policy document size"); - } - try { - final JsonObject document = JsonParser.object().from( - new String(encoded, StandardCharsets.UTF_8)); - if (document.size() != 6 - || !document.has("revision") - || !document.has("validFromMs") - || !document.has("validUntilMs") - || !document.has("source") - || !document.has("signature") - || requireLong(document, "format") != FORMAT_VERSION) { - throw new IllegalArgumentException("Unsupported SABR policy document"); - } - final String source = document.getString("source"); - final String encodedSignature = document.getString("signature"); - if (source == null || encodedSignature == null) { - throw new IllegalArgumentException("Invalid SABR policy document fields"); - } - final byte[] signature = Base64.getDecoder().decode(encodedSignature); - validateSignature(signature); - final SabrScriptPolicy policy = new SabrScriptPolicy( - requireLong(document, "revision"), - requireLong(document, "validFromMs"), - requireLong(document, "validUntilMs"), - source); - return new Parsed(policy.serialize(), signature); - } catch (final IllegalArgumentException error) { - throw error; - } catch (final JsonParserException | RuntimeException error) { - throw new IllegalArgumentException("Malformed SABR policy document", error); - } - } - - private static long requireLong(@Nonnull final JsonObject document, - @Nonnull final String field) { - final Object value = document.get(field); - if (value instanceof Integer || value instanceof Long) { - return ((Number) value).longValue(); - } - if (value instanceof BigInteger) { - try { - return ((BigInteger) value).longValueExact(); - } catch (final ArithmeticException error) { - throw new IllegalArgumentException( - "SABR policy document integer is out of range: " + field, error); - } - } - throw new IllegalArgumentException( - "SABR policy document field is not an exact integer: " + field); - } - - private static void validateSignature(@Nonnull final byte[] signature) { - if (signature.length == 0 || signature.length > MAX_SIGNATURE_BYTES) { - throw new IllegalArgumentException("Invalid SABR policy signature size"); - } - } - - /** Canonical signed payload and its detached signature. */ - public static final class Parsed { - @Nonnull private final byte[] payload; - @Nonnull private final byte[] signature; - - private Parsed(@Nonnull final byte[] payload, @Nonnull final byte[] signature) { - this.payload = payload; - this.signature = signature; - } - - @Nonnull - public byte[] getPayload() { - return payload.clone(); - } - - @Nonnull - public byte[] getSignature() { - return signature.clone(); - } - } -} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrScriptPolicyManager.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrScriptPolicyManager.java deleted file mode 100644 index 6c4e7bbe..00000000 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrScriptPolicyManager.java +++ /dev/null @@ -1,75 +0,0 @@ -package org.schabi.newpipe.extractor.services.youtube.sabr; - -import javax.annotation.Nonnull; -import javax.annotation.Nullable; -import java.security.PublicKey; -import java.util.Arrays; - -/** Thread-safe verifier and monotonic activation boundary for signed JavaScript source. */ -public final class SabrScriptPolicyManager { - @Nullable private final PublicKey publicKey; - @Nullable private final byte[] rawPublicKey; - @Nullable private volatile SabrScriptPolicy active; - private long highestRevision; - - public SabrScriptPolicyManager(@Nonnull final PublicKey key, final long minimumRevision) { - if (minimumRevision < 0) throw new IllegalArgumentException("Invalid policy revision"); - publicKey = key; - rawPublicKey = null; - highestRevision = minimumRevision; - } - - public SabrScriptPolicyManager(@Nonnull final byte[] key, final long minimumRevision) { - if (key.length != 32 || minimumRevision < 0) { - throw new IllegalArgumentException("Invalid policy verifier"); - } - publicKey = null; - rawPublicKey = key.clone(); - highestRevision = minimumRevision; - } - - @Nonnull - public synchronized SabrScriptPolicy verify(@Nonnull final byte[] payload, - @Nonnull final byte[] signature, - final long nowMs) { - return rawPublicKey == null - ? SabrScriptPolicy.parseVerified(payload, signature, publicKey, - nowMs, highestRevision) - : SabrScriptPolicy.parseVerified(payload, signature, rawPublicKey, - nowMs, highestRevision); - } - - public synchronized void activate(@Nonnull final SabrScriptPolicy verified) { - if (verified.getRevision() < highestRevision) { - throw new IllegalArgumentException("SABR policy rollback rejected"); - } - if (active != null && active.getRevision() == verified.getRevision() - && !Arrays.equals(active.serialize(), verified.serialize())) { - throw new IllegalArgumentException("Conflicting SABR policy revision"); - } - active = verified; - highestRevision = verified.getRevision(); - } - - @Nonnull - public synchronized SabrScriptPolicy install(@Nonnull final byte[] payload, - @Nonnull final byte[] signature, - final long nowMs) { - final SabrScriptPolicy verified = verify(payload, signature, nowMs); - activate(verified); - return verified; - } - - @Nullable - public SabrScriptPolicy current(final long nowMs) { - final SabrScriptPolicy value = active; - return value != null && nowMs >= value.getValidFromMs() && nowMs < value.getValidUntilMs() - ? value : null; - } - - public synchronized long getHighestRevision() { return highestRevision; } - - public synchronized void deactivate(@Nonnull final SabrScriptPolicy expected) { - if (active == expected) active = null; - } -} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrSessionPolicy.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrSessionPolicy.java index b8d56c2d..c84135b7 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrSessionPolicy.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrSessionPolicy.java @@ -217,15 +217,25 @@ final class RequestEvent extends Event { private final int poTokenBytes; private final int bufferedRangeCount; @Nonnull private final byte[] proposedBody; + @Nullable private final SabrProfileRequestData profileRequestData; public RequestEvent(final long playerTimeMs, final long bufferedEdgeMs, final int poTokenBytes, final int bufferedRangeCount, @Nonnull final byte[] proposedBody) { + this(playerTimeMs, bufferedEdgeMs, poTokenBytes, bufferedRangeCount, proposedBody, + null); + } + + RequestEvent(final long playerTimeMs, final long bufferedEdgeMs, + final int poTokenBytes, final int bufferedRangeCount, + @Nonnull final byte[] proposedBody, + @Nullable final SabrProfileRequestData profileRequestData) { this.playerTimeMs = playerTimeMs; this.bufferedEdgeMs = bufferedEdgeMs; this.poTokenBytes = poTokenBytes; this.bufferedRangeCount = bufferedRangeCount; this.proposedBody = proposedBody.clone(); + this.profileRequestData = profileRequestData; } public long getPlayerTimeMs() { return playerTimeMs; } @@ -233,6 +243,23 @@ public RequestEvent(final long playerTimeMs, final long bufferedEdgeMs, public int getPoTokenBytes() { return poTokenBytes; } public int getBufferedRangeCount() { return bufferedRangeCount; } @Nonnull public byte[] getProposedBody() { return proposedBody.clone(); } + + @Nonnull + byte[] buildProfileRequest(@Nonnull final List template) + throws SabrProtocolException { + if (profileRequestData == null) { + throw new SabrProtocolException("SABR profile request data is unavailable"); + } + return profileRequestData.build(template); + } + + @Nonnull + SabrProfileRequestData getProfileRequestData() throws SabrProtocolException { + if (profileRequestData == null) { + throw new SabrProtocolException("SABR profile request data is unavailable"); + } + return profileRequestData; + } } final class ControlResponseEvent extends Event { @@ -332,14 +359,14 @@ public static Result control(@Nonnull final State state, @Nonnull final List SabrScriptPolicy.parseVerified( - changed.getPayload(), changed.getSignature(), keys.getPublic(), NOW, 0)); - } - - @Test - void policyDocumentRejectsNonIntegralAndOutOfRangeMetadata() throws Exception { - final KeyPair keys = KeyPairGenerator.getInstance("Ed25519").generateKeyPair(); - final SabrScriptPolicy policy = new SabrScriptPolicy(7, NOW - 1, NOW + 1, SCRIPT); - final byte[] payload = policy.serialize(); - final String document = new String(SabrScriptPolicyDocument.encode( - policy, sign(payload, keys)), StandardCharsets.UTF_8); - - assertInvalidDocument(document.replace("\"format\":1", "\"format\":1.9")); - assertInvalidDocument(document.replace("\"format\":1", "\"format\":1e0")); - assertInvalidDocument(document.replace("\"revision\":7", "\"revision\":7.9")); - assertInvalidDocument(document.replace("\"revision\":7", "\"revision\":7e0")); - assertInvalidDocument(document.replace("\"validFromMs\":1999999", - "\"validFromMs\":1999999.0")); - assertInvalidDocument(document.replace("\"validUntilMs\":2000001", - "\"validUntilMs\":2000001e0")); - assertInvalidDocument(document.replace("\"revision\":7", - "\"revision\":9223372036854775808")); - assertInvalidDocument(document.replace("\"validFromMs\":1999999", - "\"validFromMs\":-9223372036854775809")); - assertInvalidDocument(document.replace("\"format\":1", "\"format\":\"1\"")); - assertInvalidDocument(document.replace("\"revision\":7", "\"revision\":true")); - assertInvalidDocument(document.replace("\"validFromMs\":1999999", - "\"validFromMs\":null")); - assertInvalidDocument(document.replace("\"validUntilMs\":2000001", - "\"validUntilMs\":\"2000001\"")); - - SabrScriptPolicyDocument.decode(document.replace("\"revision\":7", - "\"revision\":9223372036854775807").getBytes(StandardCharsets.UTF_8)); - SabrScriptPolicyDocument.decode(document.replace("\"validUntilMs\":2000001", - "\"validUntilMs\":9223372036854775807").getBytes(StandardCharsets.UTF_8)); - } - - private static void assertInvalidDocument(@Nonnull final String document) { - assertThrows(IllegalArgumentException.class, () -> SabrScriptPolicyDocument.decode( - document.getBytes(StandardCharsets.UTF_8))); - } - - @Test - void rejectsTamperingRollbackAndExpiry() throws Exception { - final KeyPair keys = KeyPairGenerator.getInstance("Ed25519").generateKeyPair(); - final byte[] payload = new SabrScriptPolicy(7, NOW - 1, NOW + 1, SCRIPT).serialize(); - assertThrows(IllegalArgumentException.class, () -> SabrScriptPolicy.parseVerified( - payload, sign(payload, keys), keys.getPublic(), NOW, 8)); - assertThrows(IllegalArgumentException.class, () -> SabrScriptPolicy.parseVerified( - payload, sign(payload, keys), keys.getPublic(), NOW + 1, 0)); - final byte[] tampered = payload.clone(); - tampered[tampered.length - 1] ^= 1; - assertThrows(IllegalArgumentException.class, () -> SabrScriptPolicy.parseVerified( - tampered, sign(payload, keys), keys.getPublic(), NOW, 0)); - } - - @Test - void hostRejectsForgedRecoveryStateAndMultipleTerminalActions() { - final SabrDecodedResponse decoded = new SabrDecodedResponse(); - final SabrSessionPolicy.ControlResponseEvent event = - new SabrSessionPolicy.ControlResponseEvent(0, true, - SabrSessionPolicy.ControlMode.PUMP, decoded); - final SabrSessionPolicy.State state = new SabrSessionPolicy.State(1, 0, 0, 0); - - assertThrows(IllegalStateException.class, () -> hostReturning( - SabrSessionPolicy.Result.control(state, Arrays.asList( - new SabrSessionPolicy.Action(SabrSessionPolicy.ActionType.CONTINUE), - new SabrSessionPolicy.Action(SabrSessionPolicy.ActionType.RETRY)), - new SabrSessionPolicy.ControlDecision(0, null, null))) - .evaluate(state, event)); - assertThrows(IllegalStateException.class, () -> hostReturning( - SabrSessionPolicy.Result.control(state, Arrays.asList( - new SabrSessionPolicy.Action( - SabrSessionPolicy.ActionType.APPLY_RESPONSE_STATE), - new SabrSessionPolicy.Action(SabrSessionPolicy.ActionType.CONTINUE)), - new SabrSessionPolicy.ControlDecision(0, null, null))) - .evaluate(state, event)); - assertThrows(IllegalStateException.class, () -> hostReturning( - SabrSessionPolicy.Result.control(state, - Collections.singletonList(new SabrSessionPolicy.Action( - SabrSessionPolicy.ActionType.CONTINUE)), - new SabrSessionPolicy.ControlDecision(0, null, null), - SabrResponseStatePatch.builder().build())) - .evaluate(state, event)); - assertThrows(IllegalStateException.class, () -> hostReturning( - SabrSessionPolicy.Result.control(new SabrSessionPolicy.State(1, 2, 0, 0), - Collections.singletonList(new SabrSessionPolicy.Action( - SabrSessionPolicy.ActionType.CONTINUE)), - new SabrSessionPolicy.ControlDecision(0, null, null))) - .evaluate(state, event)); - assertThrows(IllegalStateException.class, () -> hostReturning( - SabrSessionPolicy.Result.control(state, - Collections.singletonList(new SabrSessionPolicy.Action( - SabrSessionPolicy.ActionType.CONTINUE)), - new SabrSessionPolicy.ControlDecision(0, - "https://example.invalid", null))) - .evaluate(state, event)); - } - - @Test - void builtinDemandPolicyRecoversEveryOmittedDemandedSegment() throws Exception { - final BuiltinSabrSessionPolicy policy = new BuiltinSabrSessionPolicy(); - final SabrSessionPolicy.DemandState firstOmission = - new SabrSessionPolicy.DemandState(1_000, 1_100, 1, 0); - assertEquals(SabrSessionPolicy.DemandRoute.RECOVER_MISSING, - policy.evaluateDemandRoute(new SabrSessionPolicy.DemandRouteEvent( - 251, 7, 30_000, 25_000, firstOmission))); - - final SabrSessionPolicy.DemandResponseDecision retry = - policy.evaluateDemandResponse(new SabrSessionPolicy.DemandResponseEvent( - 251, 7, 30_000, 25_000, firstOmission, - 2, 0, Collections.singletonList( - new SabrSessionPolicy.DemandReturnedSegment(398, 9, - 30_000, 5_000)), false)); - assertEquals(SabrSessionPolicy.DemandOutcome.CONTINUE, retry.getOutcome()); - - final SabrSessionPolicy.DemandState thirdOmission = - new SabrSessionPolicy.DemandState(1_000, 1_300, 3, 1); - assertEquals(SabrSessionPolicy.DemandOutcome.FAIL_REPEATED_TARGET_OMISSION, - policy.evaluateDemandResponse(new SabrSessionPolicy.DemandResponseEvent( - 251, 7, 30_000, 25_000, thirdOmission, - 1, 0, Collections.singletonList( - new SabrSessionPolicy.DemandReturnedSegment(398, 10, - 35_000, 5_000)), false)).getOutcome()); - } - - @Test - void hostRejectsInvalidDemandEventsAndDecisions() { - final SabrSessionPolicy.State state = new SabrSessionPolicy.State(1, 0, 0, 0); - final SabrSessionPolicy policy = new SabrSessionPolicy() { - @Nonnull - @Override - public Result evaluate(@Nonnull final State ignoredState, - @Nonnull final Event ignoredEvent) { - return Result.request(state, ActionType.SEND_FOLLOW_UP_REQUEST, new byte[]{1}); - } - - @Nonnull - @Override - public DemandResponseDecision evaluateDemandResponse( - @Nonnull final DemandResponseEvent event) { - return new DemandResponseDecision(DemandOutcome.CONTINUE, - MAX_DEMAND_RETRY_DELAY_MS + 1); - } - }; - final SabrSessionPolicyHost host = new SabrSessionPolicyHost(policy, null); - assertThrows(IllegalArgumentException.class, () -> host.evaluateDemandRoute( - new SabrSessionPolicy.DemandRouteEvent(0, 1, 0, 0, - new SabrSessionPolicy.DemandState(1, 1, 0, 0)))); - assertThrows(IllegalStateException.class, () -> host.evaluateDemandResponse( - new SabrSessionPolicy.DemandResponseEvent(251, 1, 0, 0, - new SabrSessionPolicy.DemandState(1, 2, 1, 0), - 1, 0, Collections.singletonList( - new SabrSessionPolicy.DemandReturnedSegment(398, 2, - 0, 5_000)), false))); - } - - @Test - void normalizedPatchUpdatesRequestLiveAndFormatState() throws Exception { - final JsonArray formats = new JsonArray(); - final JsonObject audio = new JsonObject(); - audio.put("itag", 140); - audio.put("mimeType", "audio/mp4"); - audio.put("lastModified", "1"); - audio.put("approxDurationMs", "100000"); - formats.add(audio); - final JsonObject video = new JsonObject(); - video.put("itag", 134); - video.put("mimeType", "video/mp4"); - video.put("lastModified", "2"); - video.put("approxDurationMs", "100000"); - formats.add(video); - final java.util.List parsed = - YoutubeSabrFormat.fromAdaptiveFormats("video", formats); - final YoutubeSabrStreamState streamState = new YoutubeSabrStreamState( - parsed.get(0), parsed.get(1)); - final byte[] cookie = new byte[]{7, 8, 9}; - final SabrResponseStatePatch patch = SabrResponseStatePatch.builder() - .setNextRequestPolicy(SabrNextRequestPolicy.normalized( - 9_000, 10_000, 2_000, 0, 3_000, 4_000, cookie, "video")) - .addLiveMetadata(SabrLiveMetadata.normalized( - null, 77, 123_000, -1, "video", false, - -1, -1, -1, -1, -1)) - .addFormatMetadata(SabrFormatInitializationMetadata.normalized( - "video", 140, 1, null, 100_000, 20, - "audio/mp4", -1, -1, -1, -1, -1, 100, 1)) - .build(); - - streamState.ingest(patch); - - assertArrayEquals(cookie, streamState.getPlaybackCookie()); - assertEquals(9_000, streamState.getNextRequestPolicy().getTargetAudioReadaheadMs()); - assertTrue(streamState.isLive()); - assertEquals(77, streamState.getLiveHeadSequenceNumber()); - assertEquals(20, streamState.getEndSegment(parsed.get(0))); - } - - private static SabrSessionPolicyHost hostReturning(final SabrSessionPolicy.Result result) { - return new SabrSessionPolicyHost((state, event) -> result, null); - } - - private static byte[] sign(final byte[] payload, final KeyPair keys) throws Exception { - final Signature signer = Signature.getInstance("Ed25519"); - signer.initSign(keys.getPrivate()); - signer.update(payload); - return signer.sign(); - } -}