diff --git a/build.gradle.kts b/build.gradle.kts index 8942f4b2..6f0306df 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -36,7 +36,7 @@ dependencies { implementation("io.ktor:ktor-server-call-logging-jvm") implementation("io.ktor:ktor-server-rate-limit-jvm") implementation("ch.qos.logback:logback-classic:1.5.38") - implementation("com.github.InfinityLoop1308.PipePipeExtractor:extractor:00db0e1d9b553d941f3009eb79e492d95eaf442d") + implementation("com.github.Priveetee.PipePipeExtractor:extractor:d1a723d382014c2428b273c92bc9fc8c15d59c83") compileOnly("com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751") implementation("org.json:json:20260522") implementation("com.squareup.okhttp3:okhttp:5.4.0") diff --git a/docs/android-playback.md b/docs/android-playback.md new file mode 100644 index 00000000..96faed3c --- /dev/null +++ b/docs/android-playback.md @@ -0,0 +1,17 @@ +# Android playback contract + +TypeType-Android discovers this contract through `androidPlayback` in +`GET /api/instance`. Contract version 1 supports completed YouTube VODs only; +active livestreams remain explicitly unsupported. + +Android playback sessions use `/api/android/youtube/playback/*` and are isolated +from the web player's `/api/sabr/playback/*` sessions, generations, caches, and +window protocol. A ready VOD manifest is a complete static DASH presentation +from time zero. The server requires exact audio and video segment indexes, but +continues to fetch media bytes on demand. + +Creating a session may return `202` while the exact indexes are preparing. A +seek keeps the session ID and selected itags stable, increments the generation, +and makes older media URLs return `409`. Unknown sessions return `404`; recently +expired sessions return `410`. Session manifests and media remain same-origin +and use `Cache-Control: no-store`. diff --git a/openapi.yaml b/openapi.yaml index e800a45a..532a8bd4 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -14,6 +14,7 @@ tags: - name: extraction - name: downloader - name: youtube-session + - name: android-playback - name: user-data paths: /health: { $ref: ./openapi/paths/health.yaml#/Health } @@ -27,6 +28,12 @@ paths: /streams/bilibili: { $ref: ./openapi/paths/streams.yaml#/BiliBiliStreams } /streams/audio-only: { $ref: ./openapi/paths/streams.yaml#/AudioOnly } /streams/audio-only/source: { $ref: ./openapi/paths/streams.yaml#/AudioOnlySource } + /android/youtube/playback/{videoId}: { $ref: ./openapi/paths/android-playback.yaml#/Create } + /android/youtube/playback/{sessionId}/seek: { $ref: ./openapi/paths/android-playback.yaml#/Seek } + /android/youtube/playback/{sessionId}/manifest.mpd: { $ref: ./openapi/paths/android-playback.yaml#/Manifest } + /android/youtube/playback/{sessionId}/subtitles/{trackId}.vtt: { $ref: ./openapi/paths/android-playback.yaml#/Subtitle } + /android/youtube/playback/{sessionId}/{itag}/init: { $ref: ./openapi/paths/android-playback.yaml#/Initialization } + /android/youtube/playback/{sessionId}/{itag}/segment/{sequence}: { $ref: ./openapi/paths/android-playback.yaml#/Segment } /search: { $ref: ./openapi/paths/search.yaml#/Search } /search/filters: { $ref: ./openapi/paths/search.yaml#/SearchFilters } /channel: { $ref: ./openapi/paths/channel.yaml#/Channel } @@ -37,6 +44,7 @@ paths: /playlist: { $ref: ./openapi/paths/playlists.yaml#/Playlist } /saved-playlists: { $ref: ./openapi/paths/saved-playlists.yaml#/SavedPlaylists } /saved-playlists/{id}: { $ref: ./openapi/paths/saved-playlists.yaml#/SavedPlaylist } + /subscriptions/feed: { $ref: ./openapi/paths/subscriptions.yaml#/SubscriptionFeed } /settings: { $ref: ./openapi/paths/access-control.yaml#/Settings } /allowed/channels: { $ref: ./openapi/paths/access-control.yaml#/AllowedChannels } /allowed/channels/{channelUrl}: { $ref: ./openapi/paths/access-control.yaml#/AllowedChannel } @@ -88,6 +96,12 @@ components: $ref: ./openapi/components/instance.yaml#/InstanceMinClientVersion InstanceResponse: $ref: ./openapi/components/instance.yaml#/InstanceResponse + AndroidPlaybackCreateRequest: + $ref: ./openapi/components/android-playback.yaml#/AndroidPlaybackCreateRequest + AndroidPlaybackSeekRequest: + $ref: ./openapi/components/android-playback.yaml#/AndroidPlaybackSeekRequest + AndroidPlaybackResponse: + $ref: ./openapi/components/android-playback.yaml#/AndroidPlaybackResponse StreamResponse: $ref: ./openapi/components/streams.yaml#/StreamResponse AudioOnlyStreamResponse: @@ -118,6 +132,8 @@ components: $ref: ./openapi/components/media.yaml#/PublicPlaylistItem SavedPlaylistItem: { $ref: ./openapi/components/media.yaml#/SavedPlaylistItem } SavedPlaylistRequest: { $ref: ./openapi/components/media.yaml#/SavedPlaylistRequest } + SubscriptionFeedResponse: { $ref: ./openapi/components/subscriptions.yaml#/SubscriptionFeedResponse } + SubscriptionFeedPreparingResponse: { $ref: ./openapi/components/subscriptions.yaml#/SubscriptionFeedPreparingResponse } SettingsItem: { $ref: ./openapi/components/access-control.yaml#/SettingsItem } AdminSettingsItem: { $ref: ./openapi/components/access-control.yaml#/AdminSettingsItem } AllowedChannelItem: { $ref: ./openapi/components/access-control.yaml#/AllowedChannelItem } diff --git a/openapi/components/android-playback.yaml b/openapi/components/android-playback.yaml new file mode 100644 index 00000000..a7fe4c38 --- /dev/null +++ b/openapi/components/android-playback.yaml @@ -0,0 +1,50 @@ +AndroidPlaybackCreateRequest: + type: object + properties: + videoItag: { type: integer, nullable: true } + audioItag: { type: integer, nullable: true } + audioTrackId: { type: string, nullable: true } +AndroidPlaybackSeekRequest: + type: object + required: [generation, playerTimeMs] + properties: + generation: { type: integer, format: int64, minimum: 0 } + playerTimeMs: { type: integer, format: int64, minimum: 0 } +AndroidPlaybackResponse: + type: object + required: + - sessionId + - videoId + - manifestUrl + - videoItag + - audioItag + - generation + - ready + - status + - subtitles + properties: + sessionId: { type: string } + videoId: { type: string } + manifestUrl: { type: string } + videoItag: { type: integer } + audioItag: { type: integer } + audioTrackId: { type: string, nullable: true } + generation: { type: integer, format: int64, minimum: 0 } + ready: { type: boolean } + status: { type: string, enum: [ready, preparing] } + subtitles: + type: array + items: { $ref: '#/AndroidSubtitle' } + retryAfterMs: { type: integer, format: int64, nullable: true, minimum: 100, maximum: 2000 } +AndroidSubtitle: + type: object + required: [id, mimeType, languageTag, displayLanguageName, isAutoGenerated, url] + properties: + id: { type: string } + mimeType: { type: string, enum: [text/vtt] } + languageTag: { type: string } + displayLanguageName: { type: string } + isAutoGenerated: { type: boolean } + url: + type: string + description: Session-scoped same-origin resource that remains stable across playback generations. diff --git a/openapi/components/instance.yaml b/openapi/components/instance.yaml index 676c77c6..8cdf3b67 100644 --- a/openapi/components/instance.yaml +++ b/openapi/components/instance.yaml @@ -20,6 +20,7 @@ InstanceResponse: - oidcAutoRedirect - youtubeRemoteLoginEnabled - youtubeRemoteLoginReady + - androidPlayback properties: name: { type: string, example: TypeType } tagline: { type: string, nullable: true } @@ -49,3 +50,20 @@ InstanceResponse: type: string nullable: true enum: [disabled, not_configured, token_unreachable] + androidPlayback: + $ref: '#/AndroidPlaybackCapability' +AndroidPlaybackCapability: + type: object + required: [supported, contractVersion, youtube] + properties: + supported: { type: boolean, example: true } + contractVersion: { type: integer, enum: [2] } + youtube: + $ref: '#/AndroidYoutubePlaybackCapability' +AndroidYoutubePlaybackCapability: + type: object + required: [vod, live, subtitles] + properties: + vod: { type: boolean, example: true } + live: { type: boolean, example: false } + subtitles: { type: boolean, example: true } diff --git a/openapi/components/subscriptions.yaml b/openapi/components/subscriptions.yaml new file mode 100644 index 00000000..b0e7ad59 --- /dev/null +++ b/openapi/components/subscriptions.yaml @@ -0,0 +1,35 @@ +SubscriptionFeedResponse: + type: object + required: [videos, nextpage, generation, generatedAt, refreshing] + properties: + videos: + type: array + maxItems: 100 + items: + $ref: ./media.yaml#/VideoItem + nextpage: + type: string + nullable: true + description: Opaque continuation bound to this snapshot generation and page size. + generation: + type: integer + format: int64 + generatedAt: + type: integer + format: int64 + description: Unix timestamp in milliseconds for the snapshot build. + refreshing: + type: boolean + description: True while a stale snapshot is being rebuilt in the background. +SubscriptionFeedPreparingResponse: + type: object + required: [code, retryAfterMs] + properties: + code: + type: string + enum: [subscription_feed_preparing] + retryAfterMs: + type: integer + format: int64 + minimum: 100 + maximum: 5000 diff --git a/openapi/paths/android-playback.yaml b/openapi/paths/android-playback.yaml new file mode 100644 index 00000000..53f4919c --- /dev/null +++ b/openapi/paths/android-playback.yaml @@ -0,0 +1,160 @@ +Create: + parameters: + - name: videoId + in: path + required: true + schema: { type: string } + post: + tags: [android-playback] + summary: Create an Android-only YouTube VOD playback session + description: Active livestreams are rejected. Media remains demand-driven after the complete virtual index is ready. The response includes the authoritative subtitle inventory before Media3 prepares the item. + requestBody: + required: false + content: + application/json: + schema: { $ref: ../components/android-playback.yaml#/AndroidPlaybackCreateRequest } + responses: + '200': { $ref: '#/components/responses/ReadySession' } + '202': { $ref: '#/components/responses/PreparingSession' } + '400': { $ref: ../components/common.yaml#/JsonError } + '401': { $ref: ../components/common.yaml#/JsonError } + '403': { $ref: ../components/common.yaml#/JsonError } + '422': { $ref: ../components/common.yaml#/JsonError } + '503': { $ref: ../components/common.yaml#/JsonError } +Seek: + parameters: + - $ref: '#/components/parameters/SessionId' + post: + tags: [android-playback] + summary: Reposition an Android VOD playback session + description: The session ID and selected formats stay stable while the generation increments. + requestBody: + required: true + content: + application/json: + schema: { $ref: ../components/android-playback.yaml#/AndroidPlaybackSeekRequest } + responses: + '200': { $ref: '#/components/responses/ReadySession' } + '202': { $ref: '#/components/responses/PreparingSession' } + '400': { $ref: ../components/common.yaml#/JsonError } + '404': { $ref: ../components/common.yaml#/JsonError } + '409': { $ref: ../components/common.yaml#/JsonError } + '410': { $ref: ../components/common.yaml#/JsonError } + '422': { $ref: ../components/common.yaml#/JsonError } +Manifest: + parameters: + - $ref: '#/components/parameters/SessionId' + get: + tags: [android-playback] + summary: Get the complete static DASH presentation for an Android VOD session + responses: + '200': + description: Complete standards-compatible static DASH MPD. + headers: + Cache-Control: + schema: { type: string, example: no-store } + content: + application/dash+xml: + schema: { type: string } + '202': { $ref: '#/components/responses/PreparingSession' } + '404': { $ref: ../components/common.yaml#/JsonError } + '410': { $ref: ../components/common.yaml#/JsonError } + '422': { $ref: ../components/common.yaml#/JsonError } +Subtitle: + parameters: + - $ref: '#/components/parameters/SessionId' + - $ref: '#/components/parameters/TrackId' + get: + tags: [android-playback] + summary: Get one session-scoped Android subtitle as UTF-8 WebVTT + description: The caller must use the same account context that created the playback session. The resource remains valid across seek generations. + responses: + '200': + description: Complete UTF-8 WebVTT subtitle document. + headers: + Cache-Control: + schema: { type: string, example: no-store } + content: + text/vtt: + schema: { type: string } + '401': { $ref: ../components/common.yaml#/JsonError } + '404': { $ref: ../components/common.yaml#/JsonError } + '410': { $ref: ../components/common.yaml#/JsonError } + '422': { $ref: ../components/common.yaml#/JsonError } + '503': { $ref: ../components/common.yaml#/JsonError } +Initialization: + parameters: + - $ref: '#/components/parameters/SessionId' + - $ref: '#/components/parameters/Itag' + - $ref: '#/components/parameters/SessionQuery' + - $ref: '#/components/parameters/Generation' + get: + tags: [android-playback] + summary: Get initialization bytes for a selected Android playback track + responses: + '200': { description: Initialization bytes. } + '202': { $ref: '#/components/responses/PreparingSession' } + '206': { description: Partial initialization bytes. } + '400': { $ref: ../components/common.yaml#/JsonError } + '404': { $ref: ../components/common.yaml#/JsonError } + '409': { $ref: ../components/common.yaml#/JsonError } + '410': { $ref: ../components/common.yaml#/JsonError } +Segment: + parameters: + - $ref: '#/components/parameters/SessionId' + - $ref: '#/components/parameters/Itag' + - name: sequence + in: path + required: true + schema: { type: integer, minimum: 1 } + - $ref: '#/components/parameters/SessionQuery' + - $ref: '#/components/parameters/Generation' + get: + tags: [android-playback] + summary: Get one demand-driven media segment for an Android playback track + responses: + '200': { description: Media segment bytes. } + '202': { $ref: '#/components/responses/PreparingSession' } + '206': { description: Partial media segment bytes. } + '400': { $ref: ../components/common.yaml#/JsonError } + '404': { $ref: ../components/common.yaml#/JsonError } + '409': { $ref: ../components/common.yaml#/JsonError } + '410': { $ref: ../components/common.yaml#/JsonError } +components: + parameters: + SessionId: + name: sessionId + in: path + required: true + schema: { type: string } + Itag: + name: itag + in: path + required: true + schema: { type: integer } + TrackId: + name: trackId + in: path + required: true + schema: { type: string } + SessionQuery: + name: session + in: query + required: true + schema: { type: string } + Generation: + name: generation + in: query + required: true + schema: { type: integer, format: int64, minimum: 0 } + responses: + ReadySession: + description: Playback session with a complete VOD presentation. + content: + application/json: + schema: { $ref: ../components/android-playback.yaml#/AndroidPlaybackResponse } + PreparingSession: + description: The exact virtual segment index or requested bytes are still preparing. + content: + application/json: + schema: { $ref: ../components/android-playback.yaml#/AndroidPlaybackResponse } diff --git a/openapi/paths/subscriptions.yaml b/openapi/paths/subscriptions.yaml new file mode 100644 index 00000000..f0dcc65c --- /dev/null +++ b/openapi/paths/subscriptions.yaml @@ -0,0 +1,56 @@ +SubscriptionFeed: + get: + tags: [user-data] + summary: Read a stable page from the current user's subscription feed snapshot + parameters: + - name: page + in: query + required: false + deprecated: true + description: Zero-based compatibility page. Cursor pagination should be used for continuation. + schema: { type: integer, minimum: 0, maximum: 10000, default: 0 } + - name: limit + in: query + required: false + schema: { type: integer, minimum: 1, maximum: 100, default: 30 } + - name: cursor + in: query + required: false + description: Opaque continuation returned in nextpage. + schema: { type: string } + responses: + '200': + description: A fresh or retained subscription feed snapshot page. + headers: + X-Request-ID: + $ref: ../components/common.yaml#/RequestIdHeader + Cache-Control: + schema: { type: string, example: no-store } + content: + application/json: + schema: + $ref: ../components/subscriptions.yaml#/SubscriptionFeedResponse + '202': + description: No retained snapshot exists and one background build is in progress. + headers: + X-Request-ID: + $ref: ../components/common.yaml#/RequestIdHeader + Retry-After: + schema: { type: integer, minimum: 1 } + content: + application/json: + schema: + $ref: ../components/subscriptions.yaml#/SubscriptionFeedPreparingResponse + '400': + $ref: ../components/common.yaml#/JsonError + '401': + $ref: ../components/common.yaml#/JsonError + '409': + description: The cursor references a snapshot generation that is no longer retained. + headers: + X-Request-ID: + $ref: ../components/common.yaml#/RequestIdHeader + content: + application/json: + schema: + $ref: ../components/common.yaml#/ErrorResponse diff --git a/src/main/kotlin/dev/typetype/server/ApplicationRoutes.kt b/src/main/kotlin/dev/typetype/server/ApplicationRoutes.kt index b179a648..3963907e 100644 --- a/src/main/kotlin/dev/typetype/server/ApplicationRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/ApplicationRoutes.kt @@ -7,6 +7,7 @@ import dev.typetype.server.routes.adminIdentityRoutes import dev.typetype.server.routes.adminSessionRoutes import dev.typetype.server.routes.authRoutes import dev.typetype.server.routes.avatarRoutes +import dev.typetype.server.routes.androidPlaybackRoutes import dev.typetype.server.routes.bulletCommentRoutes import dev.typetype.server.routes.channelRoutes import dev.typetype.server.routes.commentRoutes @@ -80,6 +81,14 @@ internal fun Application.installApplicationRoutes( } installProxyRoutes(svc) rateLimit(PROXY_ZONE) { + androidPlaybackRoutes( + svc.androidSabrSessionStore, + svc.streamService, + svc.androidSubtitleService, + authService, + svc.accessControlService, + adminSettingsService, + ) sabrRoutes( svc.sabrSessionStore, svc.streamService, diff --git a/src/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt b/src/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt index 5d0fcc62..45183642 100644 --- a/src/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt +++ b/src/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt @@ -1,6 +1,8 @@ package dev.typetype.server import dev.typetype.server.cache.DragonflyService +import dev.typetype.server.services.AndroidSubtitleHttpClient +import dev.typetype.server.services.AndroidSubtitleService import dev.typetype.server.services.BilibiliRelatedService import dev.typetype.server.services.BilibiliTrendingService import dev.typetype.server.services.CachedChannelService @@ -51,7 +53,9 @@ import dev.typetype.server.services.YoutubeSessionStreamService import okhttp3.ConnectionPool import okhttp3.Dispatcher import okhttp3.OkHttpClient +import java.net.Proxy import java.net.ProxySelector +import java.time.Duration import java.util.concurrent.TimeUnit internal class ExtractionServiceRegistry( @@ -74,14 +78,36 @@ internal class ExtractionServiceRegistry( .followRedirects(true) .build() val sabrSessionStore = SabrSessionStore(subtitleServiceUrl, initCache = cache) + val androidSabrSessionStore = SabrSessionStore( + subtitleServiceUrl, + idleEviction = Duration.ofMinutes(6), + initCache = cache, + ) + val youtubeSubtitleService = YouTubeSubtitleService(httpClient, subtitleServiceUrl) + val androidSubtitleService = AndroidSubtitleService( + youtubeSubtitleService, + AndroidSubtitleHttpClient( + httpClient.newBuilder() + .followRedirects(false) + .callTimeout(10, TimeUnit.SECONDS) + .build(), + youtubeProxySelector?.let { + httpClient.newBuilder() + .proxy(Proxy.NO_PROXY) + .followRedirects(false) + .callTimeout(10, TimeUnit.SECONDS) + .build() + }, + ), + ) private val classicPipePipeStreamService = PipePipeStreamService( cache, - YouTubeSubtitleService(httpClient, subtitleServiceUrl), + youtubeSubtitleService, BilibiliRelatedService(), ) private val sabrPipePipeStreamService = PipePipeStreamService( cache, - YouTubeSubtitleService(httpClient, subtitleServiceUrl), + youtubeSubtitleService, BilibiliRelatedService(), sabrSessionStore::rememberExtractedInfo, ) diff --git a/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt b/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt index dcc498ef..8e7e325f 100644 --- a/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt +++ b/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt @@ -42,10 +42,6 @@ internal class ServiceRegistry( adminSettingsService: AdminSettingsService, youtubeProxySelector: ProxySelector? = null, ) { - init { - SubscriptionFeedCacheInvalidation.configure(SubscriptionFeedCacheInvalidator(cache)) - } - val publicHlsManifestTokenService = PublicHlsManifestTokenService(jwtSecret) val accountIdentityService = AccountIdentityService() val customAvatarService = CustomAvatarService() @@ -81,6 +77,8 @@ internal class ServiceRegistry( val audioOnlyMediaTokenService = AudioOnlyMediaTokenService(jwtSecret) val suggestionService = extraction.suggestionService val sabrSessionStore = extraction.sabrSessionStore + val androidSabrSessionStore = extraction.androidSabrSessionStore + val androidSubtitleService = extraction.androidSubtitleService val historyService = HistoryService() val subscriptionsService = SubscriptionsService() val subscriptionFeedService = SubscriptionFeedService(subscriptionsService, channelService, cache) @@ -90,6 +88,11 @@ internal class ServiceRegistry( SubscriptionShortsBlendService(trendingService), cache, ) + init { + SubscriptionFeedCacheInvalidation.configure( + SubscriptionFeedCacheInvalidator(cache, subscriptionFeedService), + ) + } val notificationsService = NotificationsService(subscriptionFeedService) val playlistService = PlaylistService() val videoMetadataRepairService = UserVideoMetadataRepairService(VideoMetadataResolver(streamService)) diff --git a/src/main/kotlin/dev/typetype/server/models/ExtractionResult.kt b/src/main/kotlin/dev/typetype/server/models/ExtractionResult.kt index 40e2bac4..01313a9a 100644 --- a/src/main/kotlin/dev/typetype/server/models/ExtractionResult.kt +++ b/src/main/kotlin/dev/typetype/server/models/ExtractionResult.kt @@ -2,6 +2,15 @@ package dev.typetype.server.models sealed class ExtractionResult { data class Success(val data: T) : ExtractionResult() - data class Failure(val message: String, val code: String = "error") : ExtractionResult() + data class Failure( + val message: String, + val code: String = "error", + val kind: ExtractionFailureKind = ExtractionFailureKind.Unknown, + ) : ExtractionResult() data class BadRequest(val message: String, val code: String = "error") : ExtractionResult() } + +enum class ExtractionFailureKind { + Unknown, + YoutubeSessionRejected, +} diff --git a/src/main/kotlin/dev/typetype/server/models/InstanceResponse.kt b/src/main/kotlin/dev/typetype/server/models/InstanceResponse.kt index 2ec0c25a..51097404 100644 --- a/src/main/kotlin/dev/typetype/server/models/InstanceResponse.kt +++ b/src/main/kotlin/dev/typetype/server/models/InstanceResponse.kt @@ -24,4 +24,19 @@ data class InstanceResponse( val youtubeRemoteLoginEnabled: Boolean = false, val youtubeRemoteLoginReady: Boolean = false, val youtubeRemoteLoginUnavailableReason: String? = null, + val androidPlayback: AndroidPlaybackCapability = AndroidPlaybackCapability(), +) + +@Serializable +data class AndroidPlaybackCapability( + val supported: Boolean = true, + val contractVersion: Int = 2, + val youtube: AndroidYoutubePlaybackCapability = AndroidYoutubePlaybackCapability(), +) + +@Serializable +data class AndroidYoutubePlaybackCapability( + val vod: Boolean = true, + val live: Boolean = false, + val subtitles: Boolean = true, ) diff --git a/src/main/kotlin/dev/typetype/server/models/SubscriptionFeedResponse.kt b/src/main/kotlin/dev/typetype/server/models/SubscriptionFeedResponse.kt index 5bfb8651..d81e7883 100644 --- a/src/main/kotlin/dev/typetype/server/models/SubscriptionFeedResponse.kt +++ b/src/main/kotlin/dev/typetype/server/models/SubscriptionFeedResponse.kt @@ -6,4 +6,13 @@ import kotlinx.serialization.Serializable data class SubscriptionFeedResponse( val videos: List, val nextpage: String?, + val generation: Long? = null, + val generatedAt: Long? = null, + val refreshing: Boolean = false, +) + +@Serializable +data class SubscriptionFeedPreparingResponse( + val code: String = "subscription_feed_preparing", + val retryAfterMs: Long, ) diff --git a/src/main/kotlin/dev/typetype/server/routes/AndroidPlaybackCallResponses.kt b/src/main/kotlin/dev/typetype/server/routes/AndroidPlaybackCallResponses.kt new file mode 100644 index 00000000..5115de06 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/routes/AndroidPlaybackCallResponses.kt @@ -0,0 +1,36 @@ +package dev.typetype.server.routes + +import dev.typetype.server.models.ErrorResponse +import dev.typetype.server.services.AndroidPlaybackSessionLookup +import dev.typetype.server.services.AndroidPlaybackSession +import dev.typetype.server.services.AndroidPlaybackService +import dev.typetype.server.services.SabrSessionHolder +import io.ktor.http.HttpStatusCode +import io.ktor.server.application.ApplicationCall +import io.ktor.server.response.respond + +internal suspend fun ApplicationCall.androidPlaybackHolder( + service: AndroidPlaybackService, + sessionId: String, +): SabrSessionHolder? = androidPlaybackSession(service, sessionId)?.holder + +internal suspend fun ApplicationCall.androidPlaybackSession( + service: AndroidPlaybackService, + sessionId: String, +): AndroidPlaybackSession? = when (val result = service.lookup(sessionId)) { + is AndroidPlaybackSessionLookup.Active -> result.session + AndroidPlaybackSessionLookup.Expired -> { + respondAndroidError(HttpStatusCode.Gone, "android_playback_expired", "Android playback session expired") + null + } + AndroidPlaybackSessionLookup.Unknown -> { + respondAndroidError(HttpStatusCode.NotFound, "android_playback_not_found", "Android playback session not found") + null + } +} + +internal suspend fun ApplicationCall.respondAndroidError( + status: HttpStatusCode, + code: String, + message: String, +): Unit = respond(status, ErrorResponse(message, code)) diff --git a/src/main/kotlin/dev/typetype/server/routes/AndroidPlaybackHandler.kt b/src/main/kotlin/dev/typetype/server/routes/AndroidPlaybackHandler.kt new file mode 100644 index 00000000..6b0499c8 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/routes/AndroidPlaybackHandler.kt @@ -0,0 +1,181 @@ +package dev.typetype.server.routes + +import dev.typetype.server.models.ExtractionResult +import dev.typetype.server.services.AccessControlService +import dev.typetype.server.services.AdminSettingsService +import dev.typetype.server.services.AndroidDashManifestResult +import dev.typetype.server.services.AndroidPlaybackCreateResult +import dev.typetype.server.services.AndroidPlaybackSeekResult +import dev.typetype.server.services.AndroidPlaybackService +import dev.typetype.server.services.AndroidSubtitleInventoryResult +import dev.typetype.server.services.AndroidSubtitleService +import dev.typetype.server.services.AuthService +import dev.typetype.server.services.SabrSessionStore +import dev.typetype.server.services.StreamService +import dev.typetype.server.services.runCatchingNonCancellation +import io.ktor.http.HttpStatusCode +import io.ktor.server.application.ApplicationCall +import io.ktor.server.request.receiveNullable +import io.ktor.server.response.respond +import io.ktor.server.response.respondText +import kotlinx.coroutines.async +import kotlinx.coroutines.coroutineScope + +internal class AndroidPlaybackHandler( + private val store: SabrSessionStore, + private val streamService: StreamService, + private val authService: AuthService?, + private val accessControlService: AccessControlService?, + private val adminSettingsService: AdminSettingsService?, + private val subtitleService: AndroidSubtitleService, + val service: AndroidPlaybackService = AndroidPlaybackService(store), +) { + suspend fun create(call: ApplicationCall, videoId: String) { + val access = call.accessProfileOrRespond(authService, accessControlService, adminSettingsService) ?: return + if (!validateAccess(call, videoId, access)) return + val requestResult = runCatchingNonCancellation { call.receiveNullable() } + if (requestResult.isFailure) { + return call.respondAndroidError( + HttpStatusCode.BadRequest, + "android_playback_invalid_request", + "Invalid playback request", + ) + } + val request = requestResult.getOrNull() ?: AndroidPlaybackCreateRequest() + val (prepared, subtitleInventory) = coroutineScope { + val prepared = async { store.fetchInfo(videoId, cachedFirst = true) } + val subtitles = async { subtitleService.inventory(videoId) } + prepared.await() to subtitles.await() + } + prepared + ?: return call.respondAndroidError( + HttpStatusCode.UnprocessableEntity, + "android_playback_probe_failed", + "SABR probe failed", + ) + val subtitles = (subtitleInventory as? AndroidSubtitleInventoryResult.Ready)?.tracks + ?: return call.respondAndroidError( + HttpStatusCode.ServiceUnavailable, + "android_subtitle_inventory_unavailable", + "Android subtitle inventory is temporarily unavailable", + ) + val audio = SabrFormatSelector.audio(prepared.info, request.audioItag, request.audioTrackId, requireAac = true) + ?: return call.respondAndroidError( + HttpStatusCode.UnprocessableEntity, + "android_playback_audio_unavailable", + "No compatible SABR audio for this video", + ) + val video = SabrFormatSelector.androidVideo(prepared.info, request.videoItag) + ?: return call.respondAndroidError( + HttpStatusCode.UnprocessableEntity, + "android_playback_video_unavailable", + "No compatible SABR video for this video", + ) + when (val result = service.create(videoId, access.userId ?: "guest", prepared, audio, video, subtitles)) { + is AndroidPlaybackCreateResult.Created -> call.respondSession( + result.session.holder.toAndroidPlaybackResponse(result.manifest, result.session.subtitles), + result.manifest, + ) + AndroidPlaybackCreateResult.UnsupportedLive -> call.respondAndroidError( + HttpStatusCode.UnprocessableEntity, + "android_live_playback_unsupported", + "Android live playback is not supported", + ) + } + } + + suspend fun seek(call: ApplicationCall, sessionId: String) { + val session = call.androidPlaybackSession(service, sessionId) ?: return + val holder = session.holder + val requestResult = runCatchingNonCancellation { call.receiveNullable() } + val request = requestResult.getOrNull() + if (requestResult.isFailure || request == null) { + return call.respondAndroidError( + HttpStatusCode.BadRequest, + "android_playback_invalid_request", + "Invalid seek request", + ) + } + if (request.playerTimeMs < 0L) { + return call.respondAndroidError( + HttpStatusCode.BadRequest, + "android_playback_invalid_seek", + "Invalid seek position", + ) + } + when (val result = service.seek(holder, request.generation, request.playerTimeMs)) { + is AndroidPlaybackSeekResult.Ready -> call.respondSession( + result.holder.toAndroidPlaybackResponse(result.manifest, session.subtitles), + result.manifest, + ) + AndroidPlaybackSeekResult.StaleGeneration -> call.respondAndroidError( + HttpStatusCode.Conflict, + "android_playback_stale_generation", + "Stale Android playback generation", + ) + } + } + + suspend fun manifest(call: ApplicationCall, sessionId: String) { + call.response.headers.append("Cache-Control", "no-store") + val session = call.androidPlaybackSession(service, sessionId) ?: return + val holder = session.holder + when (val result = service.manifest(holder)) { + is AndroidDashManifestResult.Ready -> { + call.respondText(result.manifest, DASH_CONTENT_TYPE) + } + AndroidDashManifestResult.Preparing -> call.respond( + HttpStatusCode.Accepted, + holder.toAndroidPlaybackResponse(result, session.subtitles), + ) + AndroidDashManifestResult.UnsupportedLive -> call.respondAndroidError( + HttpStatusCode.UnprocessableEntity, + "android_live_playback_unsupported", + "Android live playback is not supported", + ) + is AndroidDashManifestResult.Invalid -> call.respondAndroidError( + HttpStatusCode.UnprocessableEntity, + "android_playback_invalid_index", + result.reason, + ) + } + } + + private suspend fun ApplicationCall.respondSession( + response: AndroidPlaybackResponse, + manifest: AndroidDashManifestResult, + ): Unit = when (manifest) { + is AndroidDashManifestResult.Ready -> respond(HttpStatusCode.OK, response) + AndroidDashManifestResult.Preparing -> respond(HttpStatusCode.Accepted, response) + AndroidDashManifestResult.UnsupportedLive -> respondAndroidError( + HttpStatusCode.UnprocessableEntity, + "android_live_playback_unsupported", + "Android live playback is not supported", + ) + is AndroidDashManifestResult.Invalid -> respondAndroidError( + HttpStatusCode.UnprocessableEntity, + "android_playback_invalid_index", + manifest.reason, + ) + } + + private suspend fun validateAccess(call: ApplicationCall, videoId: String, access: AccessRouteProfile): Boolean { + if (!access.profile.enabled) return true + return when (val result = streamService.getStreamInfo("https://www.youtube.com/watch?v=$videoId")) { + is ExtractionResult.Success -> if (access.profile.allowsUploader(result.data.uploaderUrl, result.data.uploaderName)) { + true + } else { + call.respondAndroidError(HttpStatusCode.Forbidden, "channel_not_allowed", "Channel is not allowed") + false + } + is ExtractionResult.Failure -> { + call.respondAndroidError(HttpStatusCode.UnprocessableEntity, "android_playback_extraction_failed", result.message) + false + } + is ExtractionResult.BadRequest -> { + call.respondAndroidError(HttpStatusCode.BadRequest, "android_playback_invalid_video", result.message) + false + } + } + } +} diff --git a/src/main/kotlin/dev/typetype/server/routes/AndroidPlaybackMediaHandler.kt b/src/main/kotlin/dev/typetype/server/routes/AndroidPlaybackMediaHandler.kt new file mode 100644 index 00000000..cf212364 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/routes/AndroidPlaybackMediaHandler.kt @@ -0,0 +1,71 @@ +package dev.typetype.server.routes + +import dev.typetype.server.services.AndroidPlaybackMediaResult +import dev.typetype.server.services.AndroidPlaybackService +import io.ktor.http.HttpStatusCode +import io.ktor.server.application.ApplicationCall +import io.ktor.server.response.respond + +internal class AndroidPlaybackMediaHandler(private val service: AndroidPlaybackService) { + suspend fun initialization(call: ApplicationCall, sessionId: String, itag: Int) { + val session = call.androidPlaybackSession(service, sessionId) ?: return + val generation = call.validGeneration(sessionId) ?: return + call.respondMediaResult(service.initialization(session.holder, itag, generation), session) + } + + suspend fun segment(call: ApplicationCall, sessionId: String, itag: Int, sequence: Int) { + val session = call.androidPlaybackSession(service, sessionId) ?: return + val generation = call.validGeneration(sessionId) ?: return + call.respondMediaResult(service.segment(session.holder, itag, sequence, generation), session) + } + + private suspend fun ApplicationCall.validGeneration(sessionId: String): Long? { + if (request.queryParameters["session"] != sessionId) { + respondAndroidError( + HttpStatusCode.BadRequest, + "android_playback_invalid_session", + "Manifest session does not match the request path", + ) + return null + } + val generation = request.queryParameters["generation"]?.toLongOrNull() + if (generation == null || generation < 0L) { + respondAndroidError( + HttpStatusCode.BadRequest, + "android_playback_invalid_generation", + "Missing or invalid Android playback generation", + ) + return null + } + return generation + } + + private suspend fun ApplicationCall.respondMediaResult( + result: AndroidPlaybackMediaResult, + session: dev.typetype.server.services.AndroidPlaybackSession, + ): Unit = when (result) { + is AndroidPlaybackMediaResult.Ready -> respondSabrMediaBytes(result.mimeType, result.bytes) + AndroidPlaybackMediaResult.Preparing -> respond( + HttpStatusCode.Accepted, + session.holder.toAndroidPlaybackResponse( + dev.typetype.server.services.AndroidDashManifestResult.Preparing, + session.subtitles, + ), + ) + AndroidPlaybackMediaResult.StaleGeneration -> respondAndroidError( + HttpStatusCode.Conflict, + "android_playback_stale_generation", + "Stale Android playback generation", + ) + AndroidPlaybackMediaResult.TrackNotFound -> respondAndroidError( + HttpStatusCode.NotFound, + "android_playback_track_not_found", + "Android playback track not found", + ) + AndroidPlaybackMediaResult.InvalidSequence -> respondAndroidError( + HttpStatusCode.BadRequest, + "android_playback_invalid_sequence", + "Invalid Android playback segment sequence", + ) + } +} diff --git a/src/main/kotlin/dev/typetype/server/routes/AndroidPlaybackModels.kt b/src/main/kotlin/dev/typetype/server/routes/AndroidPlaybackModels.kt new file mode 100644 index 00000000..19a97860 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/routes/AndroidPlaybackModels.kt @@ -0,0 +1,72 @@ +package dev.typetype.server.routes + +import dev.typetype.server.services.AndroidDashManifestResult +import dev.typetype.server.services.AndroidSubtitleTrack +import dev.typetype.server.services.SabrSessionHolder +import kotlinx.serialization.Serializable + +@Serializable +internal data class AndroidPlaybackCreateRequest( + val videoItag: Int? = null, + val audioItag: Int? = null, + val audioTrackId: String? = null, +) + +@Serializable +internal data class AndroidPlaybackSeekRequest( + val generation: Long, + val playerTimeMs: Long, +) + +@Serializable +internal data class AndroidPlaybackResponse( + val sessionId: String, + val videoId: String, + val manifestUrl: String, + val videoItag: Int, + val audioItag: Int, + val audioTrackId: String? = null, + val generation: Long, + val ready: Boolean, + val status: String, + val subtitles: List, + val retryAfterMs: Long? = null, +) + +@Serializable +internal data class AndroidSubtitleResponse( + val id: String, + val mimeType: String, + val languageTag: String, + val displayLanguageName: String, + val isAutoGenerated: Boolean, + val url: String, +) + +internal fun SabrSessionHolder.toAndroidPlaybackResponse( + manifest: AndroidDashManifestResult, + subtitles: List, +): AndroidPlaybackResponse = AndroidPlaybackResponse( + sessionId = sessionToken, + videoId = key.videoId, + manifestUrl = "/api/android/youtube/playback/$sessionToken/manifest.mpd", + videoItag = videoFormat.itag, + audioItag = audioFormat.itag, + audioTrackId = audioFormat.audioTrackId, + generation = activeGeneration(), + ready = manifest is AndroidDashManifestResult.Ready, + status = if (manifest is AndroidDashManifestResult.Ready) "ready" else "preparing", + subtitles = subtitles.map { track -> + AndroidSubtitleResponse( + id = track.id, + mimeType = "text/vtt", + languageTag = track.languageTag, + displayLanguageName = track.displayLanguageName, + isAutoGenerated = track.isAutoGenerated, + url = "/api/android/youtube/playback/$sessionToken/subtitles/${track.id}.vtt", + ) + }, + retryAfterMs = if (manifest is AndroidDashManifestResult.Ready) null else ANDROID_RETRY_AFTER_MS, +) + +internal const val ANDROID_RETRY_AFTER_MS = 500L diff --git a/src/main/kotlin/dev/typetype/server/routes/AndroidPlaybackRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/AndroidPlaybackRoutes.kt new file mode 100644 index 00000000..a7a8b5a2 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/routes/AndroidPlaybackRoutes.kt @@ -0,0 +1,78 @@ +package dev.typetype.server.routes + +import dev.typetype.server.models.ErrorResponse +import dev.typetype.server.services.AccessControlService +import dev.typetype.server.services.AdminSettingsService +import dev.typetype.server.services.AndroidSubtitleService +import dev.typetype.server.services.AuthService +import dev.typetype.server.services.SabrSessionStore +import dev.typetype.server.services.StreamService +import io.ktor.http.HttpStatusCode +import io.ktor.server.response.respond +import io.ktor.server.routing.Route +import io.ktor.server.routing.get +import io.ktor.server.routing.post + +internal fun Route.androidPlaybackRoutes( + store: SabrSessionStore, + streamService: StreamService, + subtitleService: AndroidSubtitleService, + authService: AuthService?, + accessControlService: AccessControlService?, + adminSettingsService: AdminSettingsService?, +) { + val handler = AndroidPlaybackHandler( + store, + streamService, + authService, + accessControlService, + adminSettingsService, + subtitleService = subtitleService, + ) + val media = AndroidPlaybackMediaHandler(handler.service) + val subtitles = AndroidSubtitleHandler( + handler.service, + subtitleService, + authService, + accessControlService, + adminSettingsService, + ) + post("/android/youtube/playback/{videoId}") { + val videoId = call.parameters["videoId"] + ?: return@post call.respond(HttpStatusCode.BadRequest, ErrorResponse("Missing videoId")) + handler.create(call, videoId) + } + post("/android/youtube/playback/{sessionId}/seek") { + val sessionId = call.parameters["sessionId"] + ?: return@post call.respond(HttpStatusCode.BadRequest, ErrorResponse("Missing sessionId")) + handler.seek(call, sessionId) + } + get("/android/youtube/playback/{sessionId}/manifest.mpd") { + val sessionId = call.parameters["sessionId"] + ?: return@get call.respond(HttpStatusCode.BadRequest, ErrorResponse("Missing sessionId")) + handler.manifest(call, sessionId) + } + get("/android/youtube/playback/{sessionId}/subtitles/{trackId}.vtt") { + val sessionId = call.parameters["sessionId"] + ?: return@get call.respond(HttpStatusCode.BadRequest, ErrorResponse("Missing sessionId")) + val trackId = call.parameters["trackId"] + ?: return@get call.respond(HttpStatusCode.BadRequest, ErrorResponse("Missing trackId")) + subtitles.content(call, sessionId, trackId) + } + get("/android/youtube/playback/{sessionId}/{itag}/init") { + val sessionId = call.parameters["sessionId"] + ?: return@get call.respond(HttpStatusCode.BadRequest, ErrorResponse("Missing sessionId")) + val itag = call.parameters["itag"]?.toIntOrNull() + ?: return@get call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid itag")) + media.initialization(call, sessionId, itag) + } + get("/android/youtube/playback/{sessionId}/{itag}/segment/{sequence}") { + val sessionId = call.parameters["sessionId"] + ?: return@get call.respond(HttpStatusCode.BadRequest, ErrorResponse("Missing sessionId")) + val itag = call.parameters["itag"]?.toIntOrNull() + ?: return@get call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid itag")) + val sequence = call.parameters["sequence"]?.toIntOrNull() + ?: return@get call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid sequence")) + media.segment(call, sessionId, itag, sequence) + } +} diff --git a/src/main/kotlin/dev/typetype/server/routes/AndroidSubtitleHandler.kt b/src/main/kotlin/dev/typetype/server/routes/AndroidSubtitleHandler.kt new file mode 100644 index 00000000..f40f5703 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/routes/AndroidSubtitleHandler.kt @@ -0,0 +1,56 @@ +package dev.typetype.server.routes + +import dev.typetype.server.services.AccessControlService +import dev.typetype.server.services.AdminSettingsService +import dev.typetype.server.services.AndroidPlaybackService +import dev.typetype.server.services.AndroidSubtitleContentResult +import dev.typetype.server.services.AndroidSubtitleService +import dev.typetype.server.services.AuthService +import io.ktor.http.ContentType +import io.ktor.http.HttpStatusCode +import io.ktor.server.application.ApplicationCall +import io.ktor.server.response.respondBytes + +internal class AndroidSubtitleHandler( + private val playbackService: AndroidPlaybackService, + private val subtitleService: AndroidSubtitleService, + private val authService: AuthService?, + private val accessControlService: AccessControlService?, + private val adminSettingsService: AdminSettingsService?, +) { + suspend fun content(call: ApplicationCall, sessionId: String, trackId: String) { + call.response.headers.append("Cache-Control", "no-store") + val access = call.accessProfileOrRespond(authService, accessControlService, adminSettingsService) ?: return + val session = call.androidPlaybackSession(playbackService, sessionId) ?: return + if (session.holder.key.userId != (access.userId ?: "guest")) { + return call.notFound() + } + val track = session.subtitles.firstOrNull { it.id == trackId } + ?: return call.notFound() + when (val result = subtitleService.content(session.holder.key.videoId, track)) { + is AndroidSubtitleContentResult.Ready -> call.respondBytes( + result.bytes, + ContentType.parse("text/vtt; charset=utf-8"), + HttpStatusCode.OK, + ) + AndroidSubtitleContentResult.TemporaryFailure -> call.respondAndroidError( + HttpStatusCode.ServiceUnavailable, + "android_subtitle_upstream_unavailable", + "Android subtitle is temporarily unavailable", + ) + AndroidSubtitleContentResult.Unavailable -> call.respondAndroidError( + HttpStatusCode.UnprocessableEntity, + "android_subtitle_unavailable", + "Android subtitle cannot be produced", + ) + } + } + + private suspend fun ApplicationCall.notFound() { + respondAndroidError( + HttpStatusCode.NotFound, + "android_subtitle_not_found", + "Android subtitle track not found", + ) + } +} diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrFormatSelector.kt b/src/main/kotlin/dev/typetype/server/routes/SabrFormatSelector.kt index 1e8e5cdc..47b9c700 100644 --- a/src/main/kotlin/dev/typetype/server/routes/SabrFormatSelector.kt +++ b/src/main/kotlin/dev/typetype/server/routes/SabrFormatSelector.kt @@ -14,6 +14,13 @@ internal object SabrFormatSelector { info.formats.filter { it.isSupportedVideo() } .minWithOrNull(compareBy { it.height }.thenBy { it.bitrate }) + fun androidVideo(info: YoutubeSabrInfo, itag: Int?): YoutubeSabrFormat? { + if (itag != null) return video(info, itag) + val supported = info.formats.filter { it.isSupportedVideo() } + return supported.filter { it.videoCodec().contains("avc1") }.maxByOrNull { it.height } + ?: supported.maxByOrNull { it.height } + } + fun audio(info: YoutubeSabrInfo, itag: Int?, trackId: String?, requireAac: Boolean): YoutubeSabrFormat? { if (itag != null) return info.formats.filter { it.matchesAudio(itag, trackId, requireAac) } .maxWithOrNull(audioComparator) diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackHandler.kt b/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackHandler.kt index 3e8a7c58..c4abe087 100644 --- a/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackHandler.kt +++ b/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackHandler.kt @@ -54,6 +54,10 @@ internal class SabrPlaybackHandler( ?: return call.respond(HttpStatusCode.NotFound, ErrorResponse("No active SABR playback session")) val request = call.playbackRequest() val playerTimeMs = request.effectiveStartTimeMs() + if (request.keepsCurrentFormats(holder)) { + val preparation = playbackService.seekExisting(holder, playerTimeMs, request.audioOnly) + return respondPrepared(call, holder, holder.key.videoId, preparation.startTimeMs, preparation.ready) + } val prepared = sabrSessionStore.fetchInfo(holder.key.videoId, playerTimeMs, cachedFirst = true) ?: return call.respond(HttpStatusCode.UnprocessableEntity, ErrorResponse("SABR probe failed")) val audio = SabrFormatSelector.audio( @@ -152,6 +156,11 @@ internal class SabrPlaybackHandler( private fun SabrPlaybackRequest.effectiveStartTimeMs(): Long = (playerTimeMs ?: startTimeMs ?: 0L).coerceAtLeast(0L) + private fun SabrPlaybackRequest.keepsCurrentFormats(holder: SabrSessionHolder): Boolean = + (videoItag == null || videoItag == holder.videoFormat.itag) && + (audioItag == null || audioItag == holder.audioFormat.itag) && + (audioTrackId.isNullOrBlank() || audioTrackId == holder.audioFormat.audioTrackId) + private suspend fun selectAudio( call: ApplicationCall, prepared: SabrPreparedInfo, diff --git a/src/main/kotlin/dev/typetype/server/routes/StreamDeliveryMode.kt b/src/main/kotlin/dev/typetype/server/routes/StreamDeliveryMode.kt index 8f4793de..1ab37c08 100644 --- a/src/main/kotlin/dev/typetype/server/routes/StreamDeliveryMode.kt +++ b/src/main/kotlin/dev/typetype/server/routes/StreamDeliveryMode.kt @@ -23,7 +23,8 @@ internal enum class StreamDeliveryMode { GenericLegacy -> runCatching { URI(url).host?.lowercase() }.getOrNull()?.let { it == "youtu.be" || it.endsWith(".youtube.com") || it == "youtube.com" } == true - YoutubeLegacy, YoutubeSabr -> true + YoutubeLegacy -> true + YoutubeSabr -> false NicoNico, BiliBili -> false } diff --git a/src/main/kotlin/dev/typetype/server/routes/SubscriptionFeedRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/SubscriptionFeedRoutes.kt index a07e4d8e..110d257c 100644 --- a/src/main/kotlin/dev/typetype/server/routes/SubscriptionFeedRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/routes/SubscriptionFeedRoutes.kt @@ -1,7 +1,12 @@ package dev.typetype.server.routes +import dev.typetype.server.models.ErrorResponse +import dev.typetype.server.models.SubscriptionFeedPreparingResponse import dev.typetype.server.services.AuthService +import dev.typetype.server.services.SubscriptionFeedPageResult import dev.typetype.server.services.SubscriptionFeedService +import io.ktor.http.HttpHeaders +import io.ktor.http.HttpStatusCode import io.ktor.server.response.respond import io.ktor.server.routing.Route import io.ktor.server.routing.get @@ -13,7 +18,26 @@ fun Route.subscriptionFeedRoutes(feedService: SubscriptionFeedService, authServi call.withJwtAuth(authService) { userId -> val page = call.request.queryParameters["page"]?.toIntOrNull()?.coerceIn(0, MAX_FEED_PAGE) ?: 0 val limit = call.request.queryParameters["limit"]?.toIntOrNull()?.coerceIn(1, 100) ?: 30 - call.respond(feedService.getFeed(userId, page, limit)) + val cursor = call.request.queryParameters["cursor"] + call.response.headers.append(HttpHeaders.CacheControl, "no-store") + when (val result = feedService.getPage(userId, page, limit, cursor)) { + is SubscriptionFeedPageResult.Ready -> call.respond(result.response) + is SubscriptionFeedPageResult.Preparing -> { + call.response.headers.append(HttpHeaders.RetryAfter, "1") + call.respond( + HttpStatusCode.Accepted, + SubscriptionFeedPreparingResponse(retryAfterMs = result.retryAfterMs), + ) + } + SubscriptionFeedPageResult.InvalidCursor -> call.respond( + HttpStatusCode.BadRequest, + ErrorResponse("Invalid subscription feed cursor", "subscription_feed_invalid_cursor"), + ) + SubscriptionFeedPageResult.StaleGeneration -> call.respond( + HttpStatusCode.Conflict, + ErrorResponse("Subscription feed generation is no longer available", "subscription_feed_stale_generation"), + ) + } } } } diff --git a/src/main/kotlin/dev/typetype/server/services/AndroidDashManifestBuilder.kt b/src/main/kotlin/dev/typetype/server/services/AndroidDashManifestBuilder.kt new file mode 100644 index 00000000..f366385c --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/AndroidDashManifestBuilder.kt @@ -0,0 +1,112 @@ +package dev.typetype.server.services + +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat +import java.io.StringWriter +import javax.xml.stream.XMLOutputFactory +import javax.xml.stream.XMLStreamWriter + +internal object AndroidDashManifestBuilder { + fun build( + sessionId: String, + generation: Long, + audio: AndroidDashTrack, + video: AndroidDashTrack, + durationMs: Long, + ): String { + val output = StringWriter() + val xml = XMLOutputFactory.newFactory().createXMLStreamWriter(output) + xml.writeStartDocument("UTF-8", "1.0") + xml.writeStartElement("MPD") + xml.writeDefaultNamespace(DASH_NAMESPACE) + xml.writeAttribute("profiles", "urn:mpeg:dash:profile:full:2011") + xml.writeAttribute("type", "static") + xml.writeAttribute("mediaPresentationDuration", duration(durationMs)) + xml.writeAttribute("minBufferTime", "PT2S") + xml.writeStartElement("Period") + xml.writeAttribute("id", "0") + xml.writeAttribute("start", "PT0S") + xml.writeTrack(sessionId, generation, video) + xml.writeTrack(sessionId, generation, audio) + xml.writeEndElement() + xml.writeEndElement() + xml.writeEndDocument() + xml.close() + return output.toString() + } + + private fun XMLStreamWriter.writeTrack( + sessionId: String, + generation: Long, + track: AndroidDashTrack, + ) { + val format = track.format + val (mimeType, codecs) = splitMime(format.mimeType.orEmpty()) + writeStartElement("AdaptationSet") + writeAttribute("id", format.itag.toString()) + writeAttribute("contentType", if (format.isAudio) "audio" else "video") + writeAttribute("mimeType", mimeType) + writeAttribute("segmentAlignment", "true") + writeAttribute("startWithSAP", "1") + writeStartElement("Representation") + writeAttribute("id", format.itag.toString()) + writeAttribute("bandwidth", format.bitrate.coerceAtLeast(1).toString()) + if (codecs.isNotBlank()) writeAttribute("codecs", codecs) + if (format.isVideo) { + writeAttribute("width", format.width.coerceAtLeast(1).toString()) + writeAttribute("height", format.height.coerceAtLeast(1).toString()) + } + writeSegmentTemplate(sessionId, generation, track) + writeEndElement() + writeEndElement() + } + + private fun XMLStreamWriter.writeSegmentTemplate( + sessionId: String, + generation: Long, + track: AndroidDashTrack, + ) { + val format = track.format + val base = "/api/android/youtube/playback/$sessionId/${format.itag}" + val query = "?session=$sessionId&generation=$generation" + writeStartElement("SegmentTemplate") + writeAttribute("timescale", "1000") + writeAttribute("startNumber", track.timeline.startNumber.toString()) + writeAttribute("initialization", "$base/init$query") + writeAttribute("media", "$base/segment/\$Number\$$query") + writeStartElement("SegmentTimeline") + compressed(track.timeline.segments).forEach { segment -> + writeEmptyElement("S") + writeAttribute("t", segment.startMs.toString()) + writeAttribute("d", segment.durationMs.toString()) + if (segment.repeat > 0) writeAttribute("r", segment.repeat.toString()) + } + writeEndElement() + writeEndElement() + } + + private fun compressed(segments: List): List { + val result = ArrayList() + for (segment in segments) { + val previous = result.lastOrNull() + if (previous != null && previous.durationMs == segment.durationMs && previous.endMs == segment.startMs) { + previous.repeat++ + } else { + result += CompressedSegment(segment.startMs, segment.durationMs) + } + } + return result + } + + private fun duration(ms: Long): String = "PT${ms / 1_000}.${(ms % 1_000).toString().padStart(3, '0')}S" + + private class CompressedSegment(val startMs: Long, val durationMs: Long, var repeat: Int = 0) { + val endMs: Long get() = startMs + durationMs * (repeat + 1L) + } + + private const val DASH_NAMESPACE = "urn:mpeg:dash:schema:mpd:2011" +} + +internal data class AndroidDashTrack( + val format: YoutubeSabrFormat, + val timeline: AndroidDashTimeline, +) diff --git a/src/main/kotlin/dev/typetype/server/services/AndroidDashManifestService.kt b/src/main/kotlin/dev/typetype/server/services/AndroidDashManifestService.kt new file mode 100644 index 00000000..86e76799 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/AndroidDashManifestService.kt @@ -0,0 +1,46 @@ +package dev.typetype.server.services + +internal class AndroidDashManifestService { + fun build(holder: SabrSessionHolder): AndroidDashManifestResult { + if (holder.expectsLive()) return AndroidDashManifestResult.UnsupportedLive + val state = holder.session.streamState + val audio = AndroidDashTimelineReader.read(state, holder.audioFormat) + val video = AndroidDashTimelineReader.read(state, holder.videoFormat) + if (audio is AndroidDashTimelineResult.Pending || video is AndroidDashTimelineResult.Pending) { + return AndroidDashManifestResult.Preparing + } + if (audio is AndroidDashTimelineResult.Invalid) return AndroidDashManifestResult.Invalid(audio.reason) + if (video is AndroidDashTimelineResult.Invalid) return AndroidDashManifestResult.Invalid(video.reason) + audio as AndroidDashTimelineResult.Ready + video as AndroidDashTimelineResult.Ready + val durationMs = maxOf(audio.timeline.endMs, video.timeline.endMs) + if (durationMs <= 0L || durationMs - audio.timeline.endMs > COVERAGE_TOLERANCE_MS || + durationMs - video.timeline.endMs > COVERAGE_TOLERANCE_MS + ) { + return AndroidDashManifestResult.Invalid("Audio and video timelines do not cover the same presentation") + } + val manifest = AndroidDashManifestBuilder.build( + sessionId = holder.sessionToken, + generation = holder.activeGeneration(), + audio = AndroidDashTrack(holder.audioFormat, audio.timeline), + video = AndroidDashTrack(holder.videoFormat, video.timeline), + durationMs = durationMs, + ) + if (manifest.toByteArray(Charsets.UTF_8).size > MAX_MANIFEST_BYTES) { + return AndroidDashManifestResult.Invalid("DASH manifest exceeds the size limit") + } + return AndroidDashManifestResult.Ready(manifest, durationMs) + } + + private companion object { + const val COVERAGE_TOLERANCE_MS = 1_500L + const val MAX_MANIFEST_BYTES = 2 * 1024 * 1024 + } +} + +internal sealed interface AndroidDashManifestResult { + data class Ready(val manifest: String, val durationMs: Long) : AndroidDashManifestResult + data object Preparing : AndroidDashManifestResult + data object UnsupportedLive : AndroidDashManifestResult + data class Invalid(val reason: String) : AndroidDashManifestResult +} diff --git a/src/main/kotlin/dev/typetype/server/services/AndroidDashTimeline.kt b/src/main/kotlin/dev/typetype/server/services/AndroidDashTimeline.kt new file mode 100644 index 00000000..5e8118ab --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/AndroidDashTimeline.kt @@ -0,0 +1,55 @@ +package dev.typetype.server.services + +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrStreamState + +internal data class AndroidDashTimeline( + val startNumber: Int, + val segments: List, +) { + val endMs: Long get() = segments.last().endMs +} + +internal data class AndroidDashTimelineSegment( + val startMs: Long, + val durationMs: Long, +) { + val endMs: Long get() = startMs + durationMs +} + +internal sealed interface AndroidDashTimelineResult { + data class Ready(val timeline: AndroidDashTimeline) : AndroidDashTimelineResult + data object Pending : AndroidDashTimelineResult + data class Invalid(val reason: String) : AndroidDashTimelineResult +} + +internal object AndroidDashTimelineReader { + fun read( + state: YoutubeSabrStreamState, + format: YoutubeSabrFormat, + ): AndroidDashTimelineResult { + if (!state.hasSegmentIndex(format)) return AndroidDashTimelineResult.Pending + val endSegment = state.getEndSegment(format) + if (endSegment !in 1..MAX_SEGMENTS) { + return AndroidDashTimelineResult.Invalid("Invalid exact segment count for itag ${format.itag}") + } + val segments = ArrayList(endSegment.toInt()) + for (sequence in 1..endSegment.toInt()) { + val startMs = state.getSegmentStartMs(format, sequence) + val endMs = state.getSegmentEndMs(format, sequence) + if (startMs < 0L || endMs <= startMs) { + return AndroidDashTimelineResult.Invalid("Invalid exact timeline for itag ${format.itag}") + } + if (sequence == 1 && startMs != 0L) { + return AndroidDashTimelineResult.Invalid("Timeline does not begin at zero for itag ${format.itag}") + } + if (segments.lastOrNull()?.startMs?.let { startMs <= it } == true) { + return AndroidDashTimelineResult.Invalid("Timeline is not monotonic for itag ${format.itag}") + } + segments += AndroidDashTimelineSegment(startMs, endMs - startMs) + } + return AndroidDashTimelineResult.Ready(AndroidDashTimeline(startNumber = 1, segments)) + } + + private const val MAX_SEGMENTS = 100_000L +} diff --git a/src/main/kotlin/dev/typetype/server/services/AndroidPlaybackService.kt b/src/main/kotlin/dev/typetype/server/services/AndroidPlaybackService.kt new file mode 100644 index 00000000..12a9462e --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/AndroidPlaybackService.kt @@ -0,0 +1,117 @@ +package dev.typetype.server.services + +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat + +internal class AndroidPlaybackService( + private val store: SabrSessionStore, + private val sessions: AndroidPlaybackSessionRegistry = AndroidPlaybackSessionRegistry(store), +) { + private val playback = SabrPlaybackSessionService(store, SabrSessionPurpose.ANDROID_PLAYBACK) + private val manifests = AndroidDashManifestService() + + suspend fun create( + videoId: String, + userId: String, + prepared: SabrPreparedInfo, + audio: YoutubeSabrFormat, + video: YoutubeSabrFormat, + subtitles: List, + ): AndroidPlaybackCreateResult { + if (prepared.isLive || prepared.isLiveContent) return AndroidPlaybackCreateResult.UnsupportedLive + val result = playback.prepare( + videoId = videoId, + userId = userId, + prepared = prepared, + audio = audio, + video = video, + startTimeMs = 0L, + ) + val session = sessions.register(result.holder, subtitles) + return AndroidPlaybackCreateResult.Created(session, manifests.build(result.holder)) + } + + fun lookup(sessionId: String): AndroidPlaybackSessionLookup = sessions.lookup(sessionId) + + fun seek( + holder: SabrSessionHolder, + generation: Long, + playerTimeMs: Long, + ): AndroidPlaybackSeekResult { + if (generation != holder.activeGeneration()) return AndroidPlaybackSeekResult.StaleGeneration + playback.seekExisting(holder, playerTimeMs.coerceAtLeast(0L)) + return AndroidPlaybackSeekResult.Ready(holder, manifests.build(holder)) + } + + suspend fun manifest(holder: SabrSessionHolder): AndroidDashManifestResult { + val initial = manifests.build(holder) + if (initial !is AndroidDashManifestResult.Preparing) return initial + SabrPlaybackInitializationPreloader.preload(store, holder, MANIFEST_PRELOAD_TIMEOUT_MS) + return manifests.build(holder) + } + + suspend fun initialization( + holder: SabrSessionHolder, + itag: Int, + generation: Long, + ): AndroidPlaybackMediaResult { + val format = holder.formatForItag(itag) ?: return AndroidPlaybackMediaResult.TrackNotFound + if (generation != holder.activeGeneration()) return AndroidPlaybackMediaResult.StaleGeneration + return playback.fetchInitialization(holder, format, MEDIA_TIMEOUT_MS, generation).toAndroidResult() + } + + suspend fun segment( + holder: SabrSessionHolder, + itag: Int, + sequence: Int, + generation: Long, + ): AndroidPlaybackMediaResult { + val format = holder.formatForItag(itag) ?: return AndroidPlaybackMediaResult.TrackNotFound + if (generation != holder.activeGeneration()) return AndroidPlaybackMediaResult.StaleGeneration + return playback.fetchMedia(holder, format, sequence, MEDIA_TIMEOUT_MS, generation).toAndroidResult() + } + + private fun SabrSessionHolder.formatForItag(itag: Int): YoutubeSabrFormat? = when (itag) { + audioFormat.itag -> audioFormat + videoFormat.itag -> videoFormat + else -> null + } + + private fun SabrPlaybackSegmentResult.toAndroidResult(): AndroidPlaybackMediaResult = when (this) { + is SabrPlaybackSegmentResult.Ready -> AndroidPlaybackMediaResult.Ready(mimeType, bytes) + is SabrPlaybackSegmentResult.Retry -> AndroidPlaybackMediaResult.Preparing + is SabrPlaybackSegmentResult.Stale -> AndroidPlaybackMediaResult.StaleGeneration + SabrPlaybackSegmentResult.InvalidGeneration -> AndroidPlaybackMediaResult.StaleGeneration + SabrPlaybackSegmentResult.InvalidSequence -> AndroidPlaybackMediaResult.InvalidSequence + } + + private companion object { + const val MANIFEST_PRELOAD_TIMEOUT_MS = 2_000L + const val MEDIA_TIMEOUT_MS = 4_000L + } +} + +internal sealed interface AndroidPlaybackCreateResult { + data class Created( + val session: AndroidPlaybackSession, + val manifest: AndroidDashManifestResult, + ) : AndroidPlaybackCreateResult + + data object UnsupportedLive : AndroidPlaybackCreateResult +} + +internal sealed interface AndroidPlaybackSeekResult { + data class Ready( + val holder: SabrSessionHolder, + val manifest: AndroidDashManifestResult, + ) : AndroidPlaybackSeekResult + + data object StaleGeneration : AndroidPlaybackSeekResult +} + +internal sealed interface AndroidPlaybackMediaResult { + data class Ready(val mimeType: String, val bytes: ByteArray) : AndroidPlaybackMediaResult + data object Preparing : AndroidPlaybackMediaResult + data object StaleGeneration : AndroidPlaybackMediaResult + data object TrackNotFound : AndroidPlaybackMediaResult + data object InvalidSequence : AndroidPlaybackMediaResult +} diff --git a/src/main/kotlin/dev/typetype/server/services/AndroidPlaybackSessionRegistry.kt b/src/main/kotlin/dev/typetype/server/services/AndroidPlaybackSessionRegistry.kt new file mode 100644 index 00000000..ecea6306 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/AndroidPlaybackSessionRegistry.kt @@ -0,0 +1,89 @@ +package dev.typetype.server.services + +import java.time.Duration +import java.time.Instant +import java.util.concurrent.ConcurrentHashMap + +internal class AndroidPlaybackSessionRegistry( + private val store: SabrSessionStore, + private val idleTimeout: Duration = Duration.ofMinutes(4), + private val tombstoneTtl: Duration = Duration.ofMinutes(10), + private val now: () -> Instant = Instant::now, +) { + private val sessions = ConcurrentHashMap() + private val tombstones = ConcurrentHashMap() + + fun register( + holder: SabrSessionHolder, + subtitles: List, + ): AndroidPlaybackSession { + val current = now() + cleanup(current) + val session = AndroidPlaybackSession(holder, subtitles) + sessions[holder.sessionToken] = Lease(session, current) + tombstones.remove(holder.sessionToken) + return session + } + + fun lookup(sessionId: String): AndroidPlaybackSessionLookup { + val current = now() + cleanup(current) + val lease = sessions[sessionId] + ?: return if (tombstones.containsKey(sessionId)) { + AndroidPlaybackSessionLookup.Expired + } else { + AndroidPlaybackSessionLookup.Unknown + } + if (!lease.lastAccess.plus(idleTimeout).isAfter(current)) { + expire(sessionId, lease, current) + return AndroidPlaybackSessionLookup.Expired + } + val holder = store.lookupByToken(sessionId) + if (holder == null || holder.key.purpose != SabrSessionPurpose.ANDROID_PLAYBACK) { + expire(sessionId, lease, current) + return AndroidPlaybackSessionLookup.Expired + } + lease.lastAccess = current + return AndroidPlaybackSessionLookup.Active(AndroidPlaybackSession(holder, lease.session.subtitles)) + } + + internal fun expire(sessionId: String): Unit { + val current = now() + sessions[sessionId]?.let { expire(sessionId, it, current) } + cleanup(current) + } + + private fun expire(sessionId: String, lease: Lease, current: Instant): Unit { + if (!sessions.remove(sessionId, lease)) return + tombstones[sessionId] = current.plus(tombstoneTtl) + store.release(lease.session.holder) + } + + private fun cleanup(current: Instant): Unit { + sessions.entries + .filter { !it.value.lastAccess.plus(idleTimeout).isAfter(current) } + .forEach { expire(it.key, it.value, current) } + tombstones.entries.removeIf { !it.value.isAfter(current) } + if (tombstones.size <= MAX_TOMBSTONES) return + tombstones.entries.sortedBy { it.value } + .take(tombstones.size - MAX_TOMBSTONES) + .forEach { tombstones.remove(it.key, it.value) } + } + + private class Lease(val session: AndroidPlaybackSession, @Volatile var lastAccess: Instant) + + private companion object { + const val MAX_TOMBSTONES = 2_048 + } +} + +internal data class AndroidPlaybackSession( + val holder: SabrSessionHolder, + val subtitles: List, +) + +internal sealed interface AndroidPlaybackSessionLookup { + data class Active(val session: AndroidPlaybackSession) : AndroidPlaybackSessionLookup + data object Expired : AndroidPlaybackSessionLookup + data object Unknown : AndroidPlaybackSessionLookup +} diff --git a/src/main/kotlin/dev/typetype/server/services/AndroidSubtitleHttpClient.kt b/src/main/kotlin/dev/typetype/server/services/AndroidSubtitleHttpClient.kt new file mode 100644 index 00000000..77feaf80 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/AndroidSubtitleHttpClient.kt @@ -0,0 +1,91 @@ +package dev.typetype.server.services + +import kotlinx.coroutines.suspendCancellableCoroutine +import okhttp3.Call +import okhttp3.Callback +import okhttp3.HttpUrl +import okhttp3.OkHttpClient +import okhttp3.Request +import okhttp3.Response +import java.io.IOException +import java.nio.ByteBuffer +import java.nio.charset.CodingErrorAction +import java.nio.charset.StandardCharsets +import kotlin.coroutines.resume + +internal class AndroidSubtitleHttpClient( + private val primaryClient: OkHttpClient, + private val directClient: OkHttpClient?, +) { + suspend fun fetch(sourceUrl: HttpUrl): AndroidSubtitleUpstreamResult { + val url = sourceUrl.newBuilder() + .removeAllQueryParameters("fmt") + .addQueryParameter("fmt", "vtt") + .build() + val primary = request(primaryClient, url) + if (primary !is AndroidSubtitleUpstreamResult.TemporaryFailure) return primary + return directClient?.let { request(it, url) } ?: primary + } + + private suspend fun request(client: OkHttpClient, url: HttpUrl): AndroidSubtitleUpstreamResult = + suspendCancellableCoroutine { continuation -> + val request = Request.Builder() + .url(url) + .header("Accept", "text/vtt") + .header("Origin", YOUTUBE_ORIGIN) + .header("Referer", "$YOUTUBE_ORIGIN/") + .header("User-Agent", ANDROID_VR_USER_AGENT) + .build() + val call = client.newCall(request) + continuation.invokeOnCancellation { call.cancel() } + call.enqueue(object : Callback { + override fun onFailure(call: Call, e: IOException) { + if (continuation.isActive) { + continuation.resume(AndroidSubtitleUpstreamResult.TemporaryFailure) + } + } + + override fun onResponse(call: Call, response: Response) { + val result = runCatching { response.use(::readResponse) } + .getOrDefault(AndroidSubtitleUpstreamResult.TemporaryFailure) + if (continuation.isActive) continuation.resume(result) + } + }) + } + + private fun readResponse(response: Response): AndroidSubtitleUpstreamResult { + if (response.code == 403 || response.code == 429 || response.code >= 500) { + return AndroidSubtitleUpstreamResult.TemporaryFailure + } + if (!response.isSuccessful) return AndroidSubtitleUpstreamResult.Unavailable + val body = response.body + if (body.contentLength() > MAX_SUBTITLE_BYTES) return AndroidSubtitleUpstreamResult.Unavailable + val bytes = body.byteStream().readNBytes(MAX_SUBTITLE_BYTES + 1) + if (bytes.size > MAX_SUBTITLE_BYTES) return AndroidSubtitleUpstreamResult.Unavailable + val text = runCatching { + StandardCharsets.UTF_8.newDecoder() + .onMalformedInput(CodingErrorAction.REPORT) + .onUnmappableCharacter(CodingErrorAction.REPORT) + .decode(ByteBuffer.wrap(bytes)) + .toString() + }.getOrNull() ?: return AndroidSubtitleUpstreamResult.Unavailable + if (!text.removePrefix("\uFEFF").startsWith("WEBVTT")) { + return AndroidSubtitleUpstreamResult.Unavailable + } + return AndroidSubtitleUpstreamResult.Ready(bytes) + } + + private companion object { + const val MAX_SUBTITLE_BYTES = 16 * 1024 * 1024 + const val YOUTUBE_ORIGIN = "https://www.youtube.com" + const val ANDROID_VR_USER_AGENT = + "com.google.android.apps.youtube.vr.oculus/1.65.10 " + + "(Linux; U; Android 12L; eureka-user Build/SQ3A.220605.009.A1) gzip" + } +} + +internal sealed interface AndroidSubtitleUpstreamResult { + data class Ready(val bytes: ByteArray) : AndroidSubtitleUpstreamResult + data object TemporaryFailure : AndroidSubtitleUpstreamResult + data object Unavailable : AndroidSubtitleUpstreamResult +} diff --git a/src/main/kotlin/dev/typetype/server/services/AndroidSubtitleService.kt b/src/main/kotlin/dev/typetype/server/services/AndroidSubtitleService.kt new file mode 100644 index 00000000..1cd84cd5 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/AndroidSubtitleService.kt @@ -0,0 +1,48 @@ +package dev.typetype.server.services + +internal class AndroidSubtitleService( + private val inventorySource: YouTubeSubtitleService, + private val httpClient: AndroidSubtitleHttpClient, +) { + suspend fun inventory(videoId: String): AndroidSubtitleInventoryResult = + when (val result = inventorySource.fetchSubtitleInventory(videoId)) { + is YouTubeSubtitleInventoryResult.Ready -> { + val tracks = AndroidSubtitleTrackFactory.create(videoId, result.tracks) + ?: return AndroidSubtitleInventoryResult.Unavailable + AndroidSubtitleInventoryResult.Ready(tracks) + } + YouTubeSubtitleInventoryResult.Unavailable -> AndroidSubtitleInventoryResult.Unavailable + } + + suspend fun content( + videoId: String, + track: AndroidSubtitleTrack, + ): AndroidSubtitleContentResult { + return when (val initial = httpClient.fetch(track.sourceUrl)) { + is AndroidSubtitleUpstreamResult.Ready -> AndroidSubtitleContentResult.Ready(initial.bytes) + AndroidSubtitleUpstreamResult.Unavailable -> AndroidSubtitleContentResult.Unavailable + AndroidSubtitleUpstreamResult.TemporaryFailure -> retryWithFreshInventory(videoId, track.id) + } + } + + private suspend fun retryWithFreshInventory( + videoId: String, + trackId: String, + ): AndroidSubtitleContentResult { + val refreshed = inventory(videoId) as? AndroidSubtitleInventoryResult.Ready + ?: return AndroidSubtitleContentResult.TemporaryFailure + val track = refreshed.tracks.firstOrNull { it.id == trackId } + ?: return AndroidSubtitleContentResult.Unavailable + return when (val retry = httpClient.fetch(track.sourceUrl)) { + is AndroidSubtitleUpstreamResult.Ready -> AndroidSubtitleContentResult.Ready(retry.bytes) + AndroidSubtitleUpstreamResult.Unavailable -> AndroidSubtitleContentResult.Unavailable + AndroidSubtitleUpstreamResult.TemporaryFailure -> AndroidSubtitleContentResult.TemporaryFailure + } + } +} + +internal sealed interface AndroidSubtitleContentResult { + data class Ready(val bytes: ByteArray) : AndroidSubtitleContentResult + data object TemporaryFailure : AndroidSubtitleContentResult + data object Unavailable : AndroidSubtitleContentResult +} diff --git a/src/main/kotlin/dev/typetype/server/services/AndroidSubtitleTrack.kt b/src/main/kotlin/dev/typetype/server/services/AndroidSubtitleTrack.kt new file mode 100644 index 00000000..75415c90 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/AndroidSubtitleTrack.kt @@ -0,0 +1,62 @@ +package dev.typetype.server.services + +import dev.typetype.server.models.SubtitleItem +import okhttp3.HttpUrl +import okhttp3.HttpUrl.Companion.toHttpUrlOrNull +import java.nio.charset.StandardCharsets +import java.security.MessageDigest + +internal data class AndroidSubtitleTrack( + val id: String, + val languageTag: String, + val displayLanguageName: String, + val isAutoGenerated: Boolean, + val sourceUrl: HttpUrl, +) + +internal object AndroidSubtitleTrackFactory { + fun create(videoId: String, items: List): List? { + val occurrences = mutableMapOf() + return items.map { item -> + val sourceUrl = item.url.toHttpUrlOrNull() + ?.takeIf(::isYoutubeCaptionUrl) + ?: return null + val baseId = stableId(videoId, item, sourceUrl) + val occurrence = occurrences.merge(baseId, 1, Int::plus) ?: 1 + AndroidSubtitleTrack( + id = if (occurrence == 1) baseId else "$baseId-$occurrence", + languageTag = item.languageTag, + displayLanguageName = item.displayLanguageName, + isAutoGenerated = item.isAutoGenerated, + sourceUrl = sourceUrl, + ) + } + } + + private fun stableId(videoId: String, item: SubtitleItem, url: HttpUrl): String { + val sourceIdentity = listOf("vssId", "lang", "kind", "tlang") + .joinToString("&") { key -> "$key=${url.queryParameter(key).orEmpty()}" } + val material = listOf( + videoId, + item.languageTag, + item.isAutoGenerated.toString(), + sourceIdentity, + ).joinToString("\u0000") + return MessageDigest.getInstance("SHA-256") + .digest(material.toByteArray(StandardCharsets.UTF_8)) + .take(ID_BYTES) + .joinToString("") { "%02x".format(it) } + } + + private fun isYoutubeCaptionUrl(url: HttpUrl): Boolean = + url.isHttps && + (url.host == "youtube.com" || url.host.endsWith(".youtube.com")) && + url.encodedPath == "/api/timedtext" + + private const val ID_BYTES = 12 +} + +internal sealed interface AndroidSubtitleInventoryResult { + data class Ready(val tracks: List) : AndroidSubtitleInventoryResult + data object Unavailable : AndroidSubtitleInventoryResult +} diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationWarmupService.kt b/src/main/kotlin/dev/typetype/server/services/HomeRecommendationWarmupService.kt index 44ddc948..59d9365a 100644 --- a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationWarmupService.kt +++ b/src/main/kotlin/dev/typetype/server/services/HomeRecommendationWarmupService.kt @@ -31,6 +31,7 @@ class HomeRecommendationWarmupService( activeUsers[userId] = System.currentTimeMillis() scope.launch { invalidate(userId) + SubscriptionFeedCacheInvalidation.awaitRefresh(userId) schedule(userId, force = true) } } @@ -54,8 +55,7 @@ class HomeRecommendationWarmupService( } private suspend fun invalidate(userId: String) { - cache.delete(SubscriptionFeedCacheKeys.feed(userId)) - cache.delete(SubscriptionFeedCacheKeys.shorts(userId)) + SubscriptionFeedCacheInvalidation.invalidate(userId) poolCache.delete(userId, YOUTUBE_SERVICE_ID, HomeRecommendationPoolMode.FULL) poolCache.delete(userId, YOUTUBE_SERVICE_ID, HomeRecommendationPoolMode.SHORTS) } diff --git a/src/main/kotlin/dev/typetype/server/services/NewPipeInitializer.kt b/src/main/kotlin/dev/typetype/server/services/NewPipeInitializer.kt index 1a506bab..93ab24a8 100644 --- a/src/main/kotlin/dev/typetype/server/services/NewPipeInitializer.kt +++ b/src/main/kotlin/dev/typetype/server/services/NewPipeInitializer.kt @@ -17,6 +17,9 @@ object NewPipeInitializer { val normalizedUrl = tokenServiceUrl?.trim()?.takeIf { it.isNotBlank() } if (normalizedUrl != null && normalizedUrl != decoderServiceUrl) { YoutubeApiDecoder.setLocalDecoder(TypetypeTokenYoutubeJavaScriptDecoder(normalizedUrl)) + NewPipe.setYoutubeSessionPoTokenProvider( + TypetypeTokenYoutubeSessionPoTokenProvider(normalizedUrl), + ) decoderServiceUrl = normalizedUrl } if (!initialized) { diff --git a/src/main/kotlin/dev/typetype/server/services/SabrCachedSegmentLocator.kt b/src/main/kotlin/dev/typetype/server/services/SabrCachedSegmentLocator.kt index 35871168..df527fc1 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrCachedSegmentLocator.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrCachedSegmentLocator.kt @@ -9,15 +9,22 @@ internal fun YoutubeSabrSession.findCachedMediaAt( format: YoutubeSabrFormat, targetMs: Long, predictedSequence: Int, + fallbackDurationMs: Long? = null, allowFollowing: Boolean = false, ): SabrMediaSegment? { for (distance in 1..MAX_SEQUENCE_DISTANCE) { - cachedMedia(format, predictedSequence + distance)?.takeIf { it.covers(targetMs) }?.let { return it } - cachedMedia(format, predictedSequence - distance)?.takeIf { it.covers(targetMs) }?.let { return it } + cachedMedia(format, predictedSequence + distance) + ?.takeIf { it.covers(targetMs, fallbackDurationMs) } + ?.let { return it } + cachedMedia(format, predictedSequence - distance) + ?.takeIf { it.covers(targetMs, fallbackDurationMs) } + ?.let { return it } } if (!allowFollowing) return null for (distance in 1..MAX_SEQUENCE_DISTANCE) { - cachedMedia(format, predictedSequence + distance)?.takeIf { it.startsWithinNextSegment(targetMs) }?.let { return it } + cachedMedia(format, predictedSequence + distance) + ?.takeIf { it.startsWithinNextSegment(targetMs, fallbackDurationMs) } + ?.let { return it } } return null } @@ -27,17 +34,18 @@ private fun YoutubeSabrSession.cachedMedia(format: YoutubeSabrFormat, sequence: return getCachedSegment(SabrSegmentRequest.media(format, sequence)) } -private fun SabrMediaSegment.covers(targetMs: Long): Boolean { +private fun SabrMediaSegment.covers(targetMs: Long, fallbackDurationMs: Long?): Boolean { val startMs = header.startMs + val durationMs = header.durationMs.takeIf { it > 0L } ?: fallbackDurationMs ?: return false return startMs >= 0L && - header.durationMs > 0L && targetMs >= startMs - TIMING_TOLERANCE_MS && - targetMs < startMs + header.durationMs + targetMs < startMs + durationMs } -private fun SabrMediaSegment.startsWithinNextSegment(targetMs: Long): Boolean { +private fun SabrMediaSegment.startsWithinNextSegment(targetMs: Long, fallbackDurationMs: Long?): Boolean { + val durationMs = header.durationMs.takeIf { it > 0L } ?: fallbackDurationMs ?: return false val leadMs = header.startMs - targetMs - return header.startMs >= 0L && header.durationMs > 0L && leadMs in -TIMING_TOLERANCE_MS..header.durationMs + return header.startMs >= 0L && leadMs in -TIMING_TOLERANCE_MS..durationMs } private const val MAX_SEQUENCE_DISTANCE = 24 diff --git a/src/main/kotlin/dev/typetype/server/services/SabrDemandAttemptFinisher.kt b/src/main/kotlin/dev/typetype/server/services/SabrDemandAttemptFinisher.kt index 9b8546a9..b50f7f9e 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrDemandAttemptFinisher.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrDemandAttemptFinisher.kt @@ -1,5 +1,6 @@ package dev.typetype.server.services +import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaSegment import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrSession @@ -45,13 +46,14 @@ internal object SabrDemandAttemptFinisher { result: YoutubeSabrSession.DemandResponseResult, runtime: SabrPumpRuntime, wasFutureLiveRequest: Boolean, + onResolved: (SabrMediaSegment) -> Unit = {}, ): Boolean = synchronized(holder) { if (!holder.isSegmentDemandActive(request, identity)) { runtime.finishDemand(identity) holder.setPlaybackState(SabrPlaybackState.IDLE) return@synchronized true } - val resolved = holder.resolveSegmentDemand(request, identity) + val resolved = holder.resolveSegmentDemand(request, identity, onResolved) SabrPumpLogger.finish(holder, "demand", request, result.segmentCount) if (!resolved && (wasFutureLiveRequest || holder.isFutureLiveRequest(request))) { runtime.finishDemand(identity) diff --git a/src/main/kotlin/dev/typetype/server/services/SabrDemandDeadline.kt b/src/main/kotlin/dev/typetype/server/services/SabrDemandDeadline.kt new file mode 100644 index 00000000..e0d55ae2 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/SabrDemandDeadline.kt @@ -0,0 +1,40 @@ +package dev.typetype.server.services + +internal class SabrDemandDeadline(private val timeoutMs: Long) { + private var identity: String? = null + private var registeredAtMs = Long.MIN_VALUE + private var expiresAtMs = Long.MIN_VALUE + private var coveredBackoffUntilMs = Long.MIN_VALUE + + fun isExpired( + identity: String, + registeredAtMs: Long, + nowMs: Long, + backoffRemainingMs: Long, + ): Boolean { + if (this.identity != identity || this.registeredAtMs != registeredAtMs) { + reset(identity, registeredAtMs) + } + extendForBackoff(nowMs, backoffRemainingMs) + return nowMs >= expiresAtMs + } + + private fun reset(identity: String, registeredAtMs: Long) { + this.identity = identity + this.registeredAtMs = registeredAtMs + expiresAtMs = saturatedAdd(registeredAtMs, timeoutMs) + coveredBackoffUntilMs = Long.MIN_VALUE + } + + private fun extendForBackoff(nowMs: Long, remainingMs: Long) { + if (remainingMs <= 0L) return + val backoffUntilMs = saturatedAdd(nowMs, remainingMs) + val uncoveredFromMs = maxOf(nowMs, coveredBackoffUntilMs) + val uncoveredMs = (backoffUntilMs - uncoveredFromMs).coerceAtLeast(0L) + expiresAtMs = saturatedAdd(expiresAtMs, uncoveredMs) + coveredBackoffUntilMs = maxOf(coveredBackoffUntilMs, backoffUntilMs) + } + + private fun saturatedAdd(value: Long, increment: Long): Long = + if (value >= Long.MAX_VALUE - increment) Long.MAX_VALUE else value + increment +} diff --git a/src/main/kotlin/dev/typetype/server/services/SabrDemandWatchdog.kt b/src/main/kotlin/dev/typetype/server/services/SabrDemandWatchdog.kt index dacd0352..29146c88 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrDemandWatchdog.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrDemandWatchdog.kt @@ -7,6 +7,7 @@ internal class SabrDemandWatchdog( private val intervalMs: Long = SabrPumpPolicy.IDLE_POLL_MS, ) { suspend fun monitor(isAlive: () -> Boolean, holder: SabrSessionHolder): Boolean { + val deadline = SabrDemandDeadline(SabrPumpPolicy.DEMAND_TARGET_DEADLINE_MS) while (isAlive()) { val state = holder.playbackState() if (state == SabrPlaybackState.TERMINAL || state == SabrPlaybackState.NETWORK_FAILED) return false @@ -14,18 +15,24 @@ internal class SabrDemandWatchdog( if (inFlightDemand != null) { if (inFlightDemand.futureLiveRequest) holder.setPlaybackState(SabrPlaybackState.WAITING_FOR_LIVE) val nowMs = clock() + val backoffRemainingMs = holder.session.demandBackoffRemainingMs val lastProgressAtMs = inFlightDemand.observeProgress(holder.session.mediaProgressVersion, nowMs) val completedIdle = holder.session.getCachedSegment(inFlightDemand.request) != null && nowMs - lastProgressAtMs >= SabrPumpPolicy.COMPLETED_DEMAND_IDLE_MS if (completedIdle && SabrDemandAttemptFinisher.interruptCompletedInFlightDemand(holder, inFlightDemand)) { return true } - if (nowMs - inFlightDemand.registeredAtMs >= SabrPumpPolicy.DEMAND_TARGET_DEADLINE_MS && + if (deadline.isExpired( + inFlightDemand.identity, + inFlightDemand.registeredAtMs, + nowMs, + backoffRemainingMs, + ) && SabrDemandAttemptFinisher.expireStalledInFlightDemand(holder, inFlightDemand) ) { return true } - delay(if (inFlightDemand.futureLiveRequest) maxOf(intervalMs, LIVE_EDGE_POLL_MS) else intervalMs) + delay(nextCheckDelayMs(backoffRemainingMs, inFlightDemand.futureLiveRequest)) continue } val request = holder.nextSegmentDemand() @@ -39,14 +46,23 @@ internal class SabrDemandWatchdog( } val identity = holder.segmentDemandIdentity(request) val registeredAtMs = identity?.let { holder.segmentDemandRegisteredAtMs(request, it) } - if (identity != null && registeredAtMs != null && - clock() - registeredAtMs >= SabrPumpPolicy.DEMAND_TARGET_DEADLINE_MS && + val nowMs = clock() + val backoffRemainingMs = holder.session.demandBackoffRemainingMs + if (identity != null && registeredAtMs != null && deadline.isExpired( + identity, + registeredAtMs, + nowMs, + backoffRemainingMs, + ) && SabrDemandAttemptFinisher.expireStalledDemand(holder, request, identity, futureLiveRequest) ) { return true } - delay(if (futureLiveRequest) maxOf(intervalMs, LIVE_EDGE_POLL_MS) else intervalMs) + delay(nextCheckDelayMs(backoffRemainingMs, futureLiveRequest)) } return false } + + private fun nextCheckDelayMs(backoffRemainingMs: Long, futureLiveRequest: Boolean): Long = + maxOf(intervalMs, backoffRemainingMs, LIVE_EDGE_POLL_MS.takeIf { futureLiveRequest } ?: 0L) } diff --git a/src/main/kotlin/dev/typetype/server/services/SabrLivePlayback.kt b/src/main/kotlin/dev/typetype/server/services/SabrLivePlayback.kt index ffbf4b3a..4a5f0762 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrLivePlayback.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrLivePlayback.kt @@ -78,6 +78,7 @@ private fun SabrSessionHolder.availableLiveMediaStartMs(): Long? { internal fun SabrSessionHolder.isFutureLiveRequest(request: SabrSegmentRequest): Boolean { if (request.isInitializationSegment) return false livePlaybackSnapshot()?.takeIf { it.active } ?: return false + if (isHistoricalLiveRequest(request)) return false if (session.getCachedSegment(request) == null && session.getReadableSegment(request) != null) return true val state = session.streamState observedMediaSegment(request.format)?.let { observed -> @@ -96,9 +97,12 @@ internal fun SabrSessionHolder.isFutureLiveRequest(request: SabrSegmentRequest): } internal fun SabrSessionHolder.isHistoricalLiveRequest(request: SabrSegmentRequest): Boolean { - if (request.isInitializationSegment || livePlaybackSnapshot()?.active != true) return false + if (request.isInitializationSegment) return false + val live = livePlaybackSnapshot()?.takeIf { it.active } ?: return false val observed = observedMediaSegment(request.format) ?: return false - return request.sequenceNumber < observed.header.sequenceNumber + if (request.sequenceNumber < observed.header.sequenceNumber) return true + val requestEndMs = playbackSegmentEndMs(request.format, request.sequenceNumber) + return requestEndMs < live.headTimeMs - LIVE_EDGE_TOLERANCE_MS } internal fun SabrSessionHolder.liveRetryAfterMs(blockedRequests: List = emptyList()): Long = @@ -118,4 +122,4 @@ internal const val DEFAULT_PLAYBACK_RETRY_MS = 500L private const val LIVE_TARGET_LATENCY_MS = 20_000L private const val LIVE_EDGE_TOLERANCE_MS = 15_000L private const val LIVE_DVR_WINDOW_MS = 12L * 60L * 60L * 1_000L -private const val LIVE_FUTURE_SEGMENT_TOLERANCE = 2 +internal const val LIVE_FUTURE_SEGMENT_TOLERANCE = 2 diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackGeneration.kt b/src/main/kotlin/dev/typetype/server/services/SabrPlaybackGeneration.kt new file mode 100644 index 00000000..ddbd5804 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/SabrPlaybackGeneration.kt @@ -0,0 +1,4 @@ +package dev.typetype.server.services + +internal fun SabrSessionHolder.nextReplacementGeneration(): Long = + activeGeneration().let { if (it == Long.MAX_VALUE) it else it + 1L } diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSegmentSelection.kt b/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSegmentSelection.kt index 43b344dc..2f957857 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSegmentSelection.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSegmentSelection.kt @@ -48,7 +48,7 @@ internal fun SabrSessionHolder.playbackSegmentEndMs(format: YoutubeSabrFormat, s playbackSegmentStartMs(format, sequence) + playbackSegmentDurationMs(format, sequence) private fun SabrSessionHolder.liveSequenceAt(format: YoutubeSabrFormat, playerTimeMs: Long): Int? { - livePlaybackSnapshot()?.takeIf { it.active } ?: return null + val live = livePlaybackSnapshot()?.takeIf { it.active } ?: return null val observed = observedMediaSegment(format) ?: return null val observedStartMs = observed.header.startMs.takeIf { it >= 0L } ?: return null @@ -58,8 +58,13 @@ private fun SabrSessionHolder.liveSequenceAt(format: YoutubeSabrFormat, playerTi val sequence = (observed.header.sequenceNumber.toLong() + offset) .coerceIn(1L, Int.MAX_VALUE.toLong()) .toInt() - val availableHead = runCatching { session.streamState.getMaxSegment(format) }.getOrDefault(0) - return if (availableHead > 0) sequence.coerceAtMost(availableHead) else sequence + val liveHead = live.headSequence.takeIf { it > 0L } ?: return sequence + val trackHead = observed.header.sequenceNumber + val distanceBehindLiveHead = liveHead - trackHead.toLong() + val effectiveHead = trackHead.takeIf { + it > 0 && distanceBehindLiveHead in 0L..LIVE_FUTURE_SEGMENT_TOLERANCE.toLong() + } ?: liveHead.coerceAtMost(Int.MAX_VALUE.toLong()).toInt() + return sequence.coerceAtMost(effectiveHead) } private fun SabrLivePlaybackSnapshot.segmentDurationFrom(observedSequence: Int, observedStartMs: Long): Long? { diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSessionService.kt b/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSessionService.kt index bcd59a58..2e69848c 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSessionService.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSessionService.kt @@ -6,7 +6,10 @@ import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat import org.slf4j.LoggerFactory -internal class SabrPlaybackSessionService(private val sessionStore: SabrSessionStore) { +internal class SabrPlaybackSessionService( + private val sessionStore: SabrSessionStore, + private val purpose: SabrSessionPurpose = SabrSessionPurpose.PLAYBACK, +) { suspend fun prepare( videoId: String, userId: String, @@ -16,6 +19,7 @@ internal class SabrPlaybackSessionService(private val sessionStore: SabrSessionS startTimeMs: Long, audioOnly: Boolean = false, isLive: Boolean = false, + initialGeneration: Long = 0L, ): SabrPlaybackPreparation { val holder = sessionStore.getOrCreate( videoId = videoId, @@ -26,8 +30,9 @@ internal class SabrPlaybackSessionService(private val sessionStore: SabrSessionS initialToken = prepared.initialToken, startTimeMs = startTimeMs, startPump = false, - purpose = SabrSessionPurpose.PLAYBACK, + purpose = purpose, audioOnly = audioOnly, + initialGeneration = initialGeneration, ) if (isLive || prepared.isLive || prepared.isLiveContent) holder.markExpectedLive() if (holder.expectsLive()) { @@ -58,6 +63,7 @@ internal class SabrPlaybackSessionService(private val sessionStore: SabrSessionS startTimeMs = playerTimeMs, audioOnly = audioOnly, isLive = source.expectsLive() || prepared.isLive || prepared.isLiveContent, + initialGeneration = source.nextReplacementGeneration(), ) } @@ -77,6 +83,7 @@ internal class SabrPlaybackSessionService(private val sessionStore: SabrSessionS } fun lookup(sessionId: String): SabrSessionHolder? = sessionStore.lookupByToken(sessionId) + ?.takeIf { it.key.purpose == purpose } fun startPump(holder: SabrSessionHolder): Unit = sessionStore.startPump(holder) diff --git a/src/main/kotlin/dev/typetype/server/services/SabrSegmentDemandResolution.kt b/src/main/kotlin/dev/typetype/server/services/SabrSegmentDemandResolution.kt index 4c871728..9acadfeb 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrSegmentDemandResolution.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrSegmentDemandResolution.kt @@ -1,18 +1,26 @@ package dev.typetype.server.services +import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaSegment import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest -internal fun SabrSessionHolder.resolveSegmentDemand(request: SabrSegmentRequest, identity: String): Boolean { +internal fun SabrSessionHolder.resolveSegmentDemand( + request: SabrSegmentRequest, + identity: String, + onResolved: (SabrMediaSegment) -> Unit = {}, +): Boolean { val requested = session.getCachedSegment(request) + val live = livePlaybackSnapshot()?.active == true val rebased = if (requested != null) null else session.findCachedMediaAt( format = request.format, targetMs = playbackSegmentStartMs(request.format, request.sequenceNumber), predictedSequence = request.sequenceNumber, - allowFollowing = livePlaybackSnapshot()?.active == true, + fallbackDurationMs = if (live) playbackSegmentDurationMs(request.format, request.sequenceNumber) else null, + allowFollowing = live, ) val resolved = requested ?: rebased ?: return false if (!clearSegmentDemand(request, identity)) return false observeMediaSegment(resolved) + onResolved(resolved) if (rebased != null) session.streamState.jumpBufferedTo(request.format, rebased.header.sequenceNumber) return true } diff --git a/src/main/kotlin/dev/typetype/server/services/SabrSessionHolder.kt b/src/main/kotlin/dev/typetype/server/services/SabrSessionHolder.kt index c1cd9fa4..e8fd308f 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrSessionHolder.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrSessionHolder.kt @@ -22,6 +22,7 @@ internal class SabrSessionHolder( @Volatile var lastRequestAt: Instant, @Volatile var playerContextToken: SabrTokenBundle? = null, val pumpMutex: Mutex = Mutex(), + initialGeneration: Long = 0L, ) { private val readerPositions = ConcurrentHashMap() private val lastServedSequences = ConcurrentHashMap() @@ -33,7 +34,7 @@ internal class SabrSessionHolder( private val pumpStarted = AtomicBoolean(false) private val unauthorizedRefreshAttempted = AtomicBoolean(false) private val expectedLive = AtomicBoolean(false) - private val activeGeneration = AtomicLong(0L) + private val activeGeneration = AtomicLong(initialGeneration.coerceAtLeast(0L)) private val segmentMemory = SabrMemorySegmentCache(MAX_SEGMENT_MEMORY_BYTES) private val playbackStatus = SabrPlaybackStatus() private val requestedSeekTimeMs = AtomicLong(-1L) @@ -42,7 +43,6 @@ internal class SabrSessionHolder( init { setActiveTracks(videoActive = true, audioActive = true) } - fun activeGeneration(): Long = activeGeneration.get() fun advancePlaybackGeneration(ms: Long): Long { diff --git a/src/main/kotlin/dev/typetype/server/services/SabrSessionPump.kt b/src/main/kotlin/dev/typetype/server/services/SabrSessionPump.kt index ae9283ee..62af21dc 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrSessionPump.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrSessionPump.kt @@ -11,7 +11,10 @@ internal class SabrSessionPump( private val segmentCache: SabrSegmentCache? = null, refreshPoToken: (String) -> SabrTokenBundle? = { null }, ) { - private val loop = SabrSessionPumpLoop(SabrUnauthorizedResponseRecovery(refreshPoToken)) + private val loop = SabrSessionPumpLoop( + unauthorizedRecovery = SabrUnauthorizedResponseRecovery(refreshPoToken), + onResolved = { holder, segment -> segmentCache?.put(holder, segment) }, + ) suspend fun ensureWarmed(holder: SabrSessionHolder, maxPumps: Int) { val localization = Localization("en", "US") diff --git a/src/main/kotlin/dev/typetype/server/services/SabrSessionPumpLoop.kt b/src/main/kotlin/dev/typetype/server/services/SabrSessionPumpLoop.kt index 8032c4be..4482c356 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrSessionPumpLoop.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrSessionPumpLoop.kt @@ -7,6 +7,7 @@ import kotlinx.coroutines.runInterruptible import kotlinx.coroutines.sync.withLock import org.schabi.newpipe.extractor.exceptions.ExtractionException import org.schabi.newpipe.extractor.localization.Localization +import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaSegment import org.schabi.newpipe.extractor.services.youtube.sabr.SabrRecoverableException import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrSession @@ -15,6 +16,7 @@ import java.io.IOException internal class SabrSessionPumpLoop( private val unauthorizedRecovery: SabrUnauthorizedResponseRecovery = SabrUnauthorizedResponseRecovery { null }, private val runtimeFactory: () -> SabrPumpRuntime = { SabrPumpRuntime() }, + private val onResolved: (SabrSessionHolder, SabrMediaSegment) -> Unit = { _, _ -> }, ) { suspend fun run(isAlive: () -> Boolean, holder: SabrSessionHolder, intervalMs: Long) { val localization = Localization("en", "US") @@ -104,6 +106,7 @@ internal class SabrSessionPumpLoop( result, runtime, wasFutureLiveRequest, + onResolved = { onResolved(holder, it) }, ) } finally { holder.finishInFlightSegmentDemand(demandIdentity) diff --git a/src/main/kotlin/dev/typetype/server/services/SabrSessionPurpose.kt b/src/main/kotlin/dev/typetype/server/services/SabrSessionPurpose.kt index 83f64b5e..71467c01 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrSessionPurpose.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrSessionPurpose.kt @@ -3,5 +3,6 @@ package dev.typetype.server.services internal enum class SabrSessionPurpose { MANIFEST, PLAYBACK, + ANDROID_PLAYBACK, DOWNLOAD, } diff --git a/src/main/kotlin/dev/typetype/server/services/SabrSessionStore.kt b/src/main/kotlin/dev/typetype/server/services/SabrSessionStore.kt index ff7bce9a..82e828f3 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrSessionStore.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrSessionStore.kt @@ -50,8 +50,13 @@ internal class SabrSessionStore( startPump: Boolean = true, purpose: SabrSessionPurpose = SabrSessionPurpose.MANIFEST, audioOnly: Boolean = false, + initialGeneration: Long = 0L, ): SabrSessionHolder { - val playbackToken = if (purpose == SabrSessionPurpose.PLAYBACK) SabrSessionTokenGenerator.newToken() else null + val playbackToken = if (purpose == SabrSessionPurpose.PLAYBACK || purpose == SabrSessionPurpose.ANDROID_PLAYBACK) { + SabrSessionTokenGenerator.newToken() + } else { + null + } val key = SabrSessionKey( videoId, userId, @@ -81,6 +86,7 @@ internal class SabrSessionStore( key, Instant.now(), initialToken, + initialGeneration = initialGeneration, ) holder.setPlayerTimeMs(normalizedStartTimeMs) registry.put(key, holder) diff --git a/src/main/kotlin/dev/typetype/server/services/SabrSessionTimeRequests.kt b/src/main/kotlin/dev/typetype/server/services/SabrSessionTimeRequests.kt index 8af1f5c3..985d3015 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrSessionTimeRequests.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrSessionTimeRequests.kt @@ -11,10 +11,15 @@ internal fun SabrSessionHolder.repositionTargets( playerTimeMs: Long, generation: Long, ): List = targets.filter { request -> + val cached = session.getCachedSegment(request) val warmedStartMs = adjacentWarmedLiveMediaStartMs(request, playerTimeMs) - val targetStartMs = warmedStartMs ?: playbackSegmentStartMs(request.format, request.sequenceNumber) - setReaderPosition(request.format, targetStartMs, generation) - session.getCachedSegment(request) == null && warmedStartMs == null + val targetPositionMs = when { + cached != null -> playbackSegmentEndMs(request.format, request.sequenceNumber) + warmedStartMs != null -> warmedStartMs + else -> playbackSegmentStartMs(request.format, request.sequenceNumber) + } + setReaderPosition(request.format, targetPositionMs, generation) + cached == null && warmedStartMs == null } private fun SabrSessionHolder.adjacentWarmedLiveMediaStartMs( @@ -27,8 +32,10 @@ private fun SabrSessionHolder.adjacentWarmedLiveMediaStartMs( val observedStartMs = observed.header.startMs.takeIf { it >= 0L } ?: return null val observedRequest = SabrSegmentRequest.media(request.format, observed.header.sequenceNumber) if (session.getCachedSegment(observedRequest) == null) return null + val boundaryLeadMs = playbackSegmentDurationMs(request.format, request.sequenceNumber) + + LIVE_TRACK_BOUNDARY_DRIFT_MS return observedStartMs.takeIf { - it - playerTimeMs.coerceAtLeast(0L) in 0L..LIVE_TRACK_BOUNDARY_TOLERANCE_MS + it - playerTimeMs.coerceAtLeast(0L) in 0L..boundaryLeadMs } } @@ -52,4 +59,4 @@ private fun SabrSessionHolder.mediaRequestAt( return SabrSegmentRequest.media(format, sequence) } -private const val LIVE_TRACK_BOUNDARY_TOLERANCE_MS = 500L +private const val LIVE_TRACK_BOUNDARY_DRIFT_MS = 500L diff --git a/src/main/kotlin/dev/typetype/server/services/StreamExtractionErrorMapper.kt b/src/main/kotlin/dev/typetype/server/services/StreamExtractionErrorMapper.kt index 4f7956b6..2de55302 100644 --- a/src/main/kotlin/dev/typetype/server/services/StreamExtractionErrorMapper.kt +++ b/src/main/kotlin/dev/typetype/server/services/StreamExtractionErrorMapper.kt @@ -1,6 +1,7 @@ package dev.typetype.server.services import dev.typetype.server.models.ExtractionResult +import dev.typetype.server.models.ExtractionFailureKind import org.schabi.newpipe.extractor.exceptions.AgeRestrictedContentException import org.schabi.newpipe.extractor.exceptions.GeographicRestrictionException import org.schabi.newpipe.extractor.exceptions.NeedLoginException @@ -33,7 +34,14 @@ internal object StreamExtractionErrorMapper { is GeographicRestrictionException, is AgeRestrictedContentException, is PrivateContentException -> ExtractionResult.BadRequest(sanitize(error.message) ?: "Content not available") - else -> ExtractionResult.Failure(sanitize(error.message) ?: fallback) + else -> ExtractionResult.Failure( + sanitize(error.message) ?: fallback, + kind = if (error.isYoutubeSessionRejected()) { + ExtractionFailureKind.YoutubeSessionRejected + } else { + ExtractionFailureKind.Unknown + }, + ) } private fun paidContent(message: String?): ExtractionResult.BadRequest { @@ -45,5 +53,12 @@ internal object StreamExtractionErrorMapper { } } + private fun Throwable.isYoutubeSessionRejected(): Boolean = + generateSequence(this) { it.cause } + .any { it.javaClass.name == YOUTUBE_SESSION_REJECTED_EXCEPTION } + private fun sanitize(message: String?): String? = ExtractionErrorSanitizer.sanitize(message) + + private const val YOUTUBE_SESSION_REJECTED_EXCEPTION = + "org.schabi.newpipe.extractor.exceptions.YoutubeSessionRejectedException" } diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedBuilder.kt b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedBuilder.kt new file mode 100644 index 00000000..c0b46aea --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedBuilder.kt @@ -0,0 +1,107 @@ +package dev.typetype.server.services + +import dev.typetype.server.models.ExtractionResult +import dev.typetype.server.models.SubscriptionItem +import dev.typetype.server.models.VideoItem +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.async +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.sync.Semaphore +import kotlinx.coroutines.sync.withPermit +import kotlinx.coroutines.withTimeoutOrNull +import java.net.URI + +internal class SubscriptionFeedBuilder(private val channelService: ChannelService) { + private val semaphore = Semaphore(MAX_CONCURRENT_FETCHES) + + suspend fun build(subscriptions: List): SubscriptionFeedBuildResult = coroutineScope { + val outcomes = subscriptions.map { subscription -> + async { + try { + fetchSubscription(subscription.channelUrl) + } catch (error: CancellationException) { + throw error + } catch (_: Throwable) { + SubscriptionSourceResult(emptyList(), successfulSources = 0, failedSources = 1) + } + } + }.map { it.await() } + val videos = outcomes.flatMap { it.videos }.deduplicated().sortedWith(FEED_ORDER) + SubscriptionFeedBuildResult( + videos = videos, + successfulSources = outcomes.sumOf { it.successfulSources }, + failedSources = outcomes.sumOf { it.failedSources }, + ) + } + + private suspend fun fetchSubscription(channelUrl: String): SubscriptionSourceResult = coroutineScope { + val channel = async { fetchVideos(channelUrl) } + val live = if (isYoutubeUrl(channelUrl)) async { fetchVideos(channelUrl.toLivestreamsTabUrl()) } else null + val results = listOfNotNull(channel.await(), live?.await()) + SubscriptionSourceResult( + videos = mergeVideos(results.flatMap { it.videos }), + successfulSources = results.count { it.success }, + failedSources = results.count { !it.success }, + ) + } + + private suspend fun fetchVideos(url: String): SourceFetchResult = semaphore.withPermit { + try { + withTimeoutOrNull(CHANNEL_TIMEOUT_MS) { + when (val result = channelService.getChannel(url, null)) { + is ExtractionResult.Success -> SourceFetchResult(result.data.videos, true) + else -> SourceFetchResult(emptyList(), false) + } + } ?: SourceFetchResult(emptyList(), false) + } catch (error: CancellationException) { + throw error + } catch (_: Throwable) { + SourceFetchResult(emptyList(), false) + } + } + + private fun mergeVideos(videos: List): List = videos.deduplicated() + + private fun List.deduplicated(): List = buildMap { + this@deduplicated.forEach { video -> + val key = video.feedKey() + val current = get(key) + if (current == null || video.isLive && !current.isLive) put(key, video) + } + }.values.toList() + + private fun VideoItem.feedKey(): String = url.ifBlank { id.ifBlank { "$uploaderUrl|$title" } } + + private fun String.toLivestreamsTabUrl(): String { + val uri = URI(this) + val path = uri.path.trimEnd('/') + val segments = path.split('/').filter(String::isNotBlank) + val basePath = if (segments.size >= 2 && segments.last() in YOUTUBE_CHANNEL_TABS) { + path.substringBeforeLast('/') + } else { + path + } + return URI(uri.scheme, uri.userInfo, uri.host, uri.port, "$basePath/streams", null, null).toString() + } + + private data class SourceFetchResult(val videos: List, val success: Boolean) + private data class SubscriptionSourceResult( + val videos: List, + val successfulSources: Int, + val failedSources: Int, + ) + + companion object { + private const val MAX_CONCURRENT_FETCHES = 20 + private const val CHANNEL_TIMEOUT_MS = 15_000L + private val YOUTUBE_CHANNEL_TABS = setOf("featured", "videos", "shorts", "streams", "playlists", "community", "about") + private val FEED_ORDER = compareByDescending { it.isLive } + .thenByDescending { if (it.uploaded < 0L) Long.MIN_VALUE else it.uploaded } + } +} + +internal data class SubscriptionFeedBuildResult( + val videos: List, + val successfulSources: Int, + val failedSources: Int, +) diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedCacheInvalidation.kt b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedCacheInvalidation.kt index 93f7a201..410755b6 100644 --- a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedCacheInvalidation.kt +++ b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedCacheInvalidation.kt @@ -11,4 +11,8 @@ object SubscriptionFeedCacheInvalidation { suspend fun invalidate(userId: String) { invalidator?.invalidate(userId) } + + suspend fun awaitRefresh(userId: String) { + invalidator?.awaitRefresh(userId) + } } diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedCacheInvalidator.kt b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedCacheInvalidator.kt index f2cb2d94..d570645a 100644 --- a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedCacheInvalidator.kt +++ b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedCacheInvalidator.kt @@ -2,9 +2,16 @@ package dev.typetype.server.services import dev.typetype.server.cache.CacheService -class SubscriptionFeedCacheInvalidator(private val cache: CacheService) { +class SubscriptionFeedCacheInvalidator( + private val cache: CacheService, + private val feedService: SubscriptionFeedService, +) { suspend fun invalidate(userId: String) { - runCatching { cache.delete(SubscriptionFeedCacheKeys.feed(userId)) } + feedService.invalidate(userId) runCatching { cache.delete(SubscriptionFeedCacheKeys.shorts(userId)) } } + + suspend fun awaitRefresh(userId: String) { + feedService.awaitRefresh(userId) + } } diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedCacheKeys.kt b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedCacheKeys.kt index d6921d81..d305a5da 100644 --- a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedCacheKeys.kt +++ b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedCacheKeys.kt @@ -5,6 +5,10 @@ import java.security.MessageDigest object SubscriptionFeedCacheKeys { fun feed(userId: String): String = "feed:${hash(userId)}" + fun previousFeed(userId: String): String = "feed:previous:${hash(userId)}" + + fun invalidation(userId: String): String = "feed:invalidation:${hash(userId)}" + fun shorts(userId: String): String = "feed:shorts:${hash(userId)}" private fun hash(userId: String): String = MessageDigest diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedService.kt b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedService.kt index 496cdb00..56f357b5 100644 --- a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedService.kt +++ b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedService.kt @@ -1,121 +1,157 @@ package dev.typetype.server.services -import dev.typetype.server.cache.CacheJson import dev.typetype.server.cache.CacheService -import dev.typetype.server.models.ExtractionResult +import dev.typetype.server.currentRequestId import dev.typetype.server.models.SubscriptionFeedResponse import dev.typetype.server.models.VideoItem -import kotlinx.coroutines.async -import kotlinx.coroutines.coroutineScope -import kotlinx.coroutines.sync.Semaphore -import kotlinx.coroutines.sync.withPermit -import kotlinx.coroutines.withTimeout -import kotlinx.serialization.builtins.ListSerializer -import java.net.URI -import java.util.Base64 +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.CoroutineStart +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.joinAll +import kotlinx.coroutines.launch +import kotlinx.coroutines.withTimeoutOrNull +import org.slf4j.LoggerFactory +import java.util.UUID +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.atomic.AtomicLong class SubscriptionFeedService( private val subscriptionsService: SubscriptionsService, - private val channelService: ChannelService, - private val cache: CacheService, + channelService: ChannelService, + cache: CacheService, + private val clock: () -> Long = System::currentTimeMillis, + private val refreshScope: CoroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO), ) { - private val semaphore = Semaphore(MAX_CONCURRENT_FETCHES) + private val store = SubscriptionFeedSnapshotStore(cache, clock) + private val builder = SubscriptionFeedBuilder(channelService) + private val refreshJobs = ConcurrentHashMap() + private val generation = AtomicLong(clock()) - suspend fun getFeed(userId: String, page: Int, limit: Int): SubscriptionFeedResponse { - val all = cachedAll(userId) - val from = page * limit - if (from >= all.size) return SubscriptionFeedResponse(videos = emptyList(), nextpage = null) - val to = minOf(from + limit, all.size) - val nextpage = if (to < all.size) encodeNextPage(page + 1) else null - return SubscriptionFeedResponse(videos = all.subList(from, to), nextpage = nextpage) + internal suspend fun getPage( + userId: String, + page: Int, + limit: Int, + cursor: String?, + requestId: String? = currentRequestId(), + ): SubscriptionFeedPageResult { + val current = store.current(userId) + if (current == null) { + scheduleRefresh(userId, requestId) + return SubscriptionFeedPageResult.Preparing(PREPARING_RETRY_AFTER_MS) + } + if (current.stale || clock() - current.generatedAt >= FRESHNESS_MS) { + scheduleRefresh(userId, requestId) + } + val cursorState = cursor?.let(SubscriptionFeedCursorCodec::decode) + if (cursor != null && cursorState == null) return SubscriptionFeedPageResult.InvalidCursor + if (cursorState != null && cursorState.limit != limit) return SubscriptionFeedPageResult.InvalidCursor + val snapshot = when { + cursorState == null -> current + cursorState.generation == current.generation -> current + else -> store.previous(userId)?.takeIf { it.generation == cursorState.generation } + ?: return SubscriptionFeedPageResult.StaleGeneration + } + val offset = cursorState?.offset ?: page * limit + return SubscriptionFeedPageResult.Ready(snapshot.page(offset, limit, isRefreshing(userId))) } - suspend fun getAll(userId: String): List = cachedAll(userId) + suspend fun getFeed(userId: String, page: Int, limit: Int): SubscriptionFeedResponse = + when (val result = getPage(userId, page, limit, cursor = null)) { + is SubscriptionFeedPageResult.Ready -> result.response + else -> SubscriptionFeedResponse(emptyList(), null, refreshing = true) + } + + suspend fun getAll(userId: String): List { + val snapshot = store.current(userId) + if (snapshot == null || snapshot.stale || clock() - snapshot.generatedAt >= FRESHNESS_MS) { + scheduleRefresh(userId, currentRequestId()) + } + if (snapshot != null) return snapshot.videos + withTimeoutOrNull(INTERNAL_COLD_WAIT_MS) { awaitRefresh(userId) } + return store.current(userId)?.videos.orEmpty() + } suspend fun getCachedFeed(userId: String, page: Int, limit: Int): SubscriptionFeedResponse? { - val key = SubscriptionFeedCacheKeys.feed(userId) - val raw = runCatching { cache.get(key) }.getOrNull() ?: return null - val all = runCatching { - CacheJson.decodeFromString(ListSerializer(VideoItem.serializer()), raw) - }.getOrNull() ?: return null - val from = page * limit - if (from >= all.size) return SubscriptionFeedResponse(videos = emptyList(), nextpage = null) - val to = minOf(from + limit, all.size) - val nextpage = if (to < all.size) encodeNextPage(page + 1) else null - return SubscriptionFeedResponse(videos = all.subList(from, to), nextpage = nextpage) + val snapshot = store.current(userId) ?: return null + return snapshot.page(page * limit, limit, isRefreshing(userId)) } - private suspend fun cachedAll(userId: String): List { - val key = SubscriptionFeedCacheKeys.feed(userId) - runCatching { cache.get(key) }.getOrNull()?.let { raw -> - return runCatching { CacheJson.decodeFromString(ListSerializer(VideoItem.serializer()), raw) } - .getOrElse { fetchAndCache(userId, key) } - } - return fetchAndCache(userId, key) + suspend fun invalidate(userId: String) { + runCatching { store.invalidate(userId, UUID.randomUUID().toString()) } + .onFailure { logger.warn("subscription_feed event=invalidate_failed user={} error={}", userKey(userId), it.message) } + scheduleRefresh(userId, currentRequestId()) } - private suspend fun fetchAndCache(userId: String, key: String): List { - val subs = subscriptionsService.getAll(userId) - val videos = coroutineScope { - subs.map { sub -> - async { - runCatching { fetchForSubscription(sub.channelUrl) }.getOrElse { emptyList() } - } - }.map { it.await() }.flatten() + internal suspend fun awaitRefresh(userId: String) { + while (true) { + val jobs = listOfNotNull(refreshJobs[userId]) + if (jobs.isEmpty()) return + jobs.joinAll() } - val sorted = videos.sortedWith( - compareByDescending { it.isLive } - .thenByDescending { if (it.uploaded == -1L) Long.MIN_VALUE else it.uploaded } - ) - runCatching { cache.set(key, CacheJson.encodeToString(ListSerializer(VideoItem.serializer()), sorted), FEED_TTL_SECONDS) } - return sorted } - private suspend fun fetchForSubscription(channelUrl: String): List = coroutineScope { - val channel = async { fetchVideos(channelUrl) } - val livestreams = if (isYoutubeUrl(channelUrl)) { - async { fetchVideos(channelUrl.toLivestreamsTabUrl()) } - } else null - mergeVideos(channel.await(), livestreams?.await().orEmpty()) - } + internal fun isRefreshing(userId: String): Boolean = refreshJobs[userId]?.isActive == true - private suspend fun fetchVideos(url: String): List = semaphore.withPermit { - runCatching { - withTimeout(CHANNEL_TIMEOUT_MS) { - (channelService.getChannel(url, null) as? ExtractionResult.Success)?.data?.videos.orEmpty() + private fun scheduleRefresh(userId: String, requestId: String?) { + val job = refreshScope.launch(start = CoroutineStart.LAZY) { + var retry = false + val invalidation = store.invalidationToken(userId) + try { + retry = refresh(userId, requestId, invalidation) + } catch (error: CancellationException) { + throw error + } catch (error: Throwable) { + logger.warn("subscription_feed event=refresh_failed user={} error={}", userKey(userId), error.message) + } finally { + coroutineContext[Job]?.let { refreshJobs.remove(userId, it) } } - }.getOrElse { - emptyList() + if (retry || store.invalidationToken(userId) != invalidation) scheduleRefresh(userId, requestId) } + val existing = refreshJobs.putIfAbsent(userId, job) + if (existing == null) job.start() else job.cancel() } - private fun mergeVideos(channel: List, livestreams: List): List = - buildMap { - channel.forEach { put(it.feedKey(), it) } - livestreams.forEach { put(it.feedKey(), it) } - }.values.toList() - - private fun VideoItem.feedKey(): String = url.ifBlank { id.ifBlank { "$uploaderUrl|$title" } } - - private fun String.toLivestreamsTabUrl(): String { - val uri = URI(this) - val path = uri.path.trimEnd('/') - val segments = path.split('/').filter(String::isNotBlank) - val basePath = if (segments.size >= 2 && segments.last() in YOUTUBE_CHANNEL_TABS) { - path.substringBeforeLast('/') - } else { - path + private suspend fun refresh(userId: String, requestId: String?, invalidation: String?): Boolean { + val startedAt = clock() + val previous = store.current(userId) + logger.info("subscription_feed event=refresh_started user={} requestId={}", userKey(userId), requestId ?: "none") + val subscriptions = subscriptionsService.getAll(userId) + val result = builder.build(subscriptions) + if (store.invalidationToken(userId) != invalidation) return true + val valid = result.failedSources == 0 || previous == null && result.successfulSources > 0 || subscriptions.isEmpty() + if (!valid) { + logger.warn( + "subscription_feed event=refresh_kept_previous user={} durationMs={} failedSources={}", + userKey(userId), clock() - startedAt, result.failedSources, + ) + return false } - return URI(uri.scheme, uri.userInfo, uri.host, uri.port, "$basePath/streams", null, null).toString() + val nextGeneration = generation.updateAndGet { maxOf(it + 1, clock(), (previous?.generation ?: 0L) + 1) } + val snapshot = SubscriptionFeedSnapshot(nextGeneration, clock(), stale = false, result.videos) + runCatching { store.publish(userId, snapshot) }.onFailure { + logger.warn("subscription_feed event=publish_failed user={} error={}", userKey(userId), it.message) + return false + } + if (store.invalidationToken(userId) != invalidation) { + runCatching { store.markStale(userId) } + return true + } + logger.info( + "subscription_feed event=refresh_completed user={} requestId={} generation={} videos={} failedSources={} durationMs={}", + userKey(userId), requestId ?: "none", nextGeneration, result.videos.size, result.failedSources, clock() - startedAt, + ) + return false } - private fun encodeNextPage(page: Int): String = - Base64.getEncoder().encodeToString("""{"page":$page}""".toByteArray()) + private fun userKey(userId: String): String = SubscriptionFeedCacheKeys.feed(userId).substringAfter(':') companion object { - private const val FEED_TTL_SECONDS = 60L - private const val MAX_CONCURRENT_FETCHES = 20 - private const val CHANNEL_TIMEOUT_MS = 15_000L - private val YOUTUBE_CHANNEL_TABS = setOf("featured", "videos", "shorts", "streams", "playlists", "community", "about") + private const val FRESHNESS_MS = 60_000L + private const val PREPARING_RETRY_AFTER_MS = 500L + private const val INTERNAL_COLD_WAIT_MS = 400L + private val logger = LoggerFactory.getLogger(SubscriptionFeedService::class.java) } } diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSnapshot.kt b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSnapshot.kt new file mode 100644 index 00000000..89ab75f1 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSnapshot.kt @@ -0,0 +1,73 @@ +package dev.typetype.server.services + +import dev.typetype.server.cache.CacheJson +import dev.typetype.server.models.SubscriptionFeedResponse +import dev.typetype.server.models.VideoItem +import kotlinx.serialization.Serializable +import java.util.Base64 + +@Serializable +internal data class SubscriptionFeedSnapshot( + val generation: Long, + val generatedAt: Long, + val stale: Boolean, + val videos: List, +) + +@Serializable +private data class SubscriptionFeedCursor( + val generation: Long, + val offset: Int, + val limit: Int, +) + +internal object SubscriptionFeedCursorCodec { + fun encode(generation: Long, offset: Int, limit: Int): String { + val payload = CacheJson.encodeToString( + SubscriptionFeedCursor.serializer(), + SubscriptionFeedCursor(generation, offset, limit), + ) + return Base64.getUrlEncoder().withoutPadding().encodeToString(payload.toByteArray()) + } + + fun decode(value: String): SubscriptionFeedCursorState? = runCatching { + val payload = String(Base64.getUrlDecoder().decode(value)) + val cursor = CacheJson.decodeFromString(SubscriptionFeedCursor.serializer(), payload) + cursor.takeIf { it.generation > 0L && it.offset >= 0 && it.limit in 1..100 } + ?.let { SubscriptionFeedCursorState(it.generation, it.offset, it.limit) } + }.getOrNull() +} + +internal data class SubscriptionFeedCursorState( + val generation: Long, + val offset: Int, + val limit: Int, +) + +internal fun SubscriptionFeedSnapshot.page( + offset: Int, + limit: Int, + refreshing: Boolean, +): SubscriptionFeedResponse { + val from = offset.coerceAtMost(videos.size) + val to = minOf(from + limit, videos.size) + val nextpage = if (to < videos.size) { + SubscriptionFeedCursorCodec.encode(generation, to, limit) + } else { + null + } + return SubscriptionFeedResponse( + videos = videos.subList(from, to), + nextpage = nextpage, + generation = generation, + generatedAt = generatedAt, + refreshing = refreshing, + ) +} + +internal sealed interface SubscriptionFeedPageResult { + data class Ready(val response: SubscriptionFeedResponse) : SubscriptionFeedPageResult + data class Preparing(val retryAfterMs: Long) : SubscriptionFeedPageResult + data object InvalidCursor : SubscriptionFeedPageResult + data object StaleGeneration : SubscriptionFeedPageResult +} diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSnapshotStore.kt b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSnapshotStore.kt new file mode 100644 index 00000000..365e4308 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSnapshotStore.kt @@ -0,0 +1,74 @@ +package dev.typetype.server.services + +import dev.typetype.server.cache.CacheJson +import dev.typetype.server.cache.CacheService +import dev.typetype.server.models.VideoItem +import kotlinx.serialization.builtins.ListSerializer + +internal class SubscriptionFeedSnapshotStore( + private val cache: CacheService, + private val clock: () -> Long, +) { + suspend fun current(userId: String): SubscriptionFeedSnapshot? { + val raw = runCatching { cache.get(SubscriptionFeedCacheKeys.feed(userId)) }.getOrNull() ?: return null + decodeSnapshot(raw)?.let { return it } + val legacyVideos = runCatching { + CacheJson.decodeFromString(ListSerializer(VideoItem.serializer()), raw) + }.getOrNull() ?: return null + return SubscriptionFeedSnapshot( + generation = clock().coerceAtLeast(1L), + generatedAt = clock(), + stale = true, + videos = legacyVideos, + ).also { runCatching { writeCurrent(userId, it) } } + } + + suspend fun previous(userId: String): SubscriptionFeedSnapshot? = read( + SubscriptionFeedCacheKeys.previousFeed(userId), + ) + + suspend fun invalidationToken(userId: String): String? = runCatching { + cache.get(SubscriptionFeedCacheKeys.invalidation(userId)) + }.getOrNull() + + suspend fun invalidate(userId: String, token: String) { + cache.set(SubscriptionFeedCacheKeys.invalidation(userId), token, RETENTION_SECONDS) + current(userId)?.takeUnless { it.stale }?.let { writeCurrent(userId, it.copy(stale = true)) } + } + + suspend fun markStale(userId: String) { + current(userId)?.takeUnless { it.stale }?.let { writeCurrent(userId, it.copy(stale = true)) } + } + + suspend fun publish(userId: String, snapshot: SubscriptionFeedSnapshot) { + current(userId)?.let { + cache.set( + SubscriptionFeedCacheKeys.previousFeed(userId), + CacheJson.encodeToString(SubscriptionFeedSnapshot.serializer(), it), + RETENTION_SECONDS, + ) + } + writeCurrent(userId, snapshot) + } + + private suspend fun read(key: String): SubscriptionFeedSnapshot? { + val raw = runCatching { cache.get(key) }.getOrNull() ?: return null + return decodeSnapshot(raw) + } + + private fun decodeSnapshot(raw: String): SubscriptionFeedSnapshot? = runCatching { + CacheJson.decodeFromString(SubscriptionFeedSnapshot.serializer(), raw) + }.getOrNull() + + private suspend fun writeCurrent(userId: String, snapshot: SubscriptionFeedSnapshot) { + cache.set( + SubscriptionFeedCacheKeys.feed(userId), + CacheJson.encodeToString(SubscriptionFeedSnapshot.serializer(), snapshot), + RETENTION_SECONDS, + ) + } + + companion object { + const val RETENTION_SECONDS = 24 * 60 * 60L + } +} diff --git a/src/main/kotlin/dev/typetype/server/services/TypetypeTokenSabrTokenClient.kt b/src/main/kotlin/dev/typetype/server/services/TypetypeTokenSabrTokenClient.kt index c2495705..a9447a42 100644 --- a/src/main/kotlin/dev/typetype/server/services/TypetypeTokenSabrTokenClient.kt +++ b/src/main/kotlin/dev/typetype/server/services/TypetypeTokenSabrTokenClient.kt @@ -11,21 +11,35 @@ internal class TypetypeTokenSabrTokenClient( private val client: OkHttpClient = OkHttpClient(), ) { fun fetch(videoId: String, forceRefresh: Boolean = false, refreshVideo: Boolean = false): SabrTokenBundle? { - val url = buildUrl(videoId, forceRefresh, refreshVideo) + return fetch(videoId, forceRefresh, refreshVideo, logIdentifier = true) + } + + // The Token minter accepts an opaque binding even though its internal query parameter is named videoId. + fun fetchBoundToken(binding: String): String? = + fetch(binding, forceRefresh = false, refreshVideo = false, logIdentifier = false)?.videoBoundPoToken + + private fun fetch( + binding: String, + forceRefresh: Boolean, + refreshVideo: Boolean, + logIdentifier: Boolean, + ): SabrTokenBundle? { + val url = buildUrl(binding, forceRefresh, refreshVideo) + val target = if (logIdentifier) " for $binding" else " for session binding" return try { client.newCall(Request.Builder().url(url).get().build()).execute().use { resp -> if (!resp.isSuccessful) { - System.err.println("[TypetypeTokenSabrTokenClient] /potoken HTTP ${resp.code} for $videoId") + System.err.println("[TypetypeTokenSabrTokenClient] /potoken HTTP ${resp.code}$target") return null } - SabrTokenBundle.fromResponse(videoId, JSONObject(resp.body.string())).also { + SabrTokenBundle.fromResponse(binding, JSONObject(resp.body.string())).also { if (it == null) { - System.err.println("[TypetypeTokenSabrTokenClient] incomplete token response for $videoId") + System.err.println("[TypetypeTokenSabrTokenClient] incomplete token response$target") } } } } catch (e: Exception) { - System.err.println("[TypetypeTokenSabrTokenClient] fetch failed for $videoId: ${e.message}") + System.err.println("[TypetypeTokenSabrTokenClient] fetch failed$target: ${e.message}") null } } diff --git a/src/main/kotlin/dev/typetype/server/services/TypetypeTokenYoutubeSessionPoTokenProvider.kt b/src/main/kotlin/dev/typetype/server/services/TypetypeTokenYoutubeSessionPoTokenProvider.kt new file mode 100644 index 00000000..f0fbad22 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/TypetypeTokenYoutubeSessionPoTokenProvider.kt @@ -0,0 +1,96 @@ +package dev.typetype.server.services + +import org.schabi.newpipe.extractor.ServiceList +import org.schabi.newpipe.extractor.localization.ContentCountry +import org.schabi.newpipe.extractor.localization.Localization +import org.schabi.newpipe.extractor.services.youtube.InnertubeClientRequestInfo +import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper +import org.schabi.newpipe.extractor.services.youtube.YoutubeSessionPoToken +import org.schabi.newpipe.extractor.services.youtube.YoutubeSessionPoTokenProvider +import java.security.MessageDigest +import java.util.concurrent.CompletableFuture +import java.util.concurrent.ConcurrentHashMap + +internal class TypetypeTokenYoutubeSessionPoTokenProvider( + private val boundTokenFetcher: (String) -> String?, + private val visitorDataFetcher: (Localization, ContentCountry) -> String, + private val nowMillis: () -> Long = System::currentTimeMillis, +) : YoutubeSessionPoTokenProvider { + constructor(tokenServiceUrl: String) : this( + TypetypeTokenSabrTokenClient(tokenServiceUrl).let { client -> + { binding -> client.fetchBoundToken(binding) } + }, + ::fetchAuthenticatedVisitorData, + ) + + @Volatile private var cached: CachedToken? = null + private val inFlight = ConcurrentHashMap>() + + override fun getSessionPoToken( + clientName: String, + localization: Localization, + contentCountry: ContentCountry, + loggedIn: Boolean, + ): YoutubeSessionPoToken? { + if (!loggedIn) return null + val credentialIdentity = credentialIdentity(ServiceList.YouTube.tokens) + cached?.takeIf { + it.credentialIdentity == credentialIdentity && nowMillis() - it.createdAtMs < TOKEN_TTL_MS + }?.let { return it.token } + val pending = CompletableFuture() + val existing = inFlight.putIfAbsent(credentialIdentity, pending) + if (existing != null) return existing.join()?.token + return try { + val visitorData = visitorDataFetcher(localization, contentCountry) + val token = boundTokenFetcher(visitorData)?.takeIf(String::isNotBlank) + ?: return null + val loaded = CachedToken( + credentialIdentity, + YoutubeSessionPoToken(visitorData, token), + nowMillis(), + ) + cached = loaded + pending.complete(loaded) + loaded.token + } catch (error: Throwable) { + pending.completeExceptionally(error) + throw error + } finally { + if (!pending.isDone) pending.complete(null) + inFlight.remove(credentialIdentity, pending) + } + } + + private fun credentialIdentity(cookies: String): String = MessageDigest.getInstance("SHA-256") + .digest(cookies.toByteArray()) + .joinToString("") { "%02x".format(it) } + + private data class CachedToken( + val credentialIdentity: String, + val token: YoutubeSessionPoToken, + val createdAtMs: Long, + ) + + private companion object { + const val TOKEN_TTL_MS = 6L * 60L * 60L * 1000L + + fun fetchAuthenticatedVisitorData( + localization: Localization, + contentCountry: ContentCountry, + ): String { + val headers = HashMap>() + YoutubeParsingHelper.addYoutubeHeaders(headers) + headers["Content-Type"] = listOf("application/json") + YoutubeParsingHelper.addLoggedInHeaders(headers) + return YoutubeParsingHelper.getVisitorDataFromInnertube( + InnertubeClientRequestInfo.ofWebClient(), + localization, + contentCountry, + headers, + YoutubeParsingHelper.YOUTUBEI_V1_URL, + null, + false, + ) + } + } +} diff --git a/src/main/kotlin/dev/typetype/server/services/YouTubeSubtitleService.kt b/src/main/kotlin/dev/typetype/server/services/YouTubeSubtitleService.kt index 18bbdf6c..6cb1998b 100644 --- a/src/main/kotlin/dev/typetype/server/services/YouTubeSubtitleService.kt +++ b/src/main/kotlin/dev/typetype/server/services/YouTubeSubtitleService.kt @@ -4,23 +4,57 @@ import dev.typetype.server.REQUEST_ID_HEADER import dev.typetype.server.cache.CacheJson import dev.typetype.server.currentRequestId import dev.typetype.server.models.SubtitleItem -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.withContext +import kotlinx.coroutines.suspendCancellableCoroutine +import okhttp3.Call +import okhttp3.Callback import okhttp3.OkHttpClient import okhttp3.Request +import okhttp3.Response +import java.io.IOException +import kotlin.coroutines.resume internal class YouTubeSubtitleService(private val httpClient: OkHttpClient, private val baseUrl: String) { suspend fun fetchSubtitles(videoId: String): List = - withContext(Dispatchers.IO) { - runCatching { - val response = httpClient.newCall( - Request.Builder() - .url("$baseUrl/subtitles?videoId=$videoId") - .apply { currentRequestId()?.let { header(REQUEST_ID_HEADER, it) } } - .build() - ).execute() - response.use { CacheJson.decodeFromString>(it.body.string()) } - }.getOrElse { emptyList() } + when (val result = fetchSubtitleInventory(videoId)) { + is YouTubeSubtitleInventoryResult.Ready -> result.tracks + YouTubeSubtitleInventoryResult.Unavailable -> emptyList() } + + suspend fun fetchSubtitleInventory(videoId: String): YouTubeSubtitleInventoryResult { + val requestId = currentRequestId() + return suspendCancellableCoroutine { continuation -> + val request = Request.Builder() + .url("$baseUrl/subtitles?videoId=$videoId") + .apply { requestId?.let { header(REQUEST_ID_HEADER, it) } } + .build() + val call = httpClient.newCall(request) + continuation.invokeOnCancellation { call.cancel() } + call.enqueue(object : Callback { + override fun onFailure(call: Call, e: IOException) { + if (continuation.isActive) continuation.resume(YouTubeSubtitleInventoryResult.Unavailable) + } + + override fun onResponse(call: Call, response: Response) { + val result = response.use { + if (!it.isSuccessful) { + YouTubeSubtitleInventoryResult.Unavailable + } else { + runCatching { + YouTubeSubtitleInventoryResult.Ready( + CacheJson.decodeFromString>(it.body.string()), + ) + }.getOrDefault(YouTubeSubtitleInventoryResult.Unavailable) + } + } + if (continuation.isActive) continuation.resume(result) + } + }) + } + } +} + +internal sealed interface YouTubeSubtitleInventoryResult { + data class Ready(val tracks: List) : YouTubeSubtitleInventoryResult + data object Unavailable : YouTubeSubtitleInventoryResult } diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeSessionCredentialValidator.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeSessionCredentialValidator.kt index b12f32ec..56dd1714 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeSessionCredentialValidator.kt +++ b/src/main/kotlin/dev/typetype/server/services/YoutubeSessionCredentialValidator.kt @@ -4,12 +4,14 @@ object YoutubeSessionCredentialValidator { private const val MAX_COOKIES_LENGTH = 32 * 1024 private const val MAX_PO_TOKEN_LENGTH = 16 * 1024 private val sessionCookieNames = listOf("SID", "__Secure-1PSID", "__Secure-3PSID") + private val authorizationCookieNames = listOf("SAPISID", "__Secure-3PAPISID") fun isValid(cookies: String, poToken: String): Boolean { if (cookies.isBlank() || poToken.isBlank()) return false if (cookies.length > MAX_COOKIES_LENGTH || poToken.length > MAX_PO_TOKEN_LENGTH) return false if (poToken.length < 8) return false - return sessionCookieNames.any { cookies.hasCookie(it) } + return sessionCookieNames.any { cookies.hasCookie(it) } && + authorizationCookieNames.any { cookies.hasCookie(it) } } private fun String.hasCookie(name: String): Boolean = diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeSessionStreamService.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeSessionStreamService.kt index ebb0d30e..0c14ab8c 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeSessionStreamService.kt +++ b/src/main/kotlin/dev/typetype/server/services/YoutubeSessionStreamService.kt @@ -2,6 +2,7 @@ package dev.typetype.server.services import dev.typetype.server.cache.CacheService import dev.typetype.server.models.ExtractionResult +import dev.typetype.server.models.ExtractionFailureKind import dev.typetype.server.models.StreamResponse const val YOUTUBE_SESSION_RECONNECT_ERROR = "YouTube session needs reconnect" @@ -66,31 +67,11 @@ class YoutubeSessionStreamService( } } - private fun requiresReconnect(result: ExtractionResult): Boolean = when (result) { - is ExtractionResult.Success -> false - is ExtractionResult.BadRequest -> result.message.isSessionRejected() - is ExtractionResult.Failure -> result.message.isSessionRejected() - } - - private fun String.isSessionRejected(): Boolean { - val message = lowercase() - return rejectionSignals.any { it in message } - } + private fun requiresReconnect(result: ExtractionResult): Boolean = + result is ExtractionResult.Failure && + result.kind == ExtractionFailureKind.YoutubeSessionRejected private companion object { const val AUTHENTICATED_STREAM_MAX_TTL_SECONDS = 900L - val rejectionSignals = listOf( - "sign in", - "not a bot", - "login", - "cookie", - "sapisid", - "po token", - "pot", - "unauthorized", - "forbidden", - "no suitable stream", - "failed to load stream", - ) } } diff --git a/src/test/kotlin/dev/typetype/server/InstanceRoutesTest.kt b/src/test/kotlin/dev/typetype/server/InstanceRoutesTest.kt index d0846b29..58f5a78f 100644 --- a/src/test/kotlin/dev/typetype/server/InstanceRoutesTest.kt +++ b/src/test/kotlin/dev/typetype/server/InstanceRoutesTest.kt @@ -74,6 +74,12 @@ class InstanceRoutesTest { assertEquals(false, root["youtubeRemoteLoginReady"]?.jsonPrimitive?.boolean) assertEquals("disabled", root["youtubeRemoteLoginUnavailableReason"]?.jsonPrimitive?.contentOrNull) assertEquals(listOf(0, 3, 4, 5, 6), root["supportedServices"]?.jsonArray?.map { it.jsonPrimitive.int }) + val androidPlayback = root["androidPlayback"]?.jsonObject + assertEquals(true, androidPlayback?.get("supported")?.jsonPrimitive?.boolean) + assertEquals(2, androidPlayback?.get("contractVersion")?.jsonPrimitive?.int) + assertEquals(true, androidPlayback?.get("youtube")?.jsonObject?.get("vod")?.jsonPrimitive?.boolean) + assertEquals(false, androidPlayback?.get("youtube")?.jsonObject?.get("live")?.jsonPrimitive?.boolean) + assertEquals(true, androidPlayback?.get("youtube")?.jsonObject?.get("subtitles")?.jsonPrimitive?.boolean) } @Test diff --git a/src/test/kotlin/dev/typetype/server/NotificationsRoutesTest.kt b/src/test/kotlin/dev/typetype/server/NotificationsRoutesTest.kt index 76aa9884..92d21d32 100644 --- a/src/test/kotlin/dev/typetype/server/NotificationsRoutesTest.kt +++ b/src/test/kotlin/dev/typetype/server/NotificationsRoutesTest.kt @@ -1,6 +1,5 @@ package dev.typetype.server -import dev.typetype.server.cache.CacheService import dev.typetype.server.SubscriptionFeedTestFixtures.channel import dev.typetype.server.SubscriptionFeedTestFixtures.subscription import dev.typetype.server.SubscriptionFeedTestFixtures.video @@ -31,11 +30,9 @@ import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test class NotificationsRoutesTest { - private val channelService: ChannelService = mockk() - private val cacheService: CacheService = mockk() - private val subscriptionsService = SubscriptionsService() - private val subscriptionFeedService = SubscriptionFeedService(subscriptionsService, channelService, cacheService) - private val notificationsService = NotificationsService(subscriptionFeedService) + private lateinit var channelService: ChannelService + private lateinit var subscriptionsService: SubscriptionsService + private lateinit var notificationsService: NotificationsService private val auth = AuthService.fixed(TEST_USER_ID) companion object { @BeforeAll @JvmStatic fun initDb() = TestDatabase.setup() } @@ -43,8 +40,14 @@ class NotificationsRoutesTest { @BeforeEach fun clean() { TestDatabase.truncateAll() - coEvery { cacheService.get(any()) } returns null - coEvery { cacheService.set(any(), any(), any()) } returns Unit + channelService = mockk() + subscriptionsService = SubscriptionsService() + val subscriptionFeedService = SubscriptionFeedService( + subscriptionsService, + channelService, + FakeCacheService(), + ) + notificationsService = NotificationsService(subscriptionFeedService) } private fun withApp(block: suspend ApplicationTestBuilder.() -> Unit) = testApplication { application { diff --git a/src/test/kotlin/dev/typetype/server/StreamExtractionErrorMapperTest.kt b/src/test/kotlin/dev/typetype/server/StreamExtractionErrorMapperTest.kt index b88fa204..dc8c27a3 100644 --- a/src/test/kotlin/dev/typetype/server/StreamExtractionErrorMapperTest.kt +++ b/src/test/kotlin/dev/typetype/server/StreamExtractionErrorMapperTest.kt @@ -1,9 +1,11 @@ package dev.typetype.server import dev.typetype.server.models.ExtractionResult +import dev.typetype.server.models.ExtractionFailureKind import dev.typetype.server.services.StreamExtractionErrorMapper import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Assumptions.assumeTrue import org.junit.jupiter.api.Test import org.schabi.newpipe.extractor.exceptions.NeedLoginException import org.schabi.newpipe.extractor.exceptions.PaidContentException @@ -73,5 +75,21 @@ class StreamExtractionErrorMapperTest { val result = StreamExtractionErrorMapper.map(IllegalStateException("boom")) assertTrue(result is ExtractionResult.Failure) assertEquals("boom", (result as ExtractionResult.Failure).message) + assertEquals(ExtractionFailureKind.Unknown, result.kind) + } + + @Test + fun `maps explicit YouTube session rejection without inspecting its message`() { + val type = runCatching { + Class.forName("org.schabi.newpipe.extractor.exceptions.YoutubeSessionRejectedException") + }.getOrNull() + assumeTrue(type != null, "requires the local updated PipePipeExtractor") + val rejectionType = type ?: return + val error = rejectionType.getConstructor(String::class.java).newInstance("arbitrary") as Throwable + + val result = StreamExtractionErrorMapper.map(error) + + assertTrue(result is ExtractionResult.Failure) + assertEquals(ExtractionFailureKind.YoutubeSessionRejected, (result as ExtractionResult.Failure).kind) } } diff --git a/src/test/kotlin/dev/typetype/server/StreamRoutesDeliveryModeTest.kt b/src/test/kotlin/dev/typetype/server/StreamRoutesDeliveryModeTest.kt index 179c8109..b8d4388d 100644 --- a/src/test/kotlin/dev/typetype/server/StreamRoutesDeliveryModeTest.kt +++ b/src/test/kotlin/dev/typetype/server/StreamRoutesDeliveryModeTest.kt @@ -96,8 +96,8 @@ class StreamRoutesDeliveryModeTest { } @Test - fun `sabr endpoint skips public extraction when authenticated sabr is available`() = testApplication { - coEvery { sabrService.getStreamInfo(any()) } returns ExtractionResult.Failure("unexpected public path") + fun `sabr endpoint never uses the connected YouTube session`() = testApplication { + coEvery { sabrService.getStreamInfo(any()) } returns ExtractionResult.Success(sabrResponse()) var sessionCalls = 0 application { installRoutes(session = { _, _ -> @@ -111,8 +111,8 @@ class StreamRoutesDeliveryModeTest { } assertEquals(HttpStatusCode.OK, response.status) - assertEquals(1, sessionCalls) - coVerify(exactly = 0) { sabrService.getStreamInfo(any()) } + assertEquals(0, sessionCalls) + coVerify(exactly = 1) { sabrService.getStreamInfo(VIDEO_URL) } coVerify(exactly = 0) { legacyService.getStreamInfo(any()) } } diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionFeedCacheInvalidationPipePipeTest.kt b/src/test/kotlin/dev/typetype/server/SubscriptionFeedCacheInvalidationPipePipeTest.kt index 53fc0f76..dc0b9ece 100644 --- a/src/test/kotlin/dev/typetype/server/SubscriptionFeedCacheInvalidationPipePipeTest.kt +++ b/src/test/kotlin/dev/typetype/server/SubscriptionFeedCacheInvalidationPipePipeTest.kt @@ -51,7 +51,7 @@ class SubscriptionFeedCacheInvalidationPipePipeTest { fun clean() = runBlocking { TestDatabase.truncateAll() cache.clear() - SubscriptionFeedCacheInvalidation.configure(SubscriptionFeedCacheInvalidator(cache)) + SubscriptionFeedCacheInvalidation.configure(SubscriptionFeedCacheInvalidator(cache, feed)) } @Test @@ -66,8 +66,12 @@ class SubscriptionFeedCacheInvalidationPipePipeTest { subscriptions.add(TEST_USER_ID, SubscriptionItem("https://yt.com/c/pre", "Pre", "")) val before = client.get("/subscriptions/feed") { header(HttpHeaders.Authorization, "Bearer test-jwt") } - assertEquals(HttpStatusCode.OK, before.status) - assertTrue(before.bodyAsText().contains("pre/video")) + assertEquals(HttpStatusCode.Accepted, before.status) + feed.awaitRefresh(TEST_USER_ID) + assertTrue( + client.get("/subscriptions/feed") { header(HttpHeaders.Authorization, "Bearer test-jwt") } + .bodyAsText().contains("pre/video"), + ) val zip = PipePipeBackupTestFixtures.createBackupZip() val payload = Files.readAllBytes(zip) @@ -89,6 +93,8 @@ class SubscriptionFeedCacheInvalidationPipePipeTest { val after = client.get("/subscriptions/feed") { header(HttpHeaders.Authorization, "Bearer test-jwt") } assertEquals(HttpStatusCode.OK, after.status) - assertTrue(after.bodyAsText().contains("youtube.com/@x/video")) + feed.awaitRefresh(TEST_USER_ID) + val refreshed = client.get("/subscriptions/feed") { header(HttpHeaders.Authorization, "Bearer test-jwt") } + assertTrue(refreshed.bodyAsText().contains("youtube.com/@x/video")) } } diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionFeedCacheInvalidationPipePipeUnitTest.kt b/src/test/kotlin/dev/typetype/server/SubscriptionFeedCacheInvalidationPipePipeUnitTest.kt index d85c1392..7e06f65d 100644 --- a/src/test/kotlin/dev/typetype/server/SubscriptionFeedCacheInvalidationPipePipeUnitTest.kt +++ b/src/test/kotlin/dev/typetype/server/SubscriptionFeedCacheInvalidationPipePipeUnitTest.kt @@ -6,6 +6,7 @@ import dev.typetype.server.services.PipePipeBackupSubscriptionItem import dev.typetype.server.services.SubscriptionFeedCacheInvalidation import dev.typetype.server.services.SubscriptionFeedCacheInvalidator import dev.typetype.server.services.SubscriptionFeedCacheKeys +import dev.typetype.server.services.SubscriptionFeedService import dev.typetype.server.services.SubscriptionsService import kotlinx.coroutines.runBlocking import org.junit.jupiter.api.Assertions.assertFalse @@ -27,11 +28,12 @@ class SubscriptionFeedCacheInvalidationPipePipeUnitTest { fun clean() = runBlocking { TestDatabase.truncateAll() cache.clear() - SubscriptionFeedCacheInvalidation.configure(SubscriptionFeedCacheInvalidator(cache)) + val feed = SubscriptionFeedService(SubscriptionsService(), FakeChannelService(), cache) + SubscriptionFeedCacheInvalidation.configure(SubscriptionFeedCacheInvalidator(cache, feed)) } @Test - fun `restore clears feed and shorts cache keys`() = runBlocking { + fun `restore retains feed snapshot and clears shorts cache`() = runBlocking { val userId = TEST_USER_ID val feedKey = SubscriptionFeedCacheKeys.feed(userId) val shortsKey = SubscriptionFeedCacheKeys.shorts(userId) @@ -46,7 +48,7 @@ class SubscriptionFeedCacheInvalidationPipePipeUnitTest { ) PipePipeBackupPersisterService().persist(userId, snapshot) assertTrue(SubscriptionsService().getAll(userId).isNotEmpty()) - assertFalse(cache.get(feedKey) != null) + assertTrue(cache.get(feedKey) != null) assertFalse(cache.get(shortsKey) != null) } } diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionFeedCacheInvalidationTakeoutUnitTest.kt b/src/test/kotlin/dev/typetype/server/SubscriptionFeedCacheInvalidationTakeoutUnitTest.kt index 3c351882..7cabdca9 100644 --- a/src/test/kotlin/dev/typetype/server/SubscriptionFeedCacheInvalidationTakeoutUnitTest.kt +++ b/src/test/kotlin/dev/typetype/server/SubscriptionFeedCacheInvalidationTakeoutUnitTest.kt @@ -10,6 +10,7 @@ import dev.typetype.server.services.PlaylistService import dev.typetype.server.services.SubscriptionFeedCacheInvalidation import dev.typetype.server.services.SubscriptionFeedCacheInvalidator import dev.typetype.server.services.SubscriptionFeedCacheKeys +import dev.typetype.server.services.SubscriptionFeedService import dev.typetype.server.services.SubscriptionsService import dev.typetype.server.services.WatchLaterService import dev.typetype.server.services.YoutubeTakeoutImporterService @@ -34,18 +35,19 @@ class SubscriptionFeedCacheInvalidationTakeoutUnitTest { fun clean() = runBlocking { TestDatabase.truncateAll() cache.clear() - SubscriptionFeedCacheInvalidation.configure(SubscriptionFeedCacheInvalidator(cache)) + val feed = SubscriptionFeedService(SubscriptionsService(), FakeChannelService(), cache) + SubscriptionFeedCacheInvalidation.configure(SubscriptionFeedCacheInvalidator(cache, feed)) } @Test - fun `takeout with subscriptions clears feed and shorts cache keys`() = runBlocking { + fun `takeout with subscriptions retains feed and clears shorts cache`() = runBlocking { val userId = TEST_USER_ID val feedKey = SubscriptionFeedCacheKeys.feed(userId) val shortsKey = SubscriptionFeedCacheKeys.shorts(userId) cache.set(feedKey, "warm", 300) cache.set(shortsKey, "warm", 300) importer().commit(userId, parsed(), withSubscriptions = true) - assertFalse(cache.get(feedKey) != null) + assertTrue(cache.get(feedKey) != null) assertFalse(cache.get(shortsKey) != null) } diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionFeedCacheKeysTest.kt b/src/test/kotlin/dev/typetype/server/SubscriptionFeedCacheKeysTest.kt index 3d4e783b..77f83e77 100644 --- a/src/test/kotlin/dev/typetype/server/SubscriptionFeedCacheKeysTest.kt +++ b/src/test/kotlin/dev/typetype/server/SubscriptionFeedCacheKeysTest.kt @@ -11,9 +11,14 @@ class SubscriptionFeedCacheKeysTest { val feedA = SubscriptionFeedCacheKeys.feed("user-a") val feedAAgain = SubscriptionFeedCacheKeys.feed("user-a") val feedB = SubscriptionFeedCacheKeys.feed("user-b") + val previousA = SubscriptionFeedCacheKeys.previousFeed("user-a") + val invalidationA = SubscriptionFeedCacheKeys.invalidation("user-a") val shortsA = SubscriptionFeedCacheKeys.shorts("user-a") assertEquals(feedA, feedAAgain) assertNotEquals(feedA, feedB) assertNotEquals(feedA, shortsA) + assertNotEquals(feedA, previousA) + assertNotEquals(feedA, invalidationA) + assertNotEquals(previousA, invalidationA) } } diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionFeedRoutesTest.kt b/src/test/kotlin/dev/typetype/server/SubscriptionFeedRoutesTest.kt index 44bf6238..e22484a2 100644 --- a/src/test/kotlin/dev/typetype/server/SubscriptionFeedRoutesTest.kt +++ b/src/test/kotlin/dev/typetype/server/SubscriptionFeedRoutesTest.kt @@ -1,18 +1,18 @@ package dev.typetype.server -import dev.typetype.server.cache.CacheService -import dev.typetype.server.models.ExtractionResult -import dev.typetype.server.models.SubscriptionFeedResponse import dev.typetype.server.SubscriptionFeedTestFixtures.channel import dev.typetype.server.SubscriptionFeedTestFixtures.subscription import dev.typetype.server.SubscriptionFeedTestFixtures.video +import dev.typetype.server.models.SubscriptionFeedResponse import dev.typetype.server.routes.subscriptionFeedRoutes import dev.typetype.server.services.AuthService import dev.typetype.server.services.ChannelService import dev.typetype.server.services.SubscriptionFeedService import dev.typetype.server.services.SubscriptionsService import io.ktor.client.request.get -import io.ktor.client.request.headers +import io.ktor.client.request.header +import io.ktor.client.request.parameter +import io.ktor.client.statement.HttpResponse import io.ktor.client.statement.bodyAsText import io.ktor.http.HttpHeaders import io.ktor.http.HttpStatusCode @@ -25,27 +25,36 @@ import io.ktor.server.testing.testApplication import io.mockk.coEvery import io.mockk.coVerify import io.mockk.mockk +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.async +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.delay import kotlinx.serialization.json.Json import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertNotNull import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.BeforeAll import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test +import kotlin.system.measureTimeMillis class SubscriptionFeedRoutesTest { - - private val channelService: ChannelService = mockk() - private val cacheService: CacheService = mockk() - private val subscriptionsService = SubscriptionsService() - private val feedService = SubscriptionFeedService(subscriptionsService, channelService, cacheService) + private lateinit var channelService: ChannelService + private lateinit var cacheService: FakeCacheService + private lateinit var subscriptionsService: SubscriptionsService + private lateinit var feedService: SubscriptionFeedService private val auth = AuthService.fixed(TEST_USER_ID) companion object { @BeforeAll @JvmStatic fun initDb() = TestDatabase.setup() } - @BeforeEach fun clean() { + @BeforeEach + fun clean() { TestDatabase.truncateAll() - coEvery { cacheService.get(any()) } returns null - coEvery { cacheService.set(any(), any(), any()) } returns Unit + channelService = mockk() + cacheService = FakeCacheService() + subscriptionsService = SubscriptionsService() + feedService = SubscriptionFeedService(subscriptionsService, channelService, cacheService) } private fun withApp(block: suspend ApplicationTestBuilder.() -> Unit) = testApplication { @@ -62,108 +71,159 @@ class SubscriptionFeedRoutesTest { } @Test - fun `GET subscriptions feed with no subscriptions returns empty`() = withApp { - val body = client.get("/subscriptions/feed") { headers.append(HttpHeaders.Authorization, "Bearer test-jwt") }.bodyAsText() - assertTrue(body.contains("\"videos\":[]")) + fun `cold feed with one slow source returns bounded 202`() = withApp { + repeat(100) { subscriptionsService.add(TEST_USER_ID, subscription(it + 1)) } + val slowSource = CompletableDeferred() + coEvery { channelService.getChannel(any(), null) } coAnswers { + if (firstArg().endsWith("/100")) slowSource.await() + channel(video(1000L)) + } + lateinit var response: HttpResponse + val elapsed = measureTimeMillis { response = requestFeed() } + assertEquals(HttpStatusCode.Accepted, response.status) + assertEquals("1", response.headers[HttpHeaders.RetryAfter]) + assertTrue(response.bodyAsText().contains("subscription_feed_preparing")) + assertTrue(elapsed < 500, "cold response took ${elapsed}ms") + slowSource.complete(Unit) + feedService.awaitRefresh(TEST_USER_ID) + assertEquals(HttpStatusCode.OK, requestFeed().status) } @Test - fun `GET subscriptions feed returns videos sorted by uploaded desc`() = withApp { + fun `concurrent cold requests share one refresh`() = withApp { subscriptionsService.add(TEST_USER_ID, subscription(1)) - subscriptionsService.add(TEST_USER_ID, subscription(2)) - coEvery { channelService.getChannel("https://yt.com/c/1", null) } returns channel(video(1000L), video(3000L)) - coEvery { channelService.getChannel("https://yt.com/c/2", null) } returns channel(video(2000L)) - val body = client.get("/subscriptions/feed?page=0&limit=10") { - headers.append(HttpHeaders.Authorization, "Bearer test-jwt") - }.bodyAsText() - assertTrue(body.indexOf("3000") < body.indexOf("2000")) - assertTrue(body.indexOf("2000") < body.indexOf("1000")) + coEvery { channelService.getChannel(any(), null) } coAnswers { + delay(100) + channel(video(1000L)) + } + val responses = coroutineScope { List(12) { async { requestFeed() } }.map { it.await() } } + assertTrue(responses.all { it.status == HttpStatusCode.Accepted }) + feedService.awaitRefresh(TEST_USER_ID) + coVerify(exactly = 1) { channelService.getChannel("https://yt.com/c/1", null) } } @Test - fun `GET subscriptions feed merges livestream tab and promotes active live`() = withApp { + fun `ready feed sorts live first and unknown dates last`() = withApp { val channelUrl = "https://www.youtube.com/channel/UC1" - val liveUrl = "https://www.youtube.com/watch?v=live" subscriptionsService.add(TEST_USER_ID, subscription(channelUrl, "Live channel")) coEvery { channelService.getChannel(channelUrl, null) } returns channel( video(3000L, url = "https://www.youtube.com/watch?v=normal"), - video(2000L, url = liveUrl), + video(2000L, url = "https://www.youtube.com/watch?v=live"), ) coEvery { channelService.getChannel("$channelUrl/streams", null) } returns channel( - video(-1L, url = liveUrl, live = true), - video(5000L, url = "https://www.youtube.com/watch?v=upcoming"), + video(-1L, url = "https://www.youtube.com/watch?v=live", live = true), + video(-1L, url = "https://www.youtube.com/watch?v=unknown"), ) - - val body = client.get("/subscriptions/feed") { - headers.append(HttpHeaders.Authorization, "Bearer test-jwt") - }.bodyAsText() - val feed = Json.decodeFromString(body) - - assertEquals(liveUrl, feed.videos.first().url) + val feed = buildAndRead() assertTrue(feed.videos.first().isLive) - assertEquals(1, feed.videos.count { it.url == liveUrl }) - assertTrue(feed.videos.any { it.url.endsWith("v=upcoming") }) - coVerify { cacheService.set(any(), any(), 60L) } + assertEquals(1, feed.videos.count { it.url.endsWith("v=live") }) + assertTrue(feed.videos.last().url.endsWith("v=unknown")) + assertNotNull(feed.generation) + assertNotNull(feed.generatedAt) } @Test - fun `GET subscriptions feed keeps channel videos when livestream tab fails`() = withApp { - val channelUrl = "https://www.youtube.com/channel/UC2" - subscriptionsService.add(TEST_USER_ID, subscription(channelUrl, "Channel")) - coEvery { channelService.getChannel(channelUrl, null) } returns channel(video(1000L)) - coEvery { channelService.getChannel("$channelUrl/streams", null) } returns ExtractionResult.Failure("err") - - val body = client.get("/subscriptions/feed") { - headers.append(HttpHeaders.Authorization, "Bearer test-jwt") - }.bodyAsText() - - assertTrue(body.contains("V1000")) + fun `cursor keeps pagination on one generation after refresh`() = withApp { + subscriptionsService.add(TEST_USER_ID, subscription(1)) + var newest = 5_000L + coEvery { channelService.getChannel(any(), null) } coAnswers { + channel(video(newest), video(4000L), video(3000L)) + } + val first = buildAndRead(limit = 2) + val cursor = requireNotNull(first.nextpage) + newest = 9_000L + feedService.invalidate(TEST_USER_ID) + feedService.awaitRefresh(TEST_USER_ID) + val second = requestFeed(limit = 2, cursor = cursor) + assertEquals(HttpStatusCode.OK, second.status) + val page = Json.decodeFromString(second.bodyAsText()) + assertEquals(first.generation, page.generation) + assertEquals(listOf(3000L), page.videos.map { it.uploaded }) } @Test - fun `GET subscriptions feed replaces existing channel tab with livestream tab`() = withApp { - val channelUrl = "https://www.youtube.com/channel/UC3/videos" - val livestreamUrl = "https://www.youtube.com/channel/UC3/streams" - subscriptionsService.add(TEST_USER_ID, subscription(channelUrl, "Imported channel")) - coEvery { channelService.getChannel(channelUrl, null) } returns channel(video(1000L)) - coEvery { channelService.getChannel(livestreamUrl, null) } returns channel( - video(-1L, url = "https://www.youtube.com/watch?v=live", live = true), - ) - - val body = client.get("/subscriptions/feed") { - headers.append(HttpHeaders.Authorization, "Bearer test-jwt") - }.bodyAsText() - - assertTrue(body.contains("v=live")) + fun `cursor older than retained generation returns typed 409`() = withApp { + subscriptionsService.add(TEST_USER_ID, subscription(1)) + var newest = 5_000L + coEvery { channelService.getChannel(any(), null) } coAnswers { + channel(video(newest), video(1000L)) + } + val cursor = requireNotNull(buildAndRead(limit = 1).nextpage) + repeat(2) { + newest += 1_000 + feedService.invalidate(TEST_USER_ID) + feedService.awaitRefresh(TEST_USER_ID) + } + val response = requestFeed(limit = 1, cursor = cursor) + assertEquals(HttpStatusCode.Conflict, response.status) + assertTrue(response.bodyAsText().contains("subscription_feed_stale_generation")) } @Test - fun `GET subscriptions feed pagination works`() = withApp { + fun `stale feed returns immediately while one refresh runs`() = withApp { subscriptionsService.add(TEST_USER_ID, subscription(1)) - coEvery { channelService.getChannel(any(), null) } returns channel(video(5000L), video(4000L), video(3000L)) - val body = client.get("/subscriptions/feed?page=0&limit=2") { - headers.append(HttpHeaders.Authorization, "Bearer test-jwt") - }.bodyAsText() - assertTrue(body.contains("5000") && body.contains("4000") && !body.contains("3000")) - assertTrue(body.contains("\"nextpage\":\"")) + val gate = CompletableDeferred() + var rebuilding = false + coEvery { channelService.getChannel(any(), null) } coAnswers { + if (rebuilding) gate.await() + channel(video(if (rebuilding) 2000L else 1000L)) + } + buildAndRead() + rebuilding = true + feedService.invalidate(TEST_USER_ID) + lateinit var response: HttpResponse + val elapsed = measureTimeMillis { response = requestFeed() } + val stale = Json.decodeFromString(response.bodyAsText()) + assertEquals(HttpStatusCode.OK, response.status) + assertEquals(1000L, stale.videos.single().uploaded) + assertTrue(stale.refreshing) + assertTrue(elapsed < 500, "stale response took ${elapsed}ms") + gate.complete(Unit) + feedService.awaitRefresh(TEST_USER_ID) } @Test - fun `GET subscriptions feed failed channel is silently ignored`() = withApp { + fun `partial refresh keeps the previous complete snapshot`() = withApp { subscriptionsService.add(TEST_USER_ID, subscription(1)) subscriptionsService.add(TEST_USER_ID, subscription(2)) - coEvery { channelService.getChannel("https://yt.com/c/1", null) } returns channel(video(1000L)) - coEvery { channelService.getChannel("https://yt.com/c/2", null) } returns ExtractionResult.Failure("err") - val resp = client.get("/subscriptions/feed") { headers.append(HttpHeaders.Authorization, "Bearer test-jwt") } - assertEquals(HttpStatusCode.OK, resp.status) - assertTrue(resp.bodyAsText().contains("1000")) + var rebuilding = false + coEvery { channelService.getChannel("https://yt.com/c/1", null) } coAnswers { + channel(video(if (rebuilding) 9000L else 1000L)) + } + coEvery { channelService.getChannel("https://yt.com/c/2", null) } coAnswers { + if (rebuilding) error("channel unavailable") else channel(video(2000L)) + } + val original = buildAndRead() + rebuilding = true + feedService.invalidate(TEST_USER_ID) + feedService.awaitRefresh(TEST_USER_ID) + val retained = Json.decodeFromString(requestFeed().bodyAsText()) + assertEquals(original.generation, retained.generation) + assertEquals(setOf(1000L, 2000L), retained.videos.map { it.uploaded }.toSet()) + assertFalse(retained.videos.any { it.uploaded == 9000L }) } @Test - fun `GET subscriptions feed last page has null nextpage`() = withApp { - subscriptionsService.add(TEST_USER_ID, subscription(1)) + fun `invalid cursor returns typed 400`() = withApp { coEvery { channelService.getChannel(any(), null) } returns channel(video(1000L)) - val body = client.get("/subscriptions/feed") { headers.append(HttpHeaders.Authorization, "Bearer test-jwt") }.bodyAsText() - assertTrue(body.contains("\"nextpage\":null")) + buildAndRead() + val response = requestFeed(cursor = "not-a-cursor") + assertEquals(HttpStatusCode.BadRequest, response.status) + assertTrue(response.bodyAsText().contains("subscription_feed_invalid_cursor")) } + + private suspend fun ApplicationTestBuilder.buildAndRead(limit: Int = 30): SubscriptionFeedResponse { + assertEquals(HttpStatusCode.Accepted, requestFeed(limit).status) + feedService.awaitRefresh(TEST_USER_ID) + val response = requestFeed(limit) + assertEquals(HttpStatusCode.OK, response.status) + return Json.decodeFromString(response.bodyAsText()) + } + + private suspend fun ApplicationTestBuilder.requestFeed(limit: Int = 30, cursor: String? = null): HttpResponse = + client.get("/subscriptions/feed") { + header(HttpHeaders.Authorization, "Bearer test-jwt") + parameter("limit", limit) + cursor?.let { parameter("cursor", it) } + } } diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionFeedServiceTest.kt b/src/test/kotlin/dev/typetype/server/SubscriptionFeedServiceTest.kt new file mode 100644 index 00000000..ec068392 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/SubscriptionFeedServiceTest.kt @@ -0,0 +1,37 @@ +package dev.typetype.server + +import dev.typetype.server.SubscriptionFeedTestFixtures.subscription +import dev.typetype.server.services.SubscriptionFeedPageResult +import dev.typetype.server.services.SubscriptionFeedService +import dev.typetype.server.services.SubscriptionsService +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test + +class SubscriptionFeedServiceTest { + companion object { @BeforeAll @JvmStatic fun initDb() = TestDatabase.setup() } + + @BeforeEach + fun clean() = TestDatabase.truncateAll() + + @Test + fun `snapshots and cursors remain scoped to their user`() = runTest { + val subscriptions = SubscriptionsService() + subscriptions.add("user-a", subscription("https://example.com/a", "A")) + subscriptions.add("user-b", subscription("https://example.com/b", "B")) + val service = SubscriptionFeedService(subscriptions, FakeChannelService(), FakeCacheService()) + + assertTrue(service.getPage("user-a", 0, 30, null) is SubscriptionFeedPageResult.Preparing) + assertTrue(service.getPage("user-b", 0, 30, null) is SubscriptionFeedPageResult.Preparing) + service.awaitRefresh("user-a") + service.awaitRefresh("user-b") + + val pageA = service.getPage("user-a", 0, 30, null) as SubscriptionFeedPageResult.Ready + val pageB = service.getPage("user-b", 0, 30, null) as SubscriptionFeedPageResult.Ready + assertEquals(listOf("https://example.com/a/video"), pageA.response.videos.map { it.url }) + assertEquals(listOf("https://example.com/b/video"), pageB.response.videos.map { it.url }) + } +} diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionFeedSnapshotStoreTest.kt b/src/test/kotlin/dev/typetype/server/SubscriptionFeedSnapshotStoreTest.kt new file mode 100644 index 00000000..4093e46d --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/SubscriptionFeedSnapshotStoreTest.kt @@ -0,0 +1,65 @@ +package dev.typetype.server + +import dev.typetype.server.SubscriptionFeedTestFixtures.video +import dev.typetype.server.cache.CacheJson +import dev.typetype.server.cache.CacheService +import dev.typetype.server.services.SubscriptionFeedCacheKeys +import dev.typetype.server.services.SubscriptionFeedSnapshot +import dev.typetype.server.services.SubscriptionFeedSnapshotStore +import kotlinx.coroutines.test.runTest +import kotlinx.serialization.builtins.ListSerializer +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNotNull +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test + +class SubscriptionFeedSnapshotStoreTest { + @Test + fun `published snapshots retain current and previous generations for one day`() = runTest { + val cache = RecordingCache() + val store = SubscriptionFeedSnapshotStore(cache) { 10_000L } + val first = SubscriptionFeedSnapshot(1, 1_000, stale = false, listOf(video(1000L))) + val second = SubscriptionFeedSnapshot(2, 2_000, stale = false, listOf(video(2000L))) + + store.publish("user", first) + store.publish("user", second) + + assertEquals(second, store.current("user")) + assertEquals(first, store.previous("user")) + assertTrue(cache.ttls.values.all { it == 86_400L }) + } + + @Test + fun `legacy list becomes a retained stale snapshot`() = runTest { + val cache = RecordingCache() + val raw = CacheJson.encodeToString( + ListSerializer(dev.typetype.server.models.VideoItem.serializer()), + listOf(video(1000L)), + ) + cache.set(SubscriptionFeedCacheKeys.feed("user"), raw, 60) + val store = SubscriptionFeedSnapshotStore(cache) { 10_000L } + + val migrated = store.current("user") + + assertNotNull(migrated) + assertTrue(requireNotNull(migrated).stale) + assertEquals(86_400L, cache.ttls[SubscriptionFeedCacheKeys.feed("user")]) + } + + private class RecordingCache : CacheService { + val values = mutableMapOf() + val ttls = mutableMapOf() + + override suspend fun get(key: String): String? = values[key] + + override suspend fun set(key: String, value: String, ttlSeconds: Long) { + values[key] = value + ttls[key] = ttlSeconds + } + + override suspend fun delete(key: String) { + values.remove(key) + ttls.remove(key) + } + } +} diff --git a/src/test/kotlin/dev/typetype/server/TypetypeTokenYoutubeSessionPoTokenProviderTest.kt b/src/test/kotlin/dev/typetype/server/TypetypeTokenYoutubeSessionPoTokenProviderTest.kt new file mode 100644 index 00000000..b97ce165 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/TypetypeTokenYoutubeSessionPoTokenProviderTest.kt @@ -0,0 +1,77 @@ +package dev.typetype.server + +import dev.typetype.server.services.TypetypeTokenYoutubeSessionPoTokenProvider +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.Test +import org.schabi.newpipe.extractor.ServiceList +import org.schabi.newpipe.extractor.localization.ContentCountry +import org.schabi.newpipe.extractor.localization.Localization + +class TypetypeTokenYoutubeSessionPoTokenProviderTest { + private val localization = Localization("en", "US") + private val country = ContentCountry("US") + + @AfterEach + fun clearCredentials(): Unit = ServiceList.YouTube.setTokens("") + + @Test + fun `does not mint a session token for anonymous extraction`() { + var calls = 0 + val provider = provider( + boundTokenFetcher = { calls += 1; "unused" }, + visitorDataFetcher = { _, _ -> calls += 1; "unused" }, + ) + + val result = provider.getSessionPoToken("WEB", localization, country, false) + + assertNull(result) + assertEquals(0, calls) + } + + @Test + fun `binds and caches the player token to authenticated visitor data`() { + ServiceList.YouTube.setTokens("SID=one; SAPISID=one") + var visitorCalls = 0 + val bindings = mutableListOf() + val provider = provider( + boundTokenFetcher = { bindings += it; "token-for-$it" }, + visitorDataFetcher = { _, _ -> visitorCalls += 1; "visitor-one" }, + ) + + val first = provider.getSessionPoToken("TV", localization, country, true) + val second = provider.getSessionPoToken("WEB", localization, country, true) + + assertEquals("visitor-one", first?.visitorData) + assertEquals("token-for-visitor-one", first?.poToken) + assertEquals(first?.poToken, second?.poToken) + assertEquals(listOf("visitor-one"), bindings) + assertEquals(1, visitorCalls) + } + + @Test + fun `invalidates cached token when credentials change`() { + var index = 0 + val provider = provider( + boundTokenFetcher = { "token-for-$it" }, + visitorDataFetcher = { _, _ -> "visitor-${++index}" }, + ) + ServiceList.YouTube.setTokens("SID=one; SAPISID=one") + val first = provider.getSessionPoToken("WEB", localization, country, true) + ServiceList.YouTube.setTokens("SID=two; SAPISID=two") + + val second = provider.getSessionPoToken("WEB", localization, country, true) + + assertEquals("visitor-1", first?.visitorData) + assertEquals("visitor-2", second?.visitorData) + } + + private fun provider( + boundTokenFetcher: (String) -> String?, + visitorDataFetcher: (Localization, ContentCountry) -> String, + ): TypetypeTokenYoutubeSessionPoTokenProvider = TypetypeTokenYoutubeSessionPoTokenProvider( + boundTokenFetcher, + visitorDataFetcher, + ) +} diff --git a/src/test/kotlin/dev/typetype/server/YoutubeAuthenticatedExtractionProbeTest.kt b/src/test/kotlin/dev/typetype/server/YoutubeAuthenticatedExtractionProbeTest.kt new file mode 100644 index 00000000..ffe357b0 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/YoutubeAuthenticatedExtractionProbeTest.kt @@ -0,0 +1,120 @@ +package dev.typetype.server + +import dev.typetype.server.cache.CacheService +import dev.typetype.server.models.ExtractionFailureKind +import dev.typetype.server.models.ExtractionResult +import dev.typetype.server.services.BilibiliRelatedService +import dev.typetype.server.services.NewPipeInitializer +import dev.typetype.server.services.PipePipeStreamService +import dev.typetype.server.services.TypetypeTokenSabrPoTokenProvider +import dev.typetype.server.services.TypetypeTokenSabrTokenClient +import dev.typetype.server.services.TypetypeTokenYoutubeSessionClient +import dev.typetype.server.services.YouTubeSubtitleService +import dev.typetype.server.services.YoutubePlayerClient +import dev.typetype.server.services.YoutubePlayerClientFallbackStreamService +import dev.typetype.server.services.YoutubeSessionCookieNormalizer +import dev.typetype.server.services.YoutubeSessionCredentials +import dev.typetype.server.services.YoutubeSessionTokenScope +import kotlinx.coroutines.runBlocking +import okhttp3.OkHttpClient +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Tag +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable +import org.schabi.newpipe.extractor.NewPipe +import org.schabi.newpipe.extractor.ServiceList +import org.schabi.newpipe.extractor.localization.ContentCountry +import org.schabi.newpipe.extractor.localization.Localization +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrSession +import java.nio.file.Files +import java.nio.file.Path + +@Tag("network") +@EnabledIfEnvironmentVariable(named = "YOUTUBE_AUTH_COOKIES_FILE", matches = ".+") +class YoutubeAuthenticatedExtractionProbeTest { + private val tokenServiceUrl = System.getenv("YOUTUBE_TOKEN_SERVICE_URL") ?: "http://127.0.0.1:8081" + private val videoId = System.getenv("YOUTUBE_PROBE_VIDEO_ID") ?: "dQw4w9WgXcQ" + private val expectSessionRejection = System.getenv("YOUTUBE_EXPECT_SESSION_REJECTION").toBoolean() + private val videoUrl = "https://www.youtube.com/watch?v=$videoId" + private val localization = Localization("en", "US") + private val contentCountry = ContentCountry("US") + + @Test + fun `authenticated classic extraction receives a session bound player token`() = runBlocking { + NewPipeInitializer.init(tokenServiceUrl) + val cookies = readCookies() + val credentials = YoutubeSessionCredentials("probe", "probe", cookies, "probe-token") + val pipePipe = PipePipeStreamService( + ProbeCache, + YouTubeSubtitleService(OkHttpClient(), tokenServiceUrl), + BilibiliRelatedService(), + ) + val service = YoutubePlayerClientFallbackStreamService( + pipePipe, + listOf(YoutubePlayerClient.TV_DOWNGRADED, YoutubePlayerClient.WEB_SAFARI), + ) + + val result = YoutubeSessionTokenScope.withCredentials(credentials) { + val token = NewPipe.getYoutubeSessionPoTokenProvider()?.getSessionPoToken( + "TV", + localization, + contentCountry, + true, + ) + assertTrue(token?.visitorData?.isNotBlank() == true) + assertTrue(token?.poToken?.isNotBlank() == true) + service.getStreamInfo(videoUrl) + } + + if (expectSessionRejection) { + assertTrue(result is ExtractionResult.Failure, result.toString()) + assertEquals( + ExtractionFailureKind.YoutubeSessionRejected, + (result as ExtractionResult.Failure).kind, + ) + return@runBlocking + } + assertTrue(result is ExtractionResult.Success, result.toString()) + val stream = (result as ExtractionResult.Success).data + assertTrue(stream.videoStreams.isNotEmpty() || stream.videoOnlyStreams.isNotEmpty() || stream.hlsUrl.isNotBlank()) + assertFalse(ServiceList.YouTube.hasTokens()) + } + + @Test + fun `anonymous SABR session remains independent from connected credentials`() = runBlocking { + NewPipeInitializer.init(tokenServiceUrl) + val tokenClient = TypetypeTokenSabrTokenClient(tokenServiceUrl) + val playback = YoutubeSessionTokenScope.withoutCredentials { + assertFalse(ServiceList.YouTube.hasTokens()) + TypetypeTokenYoutubeSessionClient(tokenServiceUrl).fetchPlaybackSession(videoId) + } ?: error("Anonymous SABR bootstrap failed") + assertFalse(ServiceList.YouTube.hasTokens()) + val token = playback.token ?: error("Anonymous SABR token is missing") + val audio = playback.info.findBestAudioFormat() ?: error("SABR audio format is missing") + val video = playback.info.findLowestVideoFormat() ?: error("SABR video format is missing") + val session = YoutubeSabrSession( + playback.info, + audio, + video, + TypetypeTokenSabrPoTokenProvider(tokenClient, token), + ) + + val segments = session.pumpOnce(localization) + + assertTrue(segments.isNotEmpty()) + assertFalse(ServiceList.YouTube.hasTokens()) + } + + private fun readCookies(): String { + val raw = Files.readString(Path.of(System.getenv("YOUTUBE_AUTH_COOKIES_FILE"))) + return YoutubeSessionCookieNormalizer.normalize(raw) ?: error("Cookie export is invalid") + } +} + +private object ProbeCache : CacheService { + override suspend fun get(key: String): String? = null + override suspend fun set(key: String, value: String, ttlSeconds: Long): Unit = Unit + override suspend fun delete(key: String): Unit = Unit +} diff --git a/src/test/kotlin/dev/typetype/server/YoutubeSessionCredentialValidatorTest.kt b/src/test/kotlin/dev/typetype/server/YoutubeSessionCredentialValidatorTest.kt index 08ab4ed3..e345bbc5 100644 --- a/src/test/kotlin/dev/typetype/server/YoutubeSessionCredentialValidatorTest.kt +++ b/src/test/kotlin/dev/typetype/server/YoutubeSessionCredentialValidatorTest.kt @@ -9,8 +9,14 @@ class YoutubeSessionCredentialValidatorTest { @Test fun `validates YouTube session cookie and po token shape`() { assertTrue(YoutubeSessionCredentialValidator.isValid("SID=abc; SAPISID=def", "po-token-value")) - assertTrue(YoutubeSessionCredentialValidator.isValid("__Secure-3PSID=abc", "po-token-value")) + assertTrue( + YoutubeSessionCredentialValidator.isValid( + "__Secure-3PSID=abc; __Secure-3PAPISID=def", + "po-token-value", + ), + ) assertFalse(YoutubeSessionCredentialValidator.isValid("SAPISID=def", "po-token-value")) + assertFalse(YoutubeSessionCredentialValidator.isValid("SID=abc", "po-token-value")) assertFalse(YoutubeSessionCredentialValidator.isValid("SID=abc", "short")) } } diff --git a/src/test/kotlin/dev/typetype/server/YoutubeSessionHlsManifestServiceTest.kt b/src/test/kotlin/dev/typetype/server/YoutubeSessionHlsManifestServiceTest.kt index 24a623ca..382c838a 100644 --- a/src/test/kotlin/dev/typetype/server/YoutubeSessionHlsManifestServiceTest.kt +++ b/src/test/kotlin/dev/typetype/server/YoutubeSessionHlsManifestServiceTest.kt @@ -66,7 +66,11 @@ class YoutubeSessionHlsManifestServiceTest { private suspend fun connectYoutubeSession(): Unit { val pairing = youtubeSessionService.createPairing(TEST_USER_ID) val result = youtubeSessionService.complete( - YoutubeSessionCompleteRequest(pairing.code, cookies = "SID=secret-cookie", poToken = "secret-pot-value") + YoutubeSessionCompleteRequest( + pairing.code, + cookies = "SID=secret-cookie; SAPISID=secret-sapisid", + poToken = "secret-pot-value", + ) ) assertEquals(YoutubeSessionCompleteResult.Completed, result) } diff --git a/src/test/kotlin/dev/typetype/server/YoutubeSessionRoutesTest.kt b/src/test/kotlin/dev/typetype/server/YoutubeSessionRoutesTest.kt index 30ce45aa..0f679437 100644 --- a/src/test/kotlin/dev/typetype/server/YoutubeSessionRoutesTest.kt +++ b/src/test/kotlin/dev/typetype/server/YoutubeSessionRoutesTest.kt @@ -69,7 +69,9 @@ class YoutubeSessionRoutesTest { private suspend fun ApplicationTestBuilder.completeSession(code: String) = client.post("/youtube-session/complete") { headers.append(HttpHeaders.ContentType, ContentType.Application.Json.toString()) - setBody("""{"code":"$code","cookies":"SID=secret-cookie","poToken":"secret-pot-value"}""") + setBody( + """{"code":"$code","cookies":"SID=secret-cookie; SAPISID=secret-sapisid","poToken":"secret-pot-value"}""", + ) } @Test diff --git a/src/test/kotlin/dev/typetype/server/YoutubeSessionStreamServiceTest.kt b/src/test/kotlin/dev/typetype/server/YoutubeSessionStreamServiceTest.kt index e33bea67..bcaa17d8 100644 --- a/src/test/kotlin/dev/typetype/server/YoutubeSessionStreamServiceTest.kt +++ b/src/test/kotlin/dev/typetype/server/YoutubeSessionStreamServiceTest.kt @@ -1,6 +1,7 @@ package dev.typetype.server import dev.typetype.server.models.ExtractionResult +import dev.typetype.server.models.ExtractionFailureKind import dev.typetype.server.models.StreamResponse import dev.typetype.server.models.YoutubeSessionCompleteRequest import dev.typetype.server.services.SignedHlsManifestTokenResult @@ -43,7 +44,7 @@ class YoutubeSessionStreamServiceTest { } @Test - fun `rejected authenticated extraction marks session needs reconnect`() = runBlocking { + fun `generic authenticated extraction failure keeps session connected`() = runBlocking { connectYoutubeSession() val service = YoutubeSessionStreamService( failingStreamService("No suitable stream"), @@ -52,6 +53,22 @@ class YoutubeSessionStreamServiceTest { tokenService, ) val result = service.getStreamInfo(TEST_USER_ID, "https://youtube.com/watch?v=test") + assertTrue(result is ExtractionResult.Failure) + assertEquals("connected", youtubeSessionService.status(TEST_USER_ID).status) + } + + @Test + fun `explicit session rejection marks session needs reconnect`() = runBlocking { + connectYoutubeSession() + val service = testService( + failingStreamService( + "rejected", + ExtractionFailureKind.YoutubeSessionRejected, + ), + ) + + val result = service.getStreamInfo(TEST_USER_ID, "https://youtube.com/watch?v=test") + assertTrue(result is ExtractionResult.BadRequest) assertEquals("needs_reconnect", youtubeSessionService.status(TEST_USER_ID).status) } @@ -98,16 +115,19 @@ class YoutubeSessionStreamServiceTest { val result = youtubeSessionService.complete( YoutubeSessionCompleteRequest( code = pairing.code, - cookies = "SID=secret-cookie", + cookies = "SID=secret-cookie; SAPISID=secret-sapisid", poToken = "secret-pot-value", ) ) assertEquals(YoutubeSessionCompleteResult.Completed, result) } - private fun failingStreamService(message: String = "unexpected call"): StreamService = object : StreamService { + private fun failingStreamService( + message: String = "unexpected call", + kind: ExtractionFailureKind = ExtractionFailureKind.Unknown, + ): StreamService = object : StreamService { override suspend fun getStreamInfo(url: String): ExtractionResult = - ExtractionResult.Failure(message) + ExtractionResult.Failure(message, kind = kind) } private fun testService(stream: StreamService): YoutubeSessionStreamService = diff --git a/src/test/kotlin/dev/typetype/server/routes/AndroidPlaybackRouteContractTest.kt b/src/test/kotlin/dev/typetype/server/routes/AndroidPlaybackRouteContractTest.kt new file mode 100644 index 00000000..27907fec --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/routes/AndroidPlaybackRouteContractTest.kt @@ -0,0 +1,179 @@ +package dev.typetype.server.routes + +import dev.typetype.server.services.AndroidDashManifestResult +import dev.typetype.server.services.AndroidPlaybackMediaResult +import dev.typetype.server.services.AndroidPlaybackService +import dev.typetype.server.services.AndroidPlaybackSession +import dev.typetype.server.services.AndroidPlaybackSessionLookup +import dev.typetype.server.services.AndroidSubtitleService +import dev.typetype.server.services.SabrSessionHolder +import io.ktor.client.request.get +import io.ktor.client.statement.bodyAsText +import io.ktor.http.ContentType +import io.ktor.http.HttpHeaders +import io.ktor.http.HttpStatusCode +import io.ktor.http.contentType +import io.ktor.serialization.kotlinx.json.json +import io.ktor.server.application.install +import io.ktor.server.plugins.contentnegotiation.ContentNegotiation +import io.ktor.server.routing.get +import io.ktor.server.routing.routing +import io.ktor.server.testing.testApplication +import io.mockk.coEvery +import io.mockk.every +import io.mockk.mockk +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat + +class AndroidPlaybackRouteContractTest { + @Test + fun `dedicated Android manifest path does not use the web SABR namespace`() = testApplication { + application { + install(ContentNegotiation) { json() } + routing { + androidPlaybackRoutes( + mockk(relaxed = true), + mockk(), + mockk(), + null, + null, + null, + ) + } + } + + val response = client.get("/android/youtube/playback/unknown/manifest.mpd") + + assertEquals(HttpStatusCode.NotFound, response.status) + assertEquals("no-store", response.headers[HttpHeaders.CacheControl]) + } + + @Test + fun `ready manifest returns dash xml with no store`() = testApplication { + val fixture = fixture() + every { fixture.service.lookup(SESSION_ID) } returns fixture.lookup + coEvery { fixture.service.manifest(fixture.holder) } returns AndroidDashManifestResult.Ready(MPD, 1_000L) + application { + install(ContentNegotiation) { json() } + routing { get("/manifest") { fixture.handler.manifest(call, SESSION_ID) } } + } + + val response = client.get("/manifest") + + assertEquals(HttpStatusCode.OK, response.status) + assertEquals("no-store", response.headers[HttpHeaders.CacheControl]) + assertEquals(ContentType.parse("application/dash+xml"), response.contentType()?.withoutParameters()) + assertEquals(MPD, response.bodyAsText()) + } + + @Test + fun `preparing manifest returns bounded json retry`() = testApplication { + val fixture = fixture() + every { fixture.service.lookup(SESSION_ID) } returns fixture.lookup + coEvery { fixture.service.manifest(fixture.holder) } returns AndroidDashManifestResult.Preparing + application { + install(ContentNegotiation) { json() } + routing { get("/manifest") { fixture.handler.manifest(call, SESSION_ID) } } + } + + val response = client.get("/manifest") + + assertEquals(HttpStatusCode.Accepted, response.status) + assertTrue(response.bodyAsText().contains("\"retryAfterMs\":500")) + } + + @Test + fun `unknown and expired sessions are distinguished`() = testApplication { + val fixture = fixture() + every { fixture.service.lookup("unknown") } returns AndroidPlaybackSessionLookup.Unknown + every { fixture.service.lookup("expired") } returns AndroidPlaybackSessionLookup.Expired + application { + install(ContentNegotiation) { json() } + routing { + get("/unknown") { fixture.handler.manifest(call, "unknown") } + get("/expired") { fixture.handler.manifest(call, "expired") } + } + } + + assertEquals(HttpStatusCode.NotFound, client.get("/unknown").status) + assertEquals(HttpStatusCode.Gone, client.get("/expired").status) + } + + @Test + fun `stale media generation returns conflict`() = testApplication { + val fixture = fixture() + every { fixture.service.lookup(SESSION_ID) } returns fixture.lookup + coEvery { fixture.service.segment(fixture.holder, 137, 1, 0L) } returns AndroidPlaybackMediaResult.StaleGeneration + val media = AndroidPlaybackMediaHandler(fixture.service) + application { + install(ContentNegotiation) { json() } + routing { + get("/segment") { media.segment(call, SESSION_ID, 137, 1) } + } + } + + val response = client.get("/segment?session=$SESSION_ID&generation=0") + + assertEquals(HttpStatusCode.Conflict, response.status) + assertTrue(response.bodyAsText().contains("android_playback_stale_generation")) + } + + @Test + fun `invalid complete index returns unprocessable entity`() = testApplication { + val fixture = fixture() + every { fixture.service.lookup(SESSION_ID) } returns fixture.lookup + coEvery { fixture.service.manifest(fixture.holder) } returns AndroidDashManifestResult.Invalid("invalid index") + application { + install(ContentNegotiation) { json() } + routing { get("/manifest") { fixture.handler.manifest(call, SESSION_ID) } } + } + + val response = client.get("/manifest") + + assertEquals(HttpStatusCode.UnprocessableEntity, response.status) + assertTrue(response.bodyAsText().contains("android_playback_invalid_index")) + } + + private fun fixture(): Fixture { + val service = mockk() + val audio = format(140, true) + val video = format(137, false) + val holder = mockk { + every { sessionToken } returns SESSION_ID + every { key.videoId } returns "video" + every { audioFormat } returns audio + every { videoFormat } returns video + every { activeGeneration() } returns 0L + } + val handler = AndroidPlaybackHandler( + mockk(), + mockk(), + null, + null, + null, + mockk(), + service, + ) + return Fixture(service, holder, handler, AndroidPlaybackSessionLookup.Active(AndroidPlaybackSession(holder, emptyList()))) + } + + private fun format(itag: Int, audio: Boolean): YoutubeSabrFormat = mockk { + every { this@mockk.itag } returns itag + every { audioTrackId } returns null + every { isAudio } returns audio + } + + private data class Fixture( + val service: AndroidPlaybackService, + val holder: SabrSessionHolder, + val handler: AndroidPlaybackHandler, + val lookup: AndroidPlaybackSessionLookup.Active, + ) + + private companion object { + const val SESSION_ID = "android-session" + const val MPD = "" + } +} diff --git a/src/test/kotlin/dev/typetype/server/routes/AndroidPlaybackSubtitleModelsTest.kt b/src/test/kotlin/dev/typetype/server/routes/AndroidPlaybackSubtitleModelsTest.kt new file mode 100644 index 00000000..3860d7d4 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/routes/AndroidPlaybackSubtitleModelsTest.kt @@ -0,0 +1,53 @@ +package dev.typetype.server.routes + +import dev.typetype.server.services.AndroidDashManifestResult +import dev.typetype.server.services.AndroidSubtitleTrack +import dev.typetype.server.services.SabrSessionHolder +import io.mockk.every +import io.mockk.mockk +import okhttp3.HttpUrl.Companion.toHttpUrl +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat + +class AndroidPlaybackSubtitleModelsTest { + @Test + fun `subtitle resources stay stable when the playback generation changes`() { + var generation = 0L + val holder = mockk { + every { sessionToken } returns "session" + every { key.videoId } returns "video" + every { audioFormat } returns format(140) + every { videoFormat } returns format(137) + every { activeGeneration() } answers { generation } + } + val track = AndroidSubtitleTrack( + "track-id", + "en", + "English", + false, + "https://www.youtube.com/api/timedtext?v=video&lang=en".toHttpUrl(), + ) + + val initial = holder.toAndroidPlaybackResponse(MANIFEST, listOf(track)) + generation = 1L + val afterSeek = holder.toAndroidPlaybackResponse(MANIFEST, listOf(track)) + + assertEquals(0L, initial.generation) + assertEquals(1L, afterSeek.generation) + assertEquals(initial.subtitles, afterSeek.subtitles) + assertEquals( + "/api/android/youtube/playback/session/subtitles/track-id.vtt", + initial.subtitles.single().url, + ) + } + + private fun format(itag: Int): YoutubeSabrFormat = mockk { + every { this@mockk.itag } returns itag + every { audioTrackId } returns null + } + + private companion object { + val MANIFEST = AndroidDashManifestResult.Ready("", 1_000L) + } +} diff --git a/src/test/kotlin/dev/typetype/server/routes/AndroidSubtitleRouteTest.kt b/src/test/kotlin/dev/typetype/server/routes/AndroidSubtitleRouteTest.kt new file mode 100644 index 00000000..684d07c6 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/routes/AndroidSubtitleRouteTest.kt @@ -0,0 +1,122 @@ +package dev.typetype.server.routes + +import dev.typetype.server.services.AndroidPlaybackService +import dev.typetype.server.services.AndroidPlaybackSession +import dev.typetype.server.services.AndroidPlaybackSessionLookup +import dev.typetype.server.services.AndroidSubtitleContentResult +import dev.typetype.server.services.AndroidSubtitleService +import dev.typetype.server.services.AndroidSubtitleTrack +import dev.typetype.server.services.AuthService +import dev.typetype.server.services.SabrSessionHolder +import dev.typetype.server.services.SabrSessionKey +import dev.typetype.server.services.SabrSessionPurpose +import io.ktor.client.request.bearerAuth +import io.ktor.client.request.get +import io.ktor.client.statement.bodyAsText +import io.ktor.http.ContentType +import io.ktor.http.HttpHeaders +import io.ktor.http.HttpStatusCode +import io.ktor.http.contentType +import io.ktor.serialization.kotlinx.json.json +import io.ktor.server.application.install +import io.ktor.server.plugins.contentnegotiation.ContentNegotiation +import io.ktor.server.routing.get +import io.ktor.server.routing.routing +import io.ktor.server.testing.testApplication +import io.mockk.coEvery +import io.mockk.coVerify +import io.mockk.every +import io.mockk.mockk +import okhttp3.HttpUrl.Companion.toHttpUrl +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test + +class AndroidSubtitleRouteTest { + @Test + fun `session owner receives complete WebVTT with no store`() = testApplication { + val fixture = fixture("user") + coEvery { fixture.subtitles.content(VIDEO_ID, TRACK) } returns AndroidSubtitleContentResult.Ready(VTT) + application { installRoute(fixture.handler) } + + val response = client.get("/subtitle") { bearerAuth("test-jwt") } + + assertEquals(HttpStatusCode.OK, response.status) + assertEquals(ContentType.parse("text/vtt"), response.contentType()?.withoutParameters()) + assertEquals("no-store", response.headers[HttpHeaders.CacheControl]) + assertEquals(VTT.decodeToString(), response.bodyAsText()) + } + + @Test + fun `another account cannot read a session subtitle`() = testApplication { + val fixture = fixture("owner", authenticatedUser = "other") + application { installRoute(fixture.handler) } + + val response = client.get("/subtitle") { bearerAuth("test-jwt") } + + assertEquals(HttpStatusCode.NotFound, response.status) + assertTrue(response.bodyAsText().contains("android_subtitle_not_found")) + coVerify(exactly = 0) { fixture.subtitles.content(any(), any()) } + } + + @Test + fun `bounded upstream failure returns typed service unavailable`() = testApplication { + val fixture = fixture("user") + coEvery { fixture.subtitles.content(VIDEO_ID, TRACK) } returns AndroidSubtitleContentResult.TemporaryFailure + application { installRoute(fixture.handler) } + + val response = client.get("/subtitle") { bearerAuth("test-jwt") } + + assertEquals(HttpStatusCode.ServiceUnavailable, response.status) + assertTrue(response.bodyAsText().contains("android_subtitle_upstream_unavailable")) + } + + private fun io.ktor.server.application.Application.installRoute(handler: AndroidSubtitleHandler) { + install(ContentNegotiation) { json() } + routing { get("/subtitle") { handler.content(call, SESSION_ID, TRACK.id) } } + } + + private fun fixture(owner: String, authenticatedUser: String = owner): Fixture { + val playback = mockk() + val holder = mockk { + every { key } returns SabrSessionKey( + VIDEO_ID, + owner, + 140, + null, + 137, + 0L, + SabrSessionPurpose.ANDROID_PLAYBACK, + ) + } + every { playback.lookup(SESSION_ID) } returns + AndroidPlaybackSessionLookup.Active(AndroidPlaybackSession(holder, listOf(TRACK))) + val subtitles = mockk() + val handler = AndroidSubtitleHandler( + playback, + subtitles, + AuthService.fixed(authenticatedUser), + null, + null, + ) + return Fixture(handler, subtitles) + } + + private data class Fixture( + val handler: AndroidSubtitleHandler, + val subtitles: AndroidSubtitleService, + ) + + private companion object { + const val SESSION_ID = "android-session" + const val VIDEO_ID = "dQw4w9WgXcQ" + val VTT = "WEBVTT\n\n00:00.000 --> 00:01.000\nHello\n".toByteArray() + val TRACK = AndroidSubtitleTrack( + "track", + "en", + "English", + false, + "https://www.youtube.com/api/timedtext?v=$VIDEO_ID&lang=en".toHttpUrl(), + ) + } +} diff --git a/src/test/kotlin/dev/typetype/server/routes/SabrPlaybackSeekRouteTest.kt b/src/test/kotlin/dev/typetype/server/routes/SabrPlaybackSeekRouteTest.kt new file mode 100644 index 00000000..11c5f0ef --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/routes/SabrPlaybackSeekRouteTest.kt @@ -0,0 +1,108 @@ +package dev.typetype.server.routes + +import dev.typetype.server.services.SabrSessionHolder +import dev.typetype.server.services.SabrSessionKey +import dev.typetype.server.services.SabrSessionPurpose +import dev.typetype.server.services.SabrSessionStore +import dev.typetype.server.services.StreamService +import io.ktor.client.request.post +import io.ktor.client.request.setBody +import io.ktor.http.ContentType +import io.ktor.http.HttpStatusCode +import io.ktor.http.contentType +import io.ktor.serialization.kotlinx.json.json +import io.ktor.server.application.call +import io.ktor.server.application.install +import io.ktor.server.plugins.contentnegotiation.ContentNegotiation +import io.ktor.server.routing.post +import io.ktor.server.routing.routing +import io.ktor.server.testing.ApplicationTestBuilder +import io.ktor.server.testing.testApplication +import io.mockk.coEvery +import io.mockk.coVerify +import io.mockk.every +import io.mockk.mockk +import io.mockk.verify +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrSession +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrStreamState +import java.time.Instant + +class SabrPlaybackSeekRouteTest { + @Test + fun `same formats seek repositions without extracting again`() = testApplication { + val store = mockk(relaxed = true) + val holder = holder() + every { store.lookupByToken("session-token") } returns holder + installApp(store) + + val response = client.post("/sabr/playback/session-token/seek") { + contentType(ContentType.Application.Json) + setBody("""{"playerTimeMs":90000,"videoItag":136,"audioItag":140}""") + } + + assertEquals(HttpStatusCode.Accepted, response.status) + assertEquals(1L, holder.activeGeneration()) + coVerify(exactly = 0) { store.fetchInfo(any(), any(), any()) } + verify(exactly = 1) { store.startPump(holder) } + } + + @Test + fun `format change keeps extraction path`() = testApplication { + val store = mockk(relaxed = true) + every { store.lookupByToken("session-token") } returns holder() + coEvery { store.fetchInfo("video", 90_000L, cachedFirst = true) } returns null + installApp(store) + + val response = client.post("/sabr/playback/session-token/seek") { + contentType(ContentType.Application.Json) + setBody("""{"playerTimeMs":90000,"videoItag":137,"audioItag":140}""") + } + + assertEquals(HttpStatusCode.UnprocessableEntity, response.status) + coVerify(exactly = 1) { store.fetchInfo("video", 90_000L, cachedFirst = true) } + } + + private fun ApplicationTestBuilder.installApp(store: SabrSessionStore): Unit = application { + install(ContentNegotiation) { json() } + val handler = SabrPlaybackHandler(store, mockk(), null, null, null) + routing { + post("/sabr/playback/{sessionId}/seek") { + handler.seek(call, call.parameters["sessionId"].orEmpty()) + } + } + } + + private fun holder(): SabrSessionHolder { + val audio = format(140, true) + val video = format(136, false) + val session = mockk(relaxed = true) + val state = mockk(relaxed = true) + every { session.streamState } returns state + every { session.getCachedSegment(any()) } returns null + every { session.isBeyondEnd(any()) } returns false + every { state.getSegmentNumberAtOrAfterTimeMs(any(), 90_000L) } returns 10 + every { state.getSegmentStartMs(any(), 10) } returns 90_000L + every { state.getMinBufferedEndMs() } returns 10_000L + return SabrSessionHolder( + session = session, + info = mockk(), + audioFormat = audio, + videoFormat = video, + sessionToken = "session-token", + key = SabrSessionKey("video", "user", 140, null, 136, 0L, SabrSessionPurpose.PLAYBACK), + lastRequestAt = Instant.EPOCH, + ) + } + + private fun format(itag: Int, isAudio: Boolean): YoutubeSabrFormat = + mockk(relaxed = true) { + every { this@mockk.itag } returns itag + every { this@mockk.isAudio } returns isAudio + every { audioTrackId } returns null + every { mimeType } returns if (isAudio) "audio/mp4" else "video/mp4" + } +} diff --git a/src/test/kotlin/dev/typetype/server/services/AndroidDashManifestBuilderTest.kt b/src/test/kotlin/dev/typetype/server/services/AndroidDashManifestBuilderTest.kt new file mode 100644 index 00000000..c71257c5 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/services/AndroidDashManifestBuilderTest.kt @@ -0,0 +1,96 @@ +package dev.typetype.server.services + +import io.mockk.every +import io.mockk.mockk +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat +import java.io.ByteArrayInputStream +import javax.xml.parsers.DocumentBuilderFactory + +class AndroidDashManifestBuilderTest { + @Test + fun `five minute vod describes the complete presentation from zero`() { + val manifest = manifest( + audioSegments = timeline(300, 1_000L), + videoSegments = timeline(150, 2_000L), + durationMs = 300_000L, + ) + + val document = parse(manifest) + val mpd = document.documentElement + assertEquals("static", mpd.getAttribute("type")) + assertEquals("PT300.000S", mpd.getAttribute("mediaPresentationDuration")) + assertEquals("PT0S", document.getElementsByTagNameNS(DASH_NAMESPACE, "Period").item(0).attributes + .getNamedItem("start").nodeValue) + assertEquals(2, document.getElementsByTagNameNS(DASH_NAMESPACE, "SegmentTimeline").length) + } + + @Test + fun `manifest resources contain the session and generation with valid XML escaping`() { + val manifest = manifest(timeline(3, 1_000L), timeline(3, 1_000L), 3_000L, generation = 7L) + + assertTrue(manifest.contains("session=session-token&generation=7")) + val template = parse(manifest).getElementsByTagNameNS(DASH_NAMESPACE, "SegmentTemplate").item(0) + assertEquals( + "/api/android/youtube/playback/session-token/137/segment/\$Number\$?session=session-token&generation=7", + template.attributes.getNamedItem("media").nodeValue, + ) + } + + @Test + fun `audio and video may have different exact segment counts`() { + val manifest = manifest(timeline(6, 500L), timeline(2, 1_500L), 3_000L) + val document = parse(manifest) + val templates = document.getElementsByTagNameNS(DASH_NAMESPACE, "SegmentTemplate") + + assertEquals("1", templates.item(0).attributes.getNamedItem("startNumber").nodeValue) + assertEquals("1", templates.item(1).attributes.getNamedItem("startNumber").nodeValue) + assertEquals(2, document.getElementsByTagNameNS(DASH_NAMESPACE, "S").length) + } + + @Test + fun `long vod manifest remains compact`() { + val manifest = manifest(timeline(100_000, 1_000L), timeline(100_000, 1_000L), 100_000_000L) + + assertTrue(manifest.toByteArray().size < 2 * 1024 * 1024) + assertFalse(manifest.contains("SegmentURL")) + assertEquals(2, parse(manifest).getElementsByTagNameNS(DASH_NAMESPACE, "S").length) + } + + private fun manifest( + audioSegments: List, + videoSegments: List, + durationMs: Long, + generation: Long = 0L, + ): String = AndroidDashManifestBuilder.build( + sessionId = "session-token", + generation = generation, + audio = AndroidDashTrack(format(140, audio = true), AndroidDashTimeline(1, audioSegments)), + video = AndroidDashTrack(format(137, audio = false), AndroidDashTimeline(1, videoSegments)), + durationMs = durationMs, + ) + + private fun timeline(count: Int, durationMs: Long): List = + List(count) { index -> AndroidDashTimelineSegment(index * durationMs, durationMs) } + + private fun format(itag: Int, audio: Boolean): YoutubeSabrFormat = mockk { + every { this@mockk.itag } returns itag + every { isAudio } returns audio + every { isVideo } returns !audio + every { mimeType } returns if (audio) "audio/mp4; codecs=\"mp4a.40.2\"" else "video/mp4; codecs=\"avc1.640028\"" + every { bitrate } returns if (audio) 128_000 else 2_000_000 + every { width } returns if (audio) 0 else 1920 + every { height } returns if (audio) 0 else 1080 + } + + private fun parse(manifest: String) = DocumentBuilderFactory.newInstance().apply { + isNamespaceAware = true + }.newDocumentBuilder().parse(ByteArrayInputStream(manifest.toByteArray())) + + private companion object { + const val DASH_NAMESPACE = "urn:mpeg:dash:schema:mpd:2011" + } +} diff --git a/src/test/kotlin/dev/typetype/server/services/AndroidDashManifestServiceTest.kt b/src/test/kotlin/dev/typetype/server/services/AndroidDashManifestServiceTest.kt new file mode 100644 index 00000000..deed2d3a --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/services/AndroidDashManifestServiceTest.kt @@ -0,0 +1,101 @@ +package dev.typetype.server.services + +import io.mockk.every +import io.mockk.mockk +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertInstanceOf +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrSession +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrStreamState + +class AndroidDashManifestServiceTest { + @Test + fun `complete exact indexes produce a full static vod manifest`() { + val fixture = fixture(audioCount = 150, videoCount = 150, segmentMs = 2_000L) + + val result = AndroidDashManifestService().build(fixture.holder) as AndroidDashManifestResult.Ready + + assertEquals(300_000L, result.durationMs) + assertTrue(result.manifest.contains("mediaPresentationDuration=\"PT300.000S\"")) + assertTrue(result.manifest.contains("() + every { state.hasSegmentIndex(audio) } returns true + every { state.hasSegmentIndex(video) } returns true + every { state.getEndSegment(audio) } returns audioCount.toLong() + every { state.getEndSegment(video) } returns videoCount.toLong() + every { state.getSegmentStartMs(any(), any()) } answers { (secondArg() - 1L) * segmentMs } + every { state.getSegmentEndMs(any(), any()) } answers { secondArg() * segmentMs } + val session = mockk { every { streamState } returns state } + val holder = mockk { + every { this@mockk.session } returns session + every { audioFormat } returns audio + every { videoFormat } returns video + every { sessionToken } returns "session-token" + every { activeGeneration() } returns 0L + every { expectsLive() } returns false + } + return Fixture(holder, state, audio) + } + + private fun format(itag: Int, audio: Boolean): YoutubeSabrFormat = mockk { + every { this@mockk.itag } returns itag + every { isAudio } returns audio + every { isVideo } returns !audio + every { mimeType } returns if (audio) "audio/mp4; codecs=\"mp4a.40.2\"" else "video/mp4; codecs=\"avc1.640028\"" + every { bitrate } returns if (audio) 128_000 else 2_000_000 + every { width } returns if (audio) 0 else 1920 + every { height } returns if (audio) 0 else 1080 + } + + private data class Fixture( + val holder: SabrSessionHolder, + val state: YoutubeSabrStreamState, + val audio: YoutubeSabrFormat, + ) +} diff --git a/src/test/kotlin/dev/typetype/server/services/AndroidDashTimelineReaderTest.kt b/src/test/kotlin/dev/typetype/server/services/AndroidDashTimelineReaderTest.kt new file mode 100644 index 00000000..1f17b316 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/services/AndroidDashTimelineReaderTest.kt @@ -0,0 +1,66 @@ +package dev.typetype.server.services + +import io.mockk.every +import io.mockk.mockk +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertInstanceOf +import org.junit.jupiter.api.Test +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrStreamState + +class AndroidDashTimelineReaderTest { + private val format = mockk { + every { itag } returns 137 + } + + @Test + fun `unknown exact index stays preparing`() { + val state = mockk { + every { hasSegmentIndex(format) } returns false + } + + assertEquals(AndroidDashTimelineResult.Pending, AndroidDashTimelineReader.read(state, format)) + } + + @Test + fun `exact index preserves complete segment timing`() { + val state = indexedState(count = 3, durationMs = 2_000L) + + val ready = AndroidDashTimelineReader.read(state, format) as AndroidDashTimelineResult.Ready + + assertEquals(1, ready.timeline.startNumber) + assertEquals(3, ready.timeline.segments.size) + assertEquals(0L, ready.timeline.segments.first().startMs) + assertEquals(6_000L, ready.timeline.endMs) + } + + @Test + fun `timeline beginning after zero is rejected`() { + val state = indexedState(count = 2, durationMs = 1_000L, offsetMs = 500L) + + assertInstanceOf( + AndroidDashTimelineResult.Invalid::class.java, + AndroidDashTimelineReader.read(state, format), + ) + } + + @Test + fun `unbounded exact segment count is rejected`() { + val state = mockk { + every { hasSegmentIndex(format) } returns true + every { getEndSegment(format) } returns 100_001L + } + + assertInstanceOf( + AndroidDashTimelineResult.Invalid::class.java, + AndroidDashTimelineReader.read(state, format), + ) + } + + private fun indexedState(count: Int, durationMs: Long, offsetMs: Long = 0L): YoutubeSabrStreamState = mockk { + every { hasSegmentIndex(format) } returns true + every { getEndSegment(format) } returns count.toLong() + every { getSegmentStartMs(format, any()) } answers { offsetMs + (secondArg() - 1L) * durationMs } + every { getSegmentEndMs(format, any()) } answers { offsetMs + secondArg() * durationMs } + } +} diff --git a/src/test/kotlin/dev/typetype/server/services/AndroidPlaybackServiceTest.kt b/src/test/kotlin/dev/typetype/server/services/AndroidPlaybackServiceTest.kt new file mode 100644 index 00000000..6077f322 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/services/AndroidPlaybackServiceTest.kt @@ -0,0 +1,112 @@ +package dev.typetype.server.services + +import io.mockk.coVerify +import io.mockk.every +import io.mockk.mockk +import io.mockk.verify +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertSame +import org.junit.jupiter.api.Test +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrSession +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrStreamState +import java.time.Instant + +class AndroidPlaybackServiceTest { + @Test + fun `seek keeps session and complete presentation while advancing generation`() { + val audio = format(140, audio = true) + val video = format(137, audio = false) + val state = state(audio, video) + val holder = holder(audio, video, state) + val store = mockk { + every { startPump(holder) } returns Unit + } + val service = AndroidPlaybackService(store, mockk(relaxed = true)) + + val result = service.seek(holder, generation = 0L, playerTimeMs = 45_000L) as AndroidPlaybackSeekResult.Ready + + assertSame(holder, result.holder) + assertEquals(1L, holder.activeGeneration()) + val manifest = result.manifest as AndroidDashManifestResult.Ready + assertEquals(60_000L, manifest.durationMs) + assertEquals(true, manifest.manifest.contains("(relaxed = true) + val service = AndroidPlaybackService(store, mockk(relaxed = true)) + val prepared = SabrPreparedInfo(mockk(), null, isLive = true) + + val result = service.create( + "video", + "user", + prepared, + format(140, true), + format(137, false), + emptyList(), + ) + + assertEquals(AndroidPlaybackCreateResult.UnsupportedLive, result) + coVerify(exactly = 0) { store.fetchInitializationData(any(), any()) } + verify(exactly = 0) { store.getOrCreate(any(), any(), any(), any(), any(), any(), any(), any(), any(), any()) } + } + + private fun holder( + audio: YoutubeSabrFormat, + video: YoutubeSabrFormat, + state: YoutubeSabrStreamState, + ): SabrSessionHolder { + val session = mockk(relaxed = true) { + every { streamState } returns state + every { getCachedSegment(any()) } returns null + } + return SabrSessionHolder( + session, + mockk(), + audio, + video, + "android-session", + SabrSessionKey("video", "user", 140, null, 137, 0L, SabrSessionPurpose.ANDROID_PLAYBACK), + Instant.EPOCH, + ) + } + + private fun state(audio: YoutubeSabrFormat, video: YoutubeSabrFormat): YoutubeSabrStreamState = mockk(relaxed = true) { + every { hasSegmentIndex(audio) } returns true + every { hasSegmentIndex(video) } returns true + every { getEndSegment(any()) } returns 60L + every { getSegmentStartMs(any(), any()) } answers { (secondArg() - 1L) * 1_000L } + every { getSegmentEndMs(any(), any()) } answers { secondArg() * 1_000L } + every { getSegmentNumberAtOrAfterTimeMs(any(), 45_000L) } returns 46 + every { getSegmentStartMs(any(), 46) } returns 45_000L + every { getMinBufferedEndMs() } returns 0L + } + + private fun format(itag: Int, audio: Boolean): YoutubeSabrFormat = mockk { + every { this@mockk.itag } returns itag + every { isAudio } returns audio + every { isVideo } returns !audio + every { mimeType } returns if (audio) "audio/mp4; codecs=\"mp4a.40.2\"" else "video/mp4; codecs=\"avc1.640028\"" + every { bitrate } returns if (audio) 128_000 else 2_000_000 + every { width } returns if (audio) 0 else 1920 + every { height } returns if (audio) 0 else 1080 + } +} diff --git a/src/test/kotlin/dev/typetype/server/services/AndroidPlaybackSessionRegistryTest.kt b/src/test/kotlin/dev/typetype/server/services/AndroidPlaybackSessionRegistryTest.kt new file mode 100644 index 00000000..2cb1e0d2 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/services/AndroidPlaybackSessionRegistryTest.kt @@ -0,0 +1,94 @@ +package dev.typetype.server.services + +import io.mockk.every +import io.mockk.mockk +import io.mockk.verify +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertSame +import org.junit.jupiter.api.Test +import java.time.Duration +import java.time.Instant + +class AndroidPlaybackSessionRegistryTest { + @Test + fun `registered Android session resolves and touches the isolated store`() { + val current = MutableNow(Instant.parse("2026-07-20T10:00:00Z")) + val store = mockk() + val holder = holder() + val subtitle = mockk() + every { store.lookupByToken("android-session") } returns holder + val registry = registry(store, current) + registry.register(holder, listOf(subtitle)) + + val result = registry.lookup("android-session") as AndroidPlaybackSessionLookup.Active + + assertSame(holder, result.session.holder) + assertSame(subtitle, result.session.subtitles.single()) + verify(exactly = 1) { store.lookupByToken("android-session") } + } + + @Test + fun `idle session returns gone while unknown session returns not found`() { + val current = MutableNow(Instant.parse("2026-07-20T10:00:00Z")) + val store = mockk(relaxed = true) + val holder = holder() + val registry = registry(store, current) + registry.register(holder, emptyList()) + current.value = current.value.plus(Duration.ofMinutes(5)) + + assertEquals(AndroidPlaybackSessionLookup.Expired, registry.lookup("android-session")) + assertEquals(AndroidPlaybackSessionLookup.Unknown, registry.lookup("never-created")) + verify(exactly = 1) { store.release(holder) } + } + + @Test + fun `expired session tombstone is temporary`() { + val current = MutableNow(Instant.parse("2026-07-20T10:00:00Z")) + val store = mockk(relaxed = true) + val registry = registry(store, current) + registry.register(holder(), emptyList()) + current.value = current.value.plus(Duration.ofMinutes(5)) + assertEquals(AndroidPlaybackSessionLookup.Expired, registry.lookup("android-session")) + + current.value = current.value.plus(Duration.ofMinutes(11)) + + assertEquals(AndroidPlaybackSessionLookup.Unknown, registry.lookup("android-session")) + } + + @Test + fun `missing backing Android session becomes expired`() { + val current = MutableNow(Instant.parse("2026-07-20T10:00:00Z")) + val store = mockk(relaxed = true) + val holder = holder() + every { store.lookupByToken("android-session") } returns null + val registry = registry(store, current) + registry.register(holder, emptyList()) + + assertEquals(AndroidPlaybackSessionLookup.Expired, registry.lookup("android-session")) + verify(exactly = 1) { store.release(holder) } + } + + private fun registry(store: SabrSessionStore, current: MutableNow) = AndroidPlaybackSessionRegistry( + store = store, + idleTimeout = Duration.ofMinutes(4), + tombstoneTtl = Duration.ofMinutes(10), + now = current::get, + ) + + private fun holder(): SabrSessionHolder = mockk { + every { sessionToken } returns "android-session" + every { key } returns SabrSessionKey( + "video", + "user", + 140, + null, + 137, + 0L, + SabrSessionPurpose.ANDROID_PLAYBACK, + ) + } + + private class MutableNow(var value: Instant) { + fun get(): Instant = value + } +} diff --git a/src/test/kotlin/dev/typetype/server/services/AndroidSubtitleHttpClientTest.kt b/src/test/kotlin/dev/typetype/server/services/AndroidSubtitleHttpClientTest.kt new file mode 100644 index 00000000..ca564889 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/services/AndroidSubtitleHttpClientTest.kt @@ -0,0 +1,104 @@ +package dev.typetype.server.services + +import kotlinx.coroutines.test.runTest +import okhttp3.MediaType.Companion.toMediaType +import okhttp3.OkHttpClient +import okhttp3.Protocol +import okhttp3.Request +import okhttp3.Response +import okhttp3.ResponseBody.Companion.toResponseBody +import okhttp3.HttpUrl.Companion.toHttpUrl +import org.junit.jupiter.api.Assertions.assertArrayEquals +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test +import java.util.concurrent.atomic.AtomicInteger + +class AndroidSubtitleHttpClientTest { + @Test + fun `HTTP 429 falls back to direct egress and requests WebVTT`() = runTest { + val primaryCalls = AtomicInteger() + val directCalls = AtomicInteger() + val primary = client { + primaryCalls.incrementAndGet() + response(it, 429, "") + } + val direct = client { + directCalls.incrementAndGet() + assertEquals("vtt", it.url.queryParameter("fmt")) + assertEquals("text/vtt", it.header("Accept")) + response(it, 200, VTT.decodeToString()) + } + + val result = AndroidSubtitleHttpClient(primary, direct).fetch(SOURCE_URL) + + assertArrayEquals(VTT, (result as AndroidSubtitleUpstreamResult.Ready).bytes) + assertEquals(1, primaryCalls.get()) + assertEquals(1, directCalls.get()) + } + + @Test + fun `permanent upstream rejection does not retry another egress`() = runTest { + val directCalls = AtomicInteger() + val primary = client { response(it, 404, "") } + val direct = client { + directCalls.incrementAndGet() + response(it, 200, VTT.decodeToString()) + } + + val result = AndroidSubtitleHttpClient(primary, direct).fetch(SOURCE_URL) + + assertEquals(AndroidSubtitleUpstreamResult.Unavailable, result) + assertEquals(0, directCalls.get()) + } + + @Test + fun `invalid successful body is not exposed as WebVTT`() = runTest { + val client = client { response(it, 200, "") } + + val result = AndroidSubtitleHttpClient(client, client).fetch(SOURCE_URL) + + assertEquals(AndroidSubtitleUpstreamResult.Unavailable, result) + } + + @Test + fun `temporary failure on both egress paths remains typed`() = runTest { + val client = client { response(it, 503, "") } + + val result = AndroidSubtitleHttpClient(client, client).fetch(SOURCE_URL) + + assertEquals(AndroidSubtitleUpstreamResult.TemporaryFailure, result) + } + + @Test + fun `temporary failure is not duplicated without a configured proxy`() = runTest { + val calls = AtomicInteger() + val client = client { + calls.incrementAndGet() + response(it, 429, "") + } + + val result = AndroidSubtitleHttpClient(client, null).fetch(SOURCE_URL) + + assertEquals(AndroidSubtitleUpstreamResult.TemporaryFailure, result) + assertEquals(1, calls.get()) + } + + private fun client(responder: (Request) -> Response): OkHttpClient = + OkHttpClient.Builder() + .addInterceptor { chain -> responder(chain.request()) } + .build() + + private fun response(request: Request, status: Int, body: String): Response = + Response.Builder() + .request(request) + .protocol(Protocol.HTTP_1_1) + .code(status) + .message("test") + .body(body.toResponseBody("text/vtt".toMediaType())) + .build() + + private companion object { + val SOURCE_URL = "https://www.youtube.com/api/timedtext?v=dQw4w9WgXcQ&fmt=ttml".toHttpUrl() + val VTT = "WEBVTT\n\n00:00.000 --> 00:01.000\nHello\n".toByteArray() + } +} diff --git a/src/test/kotlin/dev/typetype/server/services/AndroidSubtitleServiceTest.kt b/src/test/kotlin/dev/typetype/server/services/AndroidSubtitleServiceTest.kt new file mode 100644 index 00000000..1ad03c24 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/services/AndroidSubtitleServiceTest.kt @@ -0,0 +1,79 @@ +package dev.typetype.server.services + +import dev.typetype.server.models.SubtitleItem +import io.mockk.coEvery +import io.mockk.coVerify +import io.mockk.mockk +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Assertions.assertArrayEquals +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test + +class AndroidSubtitleServiceTest { + @Test + fun `inventory exposes all Token tracks with stable identities`() = runTest { + val source = mockk() + val items = listOf(item("en", false), item("en", true), item("fr", false)) + coEvery { source.fetchSubtitleInventory(VIDEO_ID) } returns YouTubeSubtitleInventoryResult.Ready(items) + val service = AndroidSubtitleService(source, mockk()) + + val result = service.inventory(VIDEO_ID) as AndroidSubtitleInventoryResult.Ready + + assertEquals(3, result.tracks.size) + assertEquals(3, result.tracks.map { it.id }.distinct().size) + } + + @Test + fun `temporary upstream failure refreshes inventory once`() = runTest { + val source = mockk() + val http = mockk() + val oldTrack = track(item("en", false, "signature=old")) + val refreshedItem = item("en", false, "signature=new") + val refreshedTrack = track(refreshedItem) + coEvery { http.fetch(oldTrack.sourceUrl) } returns AndroidSubtitleUpstreamResult.TemporaryFailure + coEvery { source.fetchSubtitleInventory(VIDEO_ID) } returns + YouTubeSubtitleInventoryResult.Ready(listOf(refreshedItem)) + coEvery { http.fetch(refreshedTrack.sourceUrl) } returns AndroidSubtitleUpstreamResult.Ready(VTT) + val service = AndroidSubtitleService(source, http) + + val result = service.content(VIDEO_ID, oldTrack) as AndroidSubtitleContentResult.Ready + + assertArrayEquals(VTT, result.bytes) + coVerify(exactly = 1) { source.fetchSubtitleInventory(VIDEO_ID) } + } + + @Test + fun `repeated temporary failure remains typed and bounded`() = runTest { + val source = mockk() + val http = mockk() + val track = track(item("en", false)) + coEvery { http.fetch(any()) } returns AndroidSubtitleUpstreamResult.TemporaryFailure + coEvery { source.fetchSubtitleInventory(VIDEO_ID) } returns + YouTubeSubtitleInventoryResult.Ready(listOf(item("en", false))) + val service = AndroidSubtitleService(source, http) + + assertEquals(AndroidSubtitleContentResult.TemporaryFailure, service.content(VIDEO_ID, track)) + coVerify(exactly = 2) { http.fetch(any()) } + coVerify(exactly = 1) { source.fetchSubtitleInventory(VIDEO_ID) } + } + + private fun track(item: SubtitleItem): AndroidSubtitleTrack = + AndroidSubtitleTrackFactory.create(VIDEO_ID, listOf(item))!!.single() + + private fun item( + language: String, + auto: Boolean, + extra: String = "signature=test", + ) = SubtitleItem( + url = "https://www.youtube.com/api/timedtext?v=$VIDEO_ID&lang=$language&kind=${if (auto) "asr" else ""}&$extra", + mimeType = "application/ttml+xml", + languageTag = language, + displayLanguageName = language, + isAutoGenerated = auto, + ) + + private companion object { + const val VIDEO_ID = "dQw4w9WgXcQ" + val VTT = "WEBVTT\n\n00:00.000 --> 00:01.000\nHello\n".toByteArray() + } +} diff --git a/src/test/kotlin/dev/typetype/server/services/AndroidSubtitleTrackFactoryTest.kt b/src/test/kotlin/dev/typetype/server/services/AndroidSubtitleTrackFactoryTest.kt new file mode 100644 index 00000000..b1fc14dc --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/services/AndroidSubtitleTrackFactoryTest.kt @@ -0,0 +1,66 @@ +package dev.typetype.server.services + +import dev.typetype.server.models.SubtitleItem +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNotEquals +import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.Test + +class AndroidSubtitleTrackFactoryTest { + @Test + fun `track identity ignores expiring upstream parameters`() { + val first = track(url("signature=old&expire=1"), "English") + val refreshed = track(url("signature=new&expire=2"), "Anglais") + + assertEquals(first.id, refreshed.id) + } + + @Test + fun `manual and generated tracks in one language remain distinct`() { + val tracks = AndroidSubtitleTrackFactory.create( + VIDEO_ID, + listOf( + item(url("vssId=.en"), auto = false), + item(url("vssId=a.en&kind=asr"), auto = true), + ), + )!! + + assertEquals(2, tracks.size) + assertNotEquals(tracks[0].id, tracks[1].id) + } + + @Test + fun `non YouTube caption URL rejects the inventory`() { + val tracks = AndroidSubtitleTrackFactory.create( + VIDEO_ID, + listOf(item("https://example.com/subtitles.vtt", auto = false)), + ) + + assertNull(tracks) + } + + private fun track(sourceUrl: String, displayName: String): AndroidSubtitleTrack = + AndroidSubtitleTrackFactory.create( + VIDEO_ID, + listOf(item(sourceUrl, false, displayName)), + )!!.single() + + private fun item( + sourceUrl: String, + auto: Boolean, + displayName: String = "English", + ) = SubtitleItem( + url = sourceUrl, + mimeType = "application/ttml+xml", + languageTag = "en", + displayLanguageName = displayName, + isAutoGenerated = auto, + ) + + private fun url(extra: String): String = + "https://www.youtube.com/api/timedtext?v=$VIDEO_ID&lang=en&$extra" + + private companion object { + const val VIDEO_ID = "dQw4w9WgXcQ" + } +} diff --git a/src/test/kotlin/dev/typetype/server/services/SabrCachedSegmentLocatorTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrCachedSegmentLocatorTest.kt index ce1b1de2..89e70747 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrCachedSegmentLocatorTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrCachedSegmentLocatorTest.kt @@ -49,6 +49,27 @@ class SabrCachedSegmentLocatorTest { assertSame(segment, found) } + @Test + fun `uses derived live duration when media header omits duration`() { + val format = mockk() + val session = mockk() + val segment = segment(sequence = 2_049_677, startMs = 4_099_349_989L, durationMs = -1L) + every { format.itag } returns 140 + every { session.getCachedSegment(any()) } answers { + firstArg().takeIf { it.sequenceNumber == 2_049_677 }?.let { segment } + } + + val found = session.findCachedMediaAt( + format = format, + targetMs = 4_099_347_988L, + predictedSequence = 2_049_676, + fallbackDurationMs = 2_001L, + allowFollowing = true, + ) + + assertSame(segment, found) + } + private fun segment(sequence: Int, startMs: Long, durationMs: Long): SabrMediaSegment { val header = mockk() every { header.sequenceNumber } returns sequence diff --git a/src/test/kotlin/dev/typetype/server/services/SabrDemandWatchdogBackoffTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrDemandWatchdogBackoffTest.kt new file mode 100644 index 00000000..fa6aa889 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/services/SabrDemandWatchdogBackoffTest.kt @@ -0,0 +1,118 @@ +package dev.typetype.server.services + +import io.mockk.every +import io.mockk.mockk +import kotlinx.coroutines.ExperimentalCoroutinesApi +import kotlinx.coroutines.launch +import kotlinx.coroutines.test.advanceTimeBy +import kotlinx.coroutines.test.runCurrent +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo +import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrSession +import java.time.Instant + +@OptIn(ExperimentalCoroutinesApi::class) +class SabrDemandWatchdogBackoffTest { + @Test + fun `queued demand deadline excludes thirty second backoff`() = runTest { + withTracker { holder -> + val request = SabrSegmentRequest.media(holder.audioFormat, 50) + val backoffUntilMs = 30_000L + every { holder.session.demandBackoffRemainingMs } answers { + (backoffUntilMs - testScheduler.currentTime).coerceAtLeast(0L) + } + holder.requestSegmentDemand(request, registeredAtMs = 0L) + var expired = false + val job = launch { + expired = watchdog { testScheduler.currentTime }.monitor({ true }, holder) + } + runCurrent() + + advanceTimeBy(44_900L) + runCurrent() + assertFalse(job.isCompleted) + + advanceTimeBy(100L) + runCurrent() + assertTrue(job.isCompleted) + assertTrue(expired) + assertEquals("SABR demand stalled for 140:50", holder.terminalFailure()) + } + } + + @Test + fun `in flight deadline excludes twenty second backoff`() = runTest { + withTracker { holder -> + val request = SabrSegmentRequest.media(holder.videoFormat, 50) + val backoffUntilMs = 20_000L + every { holder.session.demandBackoffRemainingMs } answers { + (backoffUntilMs - testScheduler.currentTime).coerceAtLeast(0L) + } + holder.requestSegmentDemand(request, registeredAtMs = 0L) + val identity = requireNotNull(holder.segmentDemandIdentity(request)) + assertTrue(holder.beginInFlightSegmentDemand(request, identity, futureLiveRequest = false)) + holder.clearSegmentDemand(request) + var expired = false + val job = launch { + expired = watchdog { testScheduler.currentTime }.monitor({ true }, holder) + } + runCurrent() + + advanceTimeBy(34_900L) + runCurrent() + assertFalse(job.isCompleted) + + advanceTimeBy(100L) + runCurrent() + assertTrue(job.isCompleted) + assertTrue(expired) + assertEquals("SABR demand stalled for 299:50", holder.terminalFailure()) + } + } + + private fun watchdog(clock: () -> Long): SabrDemandWatchdog = SabrDemandWatchdog( + clock = clock, + intervalMs = 100L, + ) + + private suspend fun withTracker(block: suspend (SabrSessionHolder) -> Unit) { + SabrSegmentDemandTracker.clearAll() + SabrInFlightDemandTracker.clearAll() + try { + block(holder()) + } finally { + SabrSegmentDemandTracker.clearAll() + SabrInFlightDemandTracker.clearAll() + } + } + + private fun holder(): SabrSessionHolder { + val session = mockk(relaxed = true) + val audio = format(140, true) + val video = format(299, false) + every { session.getCachedSegment(any()) } returns null + every { session.mediaProgressVersion } returns 0L + return SabrSessionHolder( + session = session, + info = mockk(), + audioFormat = audio, + videoFormat = video, + sessionToken = "backoff-session", + key = SabrSessionKey("video", "user", audio.itag, null, video.itag, 0L), + lastRequestAt = Instant.EPOCH, + ) + } + + private fun format(itag: Int, isAudio: Boolean): YoutubeSabrFormat { + val format = mockk() + every { format.itag } returns itag + every { format.isAudio } returns isAudio + return format + } +} diff --git a/src/test/kotlin/dev/typetype/server/services/SabrLivePlaybackSessionServiceTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrLivePlaybackSessionServiceTest.kt index 7ffe1409..bde2a853 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrLivePlaybackSessionServiceTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrLivePlaybackSessionServiceTest.kt @@ -9,6 +9,7 @@ import kotlinx.coroutines.test.runTest import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertFalse import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.Assertions.assertSame import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaHeader @@ -18,6 +19,7 @@ import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrSession import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrStreamState import java.time.Instant +import java.util.concurrent.atomic.AtomicInteger class SabrLivePlaybackSessionServiceTest { @Test @@ -75,6 +77,7 @@ class SabrLivePlaybackSessionServiceTest { false, SabrSessionPurpose.PLAYBACK, false, + 0L, ) } returns holder coEvery { store.ensureWarmed(holder, 8) } returns Unit @@ -91,40 +94,74 @@ class SabrLivePlaybackSessionServiceTest { } @Test - fun `live quality change skips the unavailable segment before a nearby audio boundary`() { + fun `live format change starts the replacement session from warmed track boundaries`() = runTest { val audio = format(140, isAudio = true) + val source = holder(audio, format(137, isAudio = false), initialGeneration = 4L) val video = format(248, isAudio = false) - val holder = holder(audio, video) + val replacement = holder(audio, video, initialGeneration = 5L) + val info = mockk() + val prepared = SabrPreparedInfo(info, token(), isLive = true, isLiveContent = true) val audioSegment = mediaSegment(audio.itag, 995_010L, sequence = 101) val videoSegment = mediaSegment(video.itag, 995_000L, sequence = 100) - holder.markExpectedLive() - holder.observeMediaSegment(audioSegment) - holder.observeMediaSegment(videoSegment) + replacement.markExpectedLive() + replacement.observeMediaSegment(audioSegment) + replacement.observeMediaSegment(videoSegment) + val replacementState = replacement.session.streamState + every { replacementState.liveHeadTimeMs } returns 1_005_000L every { - holder.session.getCachedSegment(match { + replacement.session.getCachedSegment(match { it.format.itag == video.itag && it.sequenceNumber == 100 }) } returns videoSegment every { - holder.session.getCachedSegment(match { + replacement.session.getCachedSegment(match { it.format.itag == audio.itag && it.sequenceNumber == 101 }) } returns audioSegment val store = mockk() - every { store.startPump(holder) } returns Unit + every { + store.getOrCreate( + "video", + "user", + info, + audio, + video, + prepared.initialToken, + 995_000L, + false, + SabrSessionPurpose.PLAYBACK, + false, + 5L, + ) + } returns replacement + coEvery { store.ensureWarmed(replacement, 8) } returns Unit + every { store.startPump(replacement) } returns Unit - SabrPlaybackSessionService(store).seekExisting(holder, 995_000L) + val result = SabrPlaybackSessionService(store).seek(source, prepared, audio, video, 995_000L) - assertNull(holder.nextSegmentDemand()) - assertEquals(995_010L, holder.readerPosition(audio)) - assertFalse(holder.mediaRequestsAt(995_000L).any { it.format.itag == audio.itag }) + assertSame(replacement, result.holder) + assertEquals(5L, result.holder.activeGeneration()) + assertEquals(audio.itag, result.holder.audioFormat.itag) + assertEquals(video.itag, result.holder.videoFormat.itag) + assertNull(replacement.nextSegmentDemand()) + assertEquals(995_010L, replacement.readerPosition(audio)) + assertEquals(1_000_000L, replacement.readerPosition(video)) + assertFalse(replacement.mediaRequestsAt(995_000L).any { it.format.itag == audio.itag }) + assertFalse(replacement.mediaRequestsAt(995_000L).any { it.format.itag == video.itag }) + coVerify(exactly = 1) { store.ensureWarmed(replacement, 8) } + verify(exactly = 1) { store.startPump(replacement) } } - private fun holder(audio: YoutubeSabrFormat, video: YoutubeSabrFormat): SabrSessionHolder { + private fun holder( + audio: YoutubeSabrFormat, + video: YoutubeSabrFormat, + initialGeneration: Long = 0L, + ): SabrSessionHolder { val session = mockk() val state = mockk(relaxed = true) every { session.streamState } returns state every { session.getCachedSegment(any()) } returns null + every { session.getReadableSegment(any()) } returns null every { session.isBeyondEnd(any()) } returns false every { session.prepareForInitialization(any()) } returns Unit every { state.setActiveTrackTypes(any(), any()) } returns Unit @@ -134,9 +171,10 @@ class SabrLivePlaybackSessionServiceTest { info = mockk(), audioFormat = audio, videoFormat = video, - sessionToken = "session-token", + sessionToken = "session-token-${sessionIds.incrementAndGet()}", key = SabrSessionKey("video", "user", audio.itag, null, video.itag, 0L), lastRequestAt = Instant.EPOCH, + initialGeneration = initialGeneration, ) } @@ -168,4 +206,8 @@ class SabrLivePlaybackSessionServiceTest { videoBoundPoToken = "video-token", videoBoundPoTokenBytes = byteArrayOf(2), ) + + private companion object { + val sessionIds = AtomicInteger() + } } diff --git a/src/test/kotlin/dev/typetype/server/services/SabrLivePlaybackTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrLivePlaybackTest.kt index 48c68125..a72a4be4 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrLivePlaybackTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrLivePlaybackTest.kt @@ -49,6 +49,7 @@ class SabrLivePlaybackTest { @Test fun `active live maps time from an observed sabr segment for every codec`() { val fixture = fixture() + every { fixture.state.getMaxSegment(fixture.video) } returns 180 val header = mockk { every { isInitSegment } returns false every { itag } returns fixture.video.itag @@ -69,6 +70,18 @@ class SabrLivePlaybackTest { assertEquals(180, fixture.holder.playbackStartSequence(fixture.video, 965_000L)) assertEquals(179, fixture.holder.playbackStartSequence(fixture.video, 964_999L)) assertEquals(200, fixture.holder.playbackStartSequence(fixture.video, 1_006_000L)) + every { fixture.state.getMaxSegment(fixture.video) } returns 200 + val nearHead = mockk { + every { this@mockk.header } returns mockk { + every { isInitSegment } returns false + every { itag } returns fixture.video.itag + every { sequenceNumber } returns 199 + every { startMs } returns 1_003_000L + every { durationMs } returns 2_000L + } + } + fixture.holder.observeMediaSegment(nearHead) + assertEquals(199, fixture.holder.playbackStartSequence(fixture.video, 1_006_000L)) } @Test @@ -90,6 +103,23 @@ class SabrLivePlaybackTest { assertEquals(180, fixture.holder.playbackStartSequence(fixture.video, 965_000L)) } + @Test + fun `active live maps time before the head sequence is reported`() { + val fixture = fixture() + every { fixture.session.liveHeadSequenceNumber } returns 0L + every { fixture.state.liveHeadSequenceNumber } returns 0L + val header = mockk { + every { isInitSegment } returns false + every { itag } returns fixture.video.itag + every { sequenceNumber } returns 180 + every { startMs } returns 965_000L + every { durationMs } returns 2_000L + } + fixture.holder.observeMediaSegment(mockk { every { this@mockk.header } returns header }) + + assertEquals(195, fixture.holder.playbackStartSequence(fixture.video, 995_000L)) + } + @Test fun `live duration tolerates millisecond drift between media and head timestamps`() { val fixture = fixture() @@ -146,6 +176,7 @@ class SabrLivePlaybackTest { every { itag } returns fixture.video.itag every { sequenceNumber } returns 198 every { startMs } returns 998_000L + every { durationMs } returns 2_000L } fixture.holder.observeMediaSegment(mockk { every { this@mockk.header } returns header }) diff --git a/src/test/kotlin/dev/typetype/server/services/SabrPlaybackSessionServiceTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrPlaybackSessionServiceTest.kt index 2ea2439f..af76683c 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrPlaybackSessionServiceTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrPlaybackSessionServiceTest.kt @@ -5,6 +5,7 @@ import io.mockk.coVerify import io.mockk.every import io.mockk.mockk import io.mockk.verify +import kotlinx.coroutines.runBlocking import kotlinx.coroutines.test.runTest import org.junit.jupiter.api.Assertions.assertArrayEquals import org.junit.jupiter.api.Assertions.assertEquals @@ -26,7 +27,7 @@ class SabrPlaybackSessionServiceTest { fun clearDemands(): Unit = SabrSegmentDemandTracker.clearAll() @Test - fun `prepare loads timing before selecting target segments`() = runTest { + fun `prepare loads timing before selecting target segments`() = runBlocking { val audio = format(140, isAudio = true) val video = format(137, isAudio = false) val info = mockk() @@ -60,6 +61,7 @@ class SabrPlaybackSessionServiceTest { false, SabrSessionPurpose.PLAYBACK, false, + 0L, ) } returns holder coEvery { store.fetchInitializationData(holder, video) } answers { @@ -223,6 +225,7 @@ class SabrPlaybackSessionServiceTest { every { holder.session.streamState.setSelectVideoFormatBeforeAudio(false) } returns Unit every { holder.session.streamState.getSegmentNumberAtOrAfterTimeMs(audio, 299L) } returns 1 every { holder.session.streamState.getSegmentStartMs(audio, 1) } returns 0L + every { holder.session.streamState.getSegmentEndMs(audio, 1) } returns 5_000L every { holder.session.getCachedSegment(match { it.format.itag == 140 && it.sequenceNumber == 1 }) } returns mockk() val store = mockk() @@ -231,7 +234,7 @@ class SabrPlaybackSessionServiceTest { SabrPlaybackSessionService(store).seekExisting(holder, playerTimeMs = 299L, audioOnly = true) assertEquals(1L, holder.activeGeneration()) - assertEquals(0L, holder.readerTailMs()) + assertEquals(5_000L, holder.readerTailMs()) assertEquals(null, holder.consumeRefetch()) assertEquals(null, holder.consumeForwardSeek()) verify { holder.session.streamState.setActiveTrackTypes(false, true) } diff --git a/src/test/kotlin/dev/typetype/server/services/SabrSeekRepositionPumpTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrSeekRepositionPumpTest.kt index 5125f12b..4edb2ff2 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrSeekRepositionPumpTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrSeekRepositionPumpTest.kt @@ -5,6 +5,7 @@ import io.mockk.mockk import io.mockk.verify import kotlinx.coroutines.test.runTest import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse import org.junit.jupiter.api.Test import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaHeader import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaSegment @@ -136,6 +137,38 @@ class SabrSeekRepositionPumpTest { } } + @Test + fun `historical live demand keeps rewinding after the observed segment`() = runTest { + SabrSegmentDemandTracker.clearAll() + try { + val audio = format(140, true) + val video = format(299, false) + val request = SabrSegmentRequest.media(video, 3_077) + val session = mockk(relaxed = true) + val state = mockk(relaxed = true) + every { session.streamState } returns state + every { session.isLive } returns true + every { state.isLive } returns true + every { state.isPostLiveDvr } returns false + every { state.liveHeadTimeMs } returns 15_750_000L + every { session.getCachedSegment(any()) } returns null + every { session.pumpOnceStreamingForDemand(any(), request) } returns mockk(relaxed = true) + val holder = holder(session, audio, video) + holder.observeMediaSegment(mediaSegment(video.itag, sequence = 3_076)) + holder.requestSegmentDemand(request) + assertFalse(holder.isFutureLiveRequest(request)) + assertEquals(DEFAULT_PLAYBACK_RETRY_MS, holder.liveRetryAfterMs(listOf(request))) + var rounds = 0 + + SabrSessionPumpLoop().run({ rounds++ < 1 }, holder, intervalMs = 0L) + + verify(exactly = 1) { session.prepareForRewind(request) } + verify(exactly = 1) { session.pumpOnceStreamingForDemand(any(), request) } + } finally { + SabrSegmentDemandTracker.clearAll() + } + } + @Test fun `historical live seek preserves its exact player position`() = runTest { SabrSegmentDemandTracker.clearAll() diff --git a/src/test/kotlin/dev/typetype/server/services/SabrSegmentDemandResolutionTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrSegmentDemandResolutionTest.kt index 9454dba1..a8768057 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrSegmentDemandResolutionTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrSegmentDemandResolutionTest.kt @@ -4,6 +4,7 @@ import io.mockk.every import io.mockk.mockk import io.mockk.verify import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.Assertions.assertArrayEquals import org.junit.jupiter.api.Assertions.assertNull import org.junit.jupiter.api.Assertions.assertSame import org.junit.jupiter.api.Assertions.assertTrue @@ -52,13 +53,14 @@ class SabrSegmentDemandResolutionTest { val session = mockk(relaxed = true) val state = mockk(relaxed = true) val request = SabrSegmentRequest.media(format, 92) - val replacement = segment(itag = 299, sequence = 93, startMs = 480_000L, durationMs = 5_000L) + val replacement = segment(itag = 299, sequence = 93, startMs = 480_000L, durationMs = -1L) every { format.itag } returns 299 every { format.isAudio } returns false every { session.streamState } returns state every { session.isLive } returns true every { state.isLive } returns true every { state.getSegmentStartMs(format, 92) } returns 475_000L + every { state.getSegmentEndMs(format, 92) } returns 480_000L every { session.getCachedSegment(any()) } answers { firstArg().takeIf { it.sequenceNumber == 93 }?.let { replacement } } @@ -94,6 +96,32 @@ class SabrSegmentDemandResolutionTest { assertNull(holder.pendingSegmentDemandSummary()) } + @Test + fun `resolved demand remains in server cache after extractor eviction`() { + val format = mockk() + val session = mockk(relaxed = true) + val request = SabrSegmentRequest.media(format, 2_752) + val cached = segment(sequence = 2_752, startMs = 13_757_066L, durationMs = 4_997L) + var extractorCached = false + every { format.itag } returns 140 + every { format.isAudio } returns true + every { format.mimeType } returns "audio/mp4" + every { format.lastModified } returns 0L + every { format.xtags } returns "" + every { cached.data } returns byteArrayOf(1, 2, 3) + every { session.getCachedSegment(request) } answers { cached.takeIf { extractorCached } } + val holder = holder(session, format) + val serverCache = SabrSegmentCache() + holder.requestSegmentDemand(request) + val identity = requireNotNull(holder.segmentDemandIdentity(request)) + extractorCached = true + + assertTrue(holder.resolveSegmentDemand(request, identity) { serverCache.put(holder, it) }) + extractorCached = false + + assertArrayEquals(byteArrayOf(1, 2, 3), serverCache.get(holder, request)?.bytes) + } + private fun holder(session: YoutubeSabrSession, audio: YoutubeSabrFormat): SabrSessionHolder { val video = mockk() every { video.itag } returns 299 diff --git a/src/test/kotlin/dev/typetype/server/services/SabrSessionTimeRequestsTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrSessionTimeRequestsTest.kt index e35480b0..d62d6ec2 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrSessionTimeRequestsTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrSessionTimeRequestsTest.kt @@ -4,6 +4,9 @@ import io.mockk.every import io.mockk.mockk import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Test +import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaHeader +import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaSegment +import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrSession @@ -65,6 +68,58 @@ class SabrSessionTimeRequestsTest { assertEquals(listOf(64), requests.map { it.sequenceNumber }) } + @Test + fun `reposition keeps a non adjacent live boundary request pending`() { + val audio = sabrFormat(itag = 140, isAudio = true) + val video = sabrFormat(itag = 248, isAudio = false) + val session = mockk() + val state = mockk(relaxed = true) + every { session.streamState } returns state + every { state.setActiveTrackTypes(any(), any()) } returns Unit + every { state.getSegmentStartMs(audio, 100) } returns 995_000L + every { session.getCachedSegment(any()) } returns null + val holder = holder(session, audio, video) + val observed = mediaSegment(audio.itag, sequence = 102, startMs = 995_010L) + holder.markExpectedLive() + holder.observeMediaSegment(observed) + every { + session.getCachedSegment(match { + it.format.itag == audio.itag && it.sequenceNumber == 102 + }) + } returns observed + val request = SabrSegmentRequest.media(audio, 100) + + val missing = holder.repositionTargets(listOf(request), playerTimeMs = 995_000L, generation = 0L) + + assertEquals(listOf(request), missing) + } + + @Test + fun `reposition advances to the next warmed live segment from inside a boundary`() { + val audio = sabrFormat(itag = 140, isAudio = true) + val video = sabrFormat(itag = 248, isAudio = false) + val session = mockk() + val state = mockk(relaxed = true) + every { session.streamState } returns state + every { state.setActiveTrackTypes(any(), any()) } returns Unit + every { session.getCachedSegment(any()) } returns null + val holder = holder(session, audio, video) + val observed = mediaSegment(audio.itag, sequence = 101, startMs = 1_000_000L, durationMs = 5_000L) + holder.markExpectedLive() + holder.observeMediaSegment(observed) + every { + session.getCachedSegment(match { + it.format.itag == audio.itag && it.sequenceNumber == 101 + }) + } returns observed + val request = SabrSegmentRequest.media(audio, 100) + + val missing = holder.repositionTargets(listOf(request), playerTimeMs = 996_200L, generation = 0L) + + assertEquals(emptyList(), missing) + assertEquals(1_000_000L, holder.readerPosition(audio)) + } + private fun sabrFormat(itag: Int, isAudio: Boolean): YoutubeSabrFormat { val format = mockk() every { format.itag } returns itag @@ -73,6 +128,21 @@ class SabrSessionTimeRequestsTest { return format } + private fun mediaSegment( + itag: Int, + sequence: Int, + startMs: Long, + durationMs: Long = 0L, + ): SabrMediaSegment { + val header = mockk(relaxed = true) + every { header.itag } returns itag + every { header.sequenceNumber } returns sequence + every { header.startMs } returns startMs + every { header.durationMs } returns durationMs + every { header.isInitSegment } returns false + return mockk { every { this@mockk.header } returns header } + } + private fun holder( session: YoutubeSabrSession, audio: YoutubeSabrFormat, diff --git a/src/test/kotlin/dev/typetype/server/services/YouTubeSubtitleServiceTest.kt b/src/test/kotlin/dev/typetype/server/services/YouTubeSubtitleServiceTest.kt new file mode 100644 index 00000000..66a31a45 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/services/YouTubeSubtitleServiceTest.kt @@ -0,0 +1,57 @@ +package dev.typetype.server.services + +import kotlinx.coroutines.test.runTest +import okhttp3.MediaType.Companion.toMediaType +import okhttp3.OkHttpClient +import okhttp3.Protocol +import okhttp3.Response +import okhttp3.ResponseBody.Companion.toResponseBody +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test + +class YouTubeSubtitleServiceTest { + @Test + fun `Token subtitle inventory is decoded without changing classic behavior`() = runTest { + val service = service( + """ + [{ + "url":"https://www.youtube.com/api/timedtext?v=video&lang=en", + "mimeType":"application/ttml+xml", + "languageTag":"en", + "displayLanguageName":"English", + "isAutoGenerated":false + }] + """.trimIndent(), + ) + + val result = service.fetchSubtitleInventory("video") as YouTubeSubtitleInventoryResult.Ready + + assertEquals(1, result.tracks.size) + assertEquals("en", result.tracks.single().languageTag) + } + + @Test + fun `invalid Token response is a typed unavailable inventory`() = runTest { + val service = service("not-json") + + assertEquals( + YouTubeSubtitleInventoryResult.Unavailable, + service.fetchSubtitleInventory("video"), + ) + } + + private fun service(body: String): YouTubeSubtitleService { + val client = OkHttpClient.Builder() + .addInterceptor { chain -> + Response.Builder() + .request(chain.request()) + .protocol(Protocol.HTTP_1_1) + .code(200) + .message("test") + .body(body.toResponseBody("application/json".toMediaType())) + .build() + } + .build() + return YouTubeSubtitleService(client, "http://token") + } +}