From ff169d3e57929463e6d8d1ea3b9947a3e37d0b79 Mon Sep 17 00:00:00 2001 From: Paul Berberian Date: Fri, 13 Dec 2024 14:50:02 +0100 Subject: [PATCH 1/2] Thumbnails: Add supplementary metadata to `getAvailableThumbnailTracks` Based on #1496 Problem ------- We're currently trying to provide a complete[1] and easy to-use API for DASH thumbnail tracks in the RxPlayer. Today the proposal is to have an API called `renderThumbnail`, to which an application would just provide an HTML element and a timestamp, and the RxPlayer would do all that's necessary to fetch the corresponding thumbnail and display it in the corresponding element. The API is like so: ```js rxPlayer.renderThumbnail({ element, time }) .then(() => console.log("The thumbnail is now rendered in the element")); ``` This works and seems to me very simple to understand. Yet, we've known of advanced use cases where an application might not just want to display a single thumbnail for a single position. For example, there's very known examples where an application displays a window of multiple thumbnails at once on the player's UI to facilitate navigation inside the content. To do that under the solution proposed in #1496, an application could just call `renderThumbnail` with several `element` and `time` values. Yet for this type of feature, what the interface would want is not really to indicate a `time` values, it actually wants basically a list of distinct thumbnails around/before/after a given position. By just being able to set a `time` value, an application is blind on which `time` value is going to lead to a different timestamp (i.e. is the thumbnail for the `time` `11` different than the thumbnail for the `time` `12`? Nobody - but the RxPlayer - knows). So we have to find a solution for this [1] By complete, I here mean that we want to be able to handle its complexities inside the RxPlayer, to ensure complex DASH situations like multi-CDN, retry settings for requests and so on while still allowing all potential use cases for an application. Solution -------- In this solution, I experiment with a second thumbnail API, `getAvailableThumbnailTracks` (it already exists in #1496, but its role there was only to list the various thumbnail qualities, if there are several size for example). As this solution build upon yet stays compatible to #1496, I chose to open this second PR on top of that previous one. I profit from the fact that most standardized thumbnail implementations I know of (BIF, DASH) seem follow the principle of having evenly-spaced (in terms of time) thumbnails (though I do see a possibility for that to change, e.g. to have thumbnails corresponding to "important" scenes instead, so our implementation has to be resilient). So here, what this commit does is to add the following properties (all optional) to a track returned by the `getAvailableThumbnailTracks` API: - `start`: The initial `time` the first thumbnail of that track will apply to - `end`: The last `time` the last thumbnail of that track will apply to - thumbnailsPerSegment: Individual thumbnails may be technically part of "segments" containing multiple consecutive thumbnails each. `thumbnailsPerSegment` is the number of thumbnails each of those segments contain. For example you could have stored on the server a segment which is a grid of 2 x 3 (2 horizontal rows and * 3 vertical columns) thumbnails, which the RxPlayer will load at once then "cut" the right way when calling `renderThumbnail`. In that example, `thumbnailsPerSegment` would be set to `6` (2*3). Note that the last segment of a content may contain less thumbnails as anounced here depending on the duration of the content. - `segmentDuration`: The "duration" (in seconds) each segments of thumbnails applies to (with the exception of the last thumbnail, which just fills until `end`) Then, an application should have all information needed to calculate a `time` which correspond to a different thumbnail. Though this solution lead to a minor issue: by letting application make the `time` operation themselves with `start`, `end`, `segmentDuration` and so on, there's a risk of rounding errors leading to a `time` which does not correspond to the thumbnail wanted but the one before or after. To me, we could just indicate in our API documentation to application developers that they should be extra careful and may add an epsilon (or even choose a `time` in the "middle" of thumbnails each time) if they want that type of thumbnail list feature. Thoughts? --- src/main_thread/api/public_api.ts | 4 +++ src/public_types.ts | 53 +++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+) diff --git a/src/main_thread/api/public_api.ts b/src/main_thread/api/public_api.ts index 23fd021a9c..ce9ad3e5e6 100644 --- a/src/main_thread/api/public_api.ts +++ b/src/main_thread/api/public_api.ts @@ -961,6 +961,10 @@ class Player extends EventEmitter { width: Math.floor(t.width / t.horizontalTiles), height: Math.floor(t.height / t.verticalTiles), mimeType: t.mimeType, + start: t.start, + end: t.end, + thumbnailDuration: t.tileDuration, + thumbnailsPerSegment: t.horizontalTiles * t.verticalTiles, }; }); } diff --git a/src/public_types.ts b/src/public_types.ts index b717b793e9..0669d9ab37 100644 --- a/src/public_types.ts +++ b/src/public_types.ts @@ -1414,6 +1414,59 @@ export interface IThumbnailTrackInfo { * `image/jpeg` or `image/png`. */ mimeType: string | undefined; + /** + * Starting `position` the first thumbnail of this thumbnail track applies to, + * if known. + */ + start: number | undefined; + /** + * Ending `position` the last thumbnail of this thumbnail track applies to, + * if known. + */ + end: number | undefined; + /** + * Individual thumbnails may be technically part of "segments" containing + * multiple consecutive thumbnails each. + * + * `thumbnailsPerSegment` is the number of `thumbnails` each segments have. + * + * For example you could have stored on the server a segment which is a grid + * of 2 x 3 (2 horizontal rows and 3 vertical columns) thumbnails, which the + * RxPlayer will load at once then "cut" the right way when calling + * `renderThumbnail`. In that example, `thumbnailsPerSegment` would be set to + * `6` (2*3). + * + * Note that the last segment of a content may contain less thumbnails as + * anounced here depending on the duration of the content. + * + * You may want to rely on this information alongside `thumbnailDuration` to + * construct a list of available thumbnails and/or of available segments of + * thumbnails. + */ + thumbnailsPerSegment: number | undefined; + /** + * When loaded, thumbnails are part of so-called "segments" which may contain + * either a single thumbnail or a grid of them (@see `thumbnailsPerSegment`). + * + * This `thumbnail` property indicates a duration in seconds each thumbnail + * apply to. + * You can multiply that value with `thumbnailsPerSegment` to get the amount + * of time each segment applies to. + * + * Set to `undefined` either the duration is unknown or if the duration + * depends from segment to segments. + * + * For example, with a `start` set to `10`, an `end` set to `26`, a + * `thumbnailsPerSegment` set to `2` and a `tileDuration` set to + * `3`, there should be 3 segments, each with 2 thumbnails of 3 seconds each: + * 1. A segment of 2 thumbnails for the seconds: 10-16 + * (The first thumbnail in that segment for 10-13, the second for 13-16) + * 2. A segment of 2 thumbnails for the seconds: 16-22 + * (The first thumbnail in that segment for 16-19, the second for 19-22) + * 3. A segment of 2 thumbnails for the seconds: 22-26 (the end) + * (The first thumbnail in that segment for 22-25, the second for 25-26) + */ + thumbnailDuration: number | undefined; } /** From b354c3ac038911778f4ce9865548137bd9372411 Mon Sep 17 00:00:00 2001 From: Paul Berberian Date: Tue, 1 Jul 2025 19:02:43 +0200 Subject: [PATCH 2/2] Add lastThumbnailTime --- src/main_thread/api/public_api.ts | 44 +++++++++++++++++++ src/public_types.ts | 5 +++ .../scenarios/dash_render_thumbnail.test.js | 5 +++ 3 files changed, 54 insertions(+) diff --git a/src/main_thread/api/public_api.ts b/src/main_thread/api/public_api.ts index ce9ad3e5e6..13e9cba695 100644 --- a/src/main_thread/api/public_api.ts +++ b/src/main_thread/api/public_api.ts @@ -68,6 +68,7 @@ import { toVideoRepresentation, toAudioRepresentation, } from "../../manifest/index.ts"; +import type { IThumbnailTrackMetadata } from "../../manifest/types.ts"; import type { IPlaybackObservation } from "../../playback_observer/index.ts"; import MediaElementPlaybackObserver from "../../playback_observer/media_element_playback_observer.ts"; import type { @@ -965,6 +966,7 @@ class Player extends EventEmitter { end: t.end, thumbnailDuration: t.tileDuration, thumbnailsPerSegment: t.horizontalTiles * t.verticalTiles, + lastThumbnailTime: this._getLastThumbnailTime(t), }; }); } @@ -3804,6 +3806,48 @@ class Player extends EventEmitter { } } + /** + * For the given track, returns the position in seconds that will + * correspond to the currently last reachable thumbnail, or `undefined` if + * unknown. + * + * That position may then be passed to the `rxPlayer.renderThumbnail()` method. + * + * @param {Object} metadata + * @returns {number|undefined} + */ + private _getLastThumbnailTime(metadata: IThumbnailTrackMetadata): number | undefined { + if (metadata.start === undefined || metadata.tileDuration === undefined) { + return; + } + + const maximumPosition = this.getMaximumPosition() ?? undefined; + if (maximumPosition === undefined) { + return; + } + + const thumbnailsPerSegment = metadata.horizontalTiles * metadata.verticalTiles; + + /** Amount of seconds a segment of thumbnails span. */ + const segmentDuration = metadata.tileDuration * (thumbnailsPerSegment ?? 1); + + /** + * Seconds at the end of the content for which a thumbnail has not yet been + * generated. + */ + const secondsWithoutThumbnailYet = + (maximumPosition - metadata.start) % segmentDuration; + + /** + * Position that will lead to the last available thumbnail being requested. + */ + const maxThumbnailTime = + Math.min(maximumPosition - secondsWithoutThumbnailYet, metadata.end ?? Infinity) - + metadata.tileDuration; + + return maxThumbnailTime; + } + /** * Returns `true` if the content concerned by those options should load in * multithread mode. diff --git a/src/public_types.ts b/src/public_types.ts index 0669d9ab37..a9f1d9bb86 100644 --- a/src/public_types.ts +++ b/src/public_types.ts @@ -1467,6 +1467,11 @@ export interface IThumbnailTrackInfo { * (The first thumbnail in that segment for 22-25, the second for 25-26) */ thumbnailDuration: number | undefined; + /** + * The time the current last thumbnail avaiable for that track applies to, + * in seconds. + */ + lastThumbnailTime: number | undefined; } /** diff --git a/tests/integration/scenarios/dash_render_thumbnail.test.js b/tests/integration/scenarios/dash_render_thumbnail.test.js index ed2c43423d..85e3aa3518 100644 --- a/tests/integration/scenarios/dash_render_thumbnail.test.js +++ b/tests/integration/scenarios/dash_render_thumbnail.test.js @@ -74,6 +74,11 @@ function runDashRenderThumbnailTests({ multithread } = {}) { width: 320, height: 180, mimeType: "image/jpeg", + end: 101.568367, + lastThumbnailTime: 99, + start: 0, + thumbnailDuration: 1, + thumbnailsPerSegment: 10, }, ]);