Skip to content

Commit

Permalink
feat: Add thumbnails support in src= (#5802)
Browse files Browse the repository at this point in the history 
Closes #5778
  • Loading branch information
@avelad
avelad authored Oct 26, 2023
1 parent a8aa4ab commit 88e4cd4
Show file tree
Hide file tree
Showing 4 changed files with 185 additions and 92 deletions.
19 changes: 19 additions & 0 deletions demo/common/assets.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -978,6 +978,25 @@ shakaAssets.testAssets = [
.addFeature(shakaAssets.Feature.MP4)
.addFeature(shakaAssets.Feature.THUMBNAILS)
.addExtraThumbnail('https://cdn.bitmovin.com/content/assets/art-of-motion-dash-hls-progressive/thumbnails/f08e80da-bf1d-4e3d-8899-f0f6155f6efa.vtt'),
new ShakaDemoAssetInfo(
/* name= */ 'Art of Motion (HLS) (external thumbnails)',
/* iconUri= */ 'https://storage.googleapis.com/shaka-asset-icons/art_of_motion.png',
/* manifestUri= */ 'https://cdn.bitmovin.com/content/assets/art-of-motion-dash-hls-progressive/m3u8s/f08e80da-bf1d-4e3d-8899-f0f6155f6efa.m3u8',
/* source= */ shakaAssets.Source.BITCODIN)
.addFeature(shakaAssets.Feature.HLS)
.addFeature(shakaAssets.Feature.HIGH_DEFINITION)
.addFeature(shakaAssets.Feature.MP2TS)
.addFeature(shakaAssets.Feature.THUMBNAILS)
.addExtraThumbnail('https://cdn.bitmovin.com/content/assets/art-of-motion-dash-hls-progressive/thumbnails/f08e80da-bf1d-4e3d-8899-f0f6155f6efa.vtt'),
new ShakaDemoAssetInfo(
/* name= */ 'Art of Motion (MP4) (external thumbnails)',
/* iconUri= */ 'https://storage.googleapis.com/shaka-asset-icons/art_of_motion.png',
/* manifestUri= */ 'https://cdn.bitmovin.com/content/assets/art-of-motion-dash-hls-progressive/MI201109210084_mpeg-4_hd_high_1080p25_10mbits.mp4',
/* source= */ shakaAssets.Source.BITCODIN)
.addFeature(shakaAssets.Feature.HIGH_DEFINITION)
.addFeature(shakaAssets.Feature.MP4)
.addFeature(shakaAssets.Feature.THUMBNAILS)
.addExtraThumbnail('https://cdn.bitmovin.com/content/assets/art-of-motion-dash-hls-progressive/thumbnails/f08e80da-bf1d-4e3d-8899-f0f6155f6efa.vtt'),
// End bitcodin assets }}}

// MetaCDN assets {{{
Expand Down
186 changes: 99 additions & 87 deletions lib/player.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -596,6 +596,9 @@ shaka.Player = class extends shaka.util.FakeEventTarget {
*/
this.nextExternalStreamId_ = 1e9;

/** @private {!Array.<shaka.extern.Stream>} */
this.externalSrcEqualsThumbnailsStreams_ = [];

/** @private {?shaka.extern.PlayerConfiguration} */
this.config_ = this.defaultConfig_();

Expand Down Expand Up @@ -1701,6 +1704,8 @@ shaka.Player = class extends shaka.util.FakeEventTarget {
this.stats_ = new shaka.util.Stats(); // Replace with a clean stats object.
this.lastTextFactory_ = null;

this.externalSrcEqualsThumbnailsStreams_ = [];

// Make sure that the app knows of the new buffering state.
this.updateBufferState_();
}
Expand Down Expand Up @@ -3961,13 +3966,12 @@ shaka.Player = class extends shaka.util.FakeEventTarget {
* @export
*/
getImageTracks() {
const StreamUtils = shaka.util.StreamUtils;
let imageStreams = this.externalSrcEqualsThumbnailsStreams_;
if (this.manifest_) {
const imageStreams = this.manifest_.imageStreams;
const StreamUtils = shaka.util.StreamUtils;
return imageStreams.map((image) => StreamUtils.imageStreamToTrack(image));
} else {
return [];
imageStreams = this.manifest_.imageStreams;
}
return imageStreams.map((image) => StreamUtils.imageStreamToTrack(image));
}

/**
Expand All @@ -3980,10 +3984,15 @@ shaka.Player = class extends shaka.util.FakeEventTarget {
* @export
*/
async getAllThumbnails(trackId) {
if (!this.manifest_) {
if (this.loadMode_ != shaka.Player.LoadMode.MEDIA_SOURCE &&
this.loadMode_ != shaka.Player.LoadMode.SRC_EQUALS) {
return null;
}
const imageStream = this.manifest_.imageStreams.find(
let imageStreams = this.externalSrcEqualsThumbnailsStreams_;
if (this.manifest_) {
imageStreams = this.manifest_.imageStreams;
}
const imageStream = imageStreams.find(
(stream) => stream.id == trackId);
if (!imageStream) {
return null;
Expand Down Expand Up @@ -4046,77 +4055,82 @@ shaka.Player = class extends shaka.util.FakeEventTarget {
* @export
*/
async getThumbnails(trackId, time) {
if (this.loadMode_ != shaka.Player.LoadMode.MEDIA_SOURCE &&
this.loadMode_ != shaka.Player.LoadMode.SRC_EQUALS) {
return null;
}
let imageStreams = this.externalSrcEqualsThumbnailsStreams_;
if (this.manifest_) {
const imageStream = this.manifest_.imageStreams.find(
(stream) => stream.id == trackId);
if (!imageStream) {
return null;
}
if (!imageStream.segmentIndex) {
await imageStream.createSegmentIndex();
}
const referencePosition = imageStream.segmentIndex.find(time);
if (referencePosition == null) {
return null;
}
const reference = imageStream.segmentIndex.get(referencePosition);
const dimensions = this.parseTilesLayout_(
reference.getTilesLayout() || imageStream.tilesLayout);
if (!dimensions) {
return null;
}
const fullImageWidth = imageStream.width || 0;
const fullImageHeight = imageStream.height || 0;
let width = fullImageWidth / dimensions.columns;
let height = fullImageHeight / dimensions.rows;
const totalImages = dimensions.columns * dimensions.rows;
const segmentDuration = reference.trueEndTime - reference.startTime;
const thumbnailDuration =
reference.getTileDuration() || (segmentDuration / totalImages);
let thumbnailTime = reference.startTime;
let positionX = 0;
let positionY = 0;
// If the number of images in the segment is greater than 1, we have to
// find the correct image. For that we will return to the app the
// coordinates of the position of the correct image.
// Image search is always from left to right and top to bottom.
// Note: The time between images within the segment is always
// equidistant.
//
// Eg: Total images 5, tileLayout 5x1, segmentDuration 5, thumbnailTime 2
// positionX = 0.4 * fullImageWidth
// positionY = 0
if (totalImages > 1) {
const thumbnailPosition =
Math.floor((time - reference.startTime) / thumbnailDuration);
thumbnailTime = reference.startTime +
(thumbnailPosition * thumbnailDuration);
positionX = (thumbnailPosition % dimensions.columns) * width;
positionY = Math.floor(thumbnailPosition / dimensions.columns) * height;
}
let sprite = false;
const thumbnailSprite = reference.getThumbnailSprite();
if (thumbnailSprite) {
sprite = true;
height = thumbnailSprite.height;
positionX = thumbnailSprite.positionX;
positionY = thumbnailSprite.positionY;
width = thumbnailSprite.width;
}
return {
imageHeight: fullImageHeight,
imageWidth: fullImageWidth,
height: height,
positionX: positionX,
positionY: positionY,
startTime: thumbnailTime,
duration: thumbnailDuration,
uris: reference.getUris(),
width: width,
sprite: sprite,
};
imageStreams = this.manifest_.imageStreams;
}
return null;
const imageStream = imageStreams.find(
(stream) => stream.id == trackId);
if (!imageStream) {
return null;
}
if (!imageStream.segmentIndex) {
await imageStream.createSegmentIndex();
}
const referencePosition = imageStream.segmentIndex.find(time);
if (referencePosition == null) {
return null;
}
const reference = imageStream.segmentIndex.get(referencePosition);
const dimensions = this.parseTilesLayout_(
reference.getTilesLayout() || imageStream.tilesLayout);
if (!dimensions) {
return null;
}
const fullImageWidth = imageStream.width || 0;
const fullImageHeight = imageStream.height || 0;
let width = fullImageWidth / dimensions.columns;
let height = fullImageHeight / dimensions.rows;
const totalImages = dimensions.columns * dimensions.rows;
const segmentDuration = reference.trueEndTime - reference.startTime;
const thumbnailDuration =
reference.getTileDuration() || (segmentDuration / totalImages);
let thumbnailTime = reference.startTime;
let positionX = 0;
let positionY = 0;
// If the number of images in the segment is greater than 1, we have to
// find the correct image. For that we will return to the app the
// coordinates of the position of the correct image.
// Image search is always from left to right and top to bottom.
// Note: The time between images within the segment is always
// equidistant.
//
// Eg: Total images 5, tileLayout 5x1, segmentDuration 5, thumbnailTime 2
// positionX = 0.4 * fullImageWidth
// positionY = 0
if (totalImages > 1) {
const thumbnailPosition =
Math.floor((time - reference.startTime) / thumbnailDuration);
thumbnailTime = reference.startTime +
(thumbnailPosition * thumbnailDuration);
positionX = (thumbnailPosition % dimensions.columns) * width;
positionY = Math.floor(thumbnailPosition / dimensions.columns) * height;
}
let sprite = false;
const thumbnailSprite = reference.getThumbnailSprite();
if (thumbnailSprite) {
sprite = true;
height = thumbnailSprite.height;
positionX = thumbnailSprite.positionX;
positionY = thumbnailSprite.positionY;
width = thumbnailSprite.width;
}
return {
imageHeight: fullImageHeight,
imageWidth: fullImageWidth,
height: height,
positionX: positionX,
positionY: positionY,
startTime: thumbnailTime,
duration: thumbnailDuration,
uris: reference.getUris(),
width: width,
sprite: sprite,
};
}

/**
Expand Down Expand Up @@ -5015,14 +5029,6 @@ shaka.Player = class extends shaka.util.FakeEventTarget {
shaka.util.Error.Code.CONTENT_NOT_LOADED);
}

if (this.loadMode_ == shaka.Player.LoadMode.SRC_EQUALS) {
shaka.log.error('Cannot add this thumbnail track when loaded with src=');
throw new shaka.util.Error(
shaka.util.Error.Severity.RECOVERABLE,
shaka.util.Error.Category.TEXT,
shaka.util.Error.Code.CANNOT_ADD_EXTERNAL_THUMBNAILS_TO_SRC_EQUALS);
}

if (!mimeType) {
mimeType = await this.getTextMimetype_(uri);
}
Expand All @@ -5036,8 +5042,10 @@ shaka.Player = class extends shaka.util.FakeEventTarget {
}

const ContentType = shaka.util.ManifestParserUtils.ContentType;

const duration = this.manifest_.presentationTimeline.getDuration();
let duration = this.video_.duration;
if (this.manifest_) {
duration = this.manifest_.presentationTimeline.getDuration();
}
if (duration == Infinity) {
throw new shaka.util.Error(
shaka.util.Error.Severity.RECOVERABLE,
Expand Down Expand Up @@ -5129,7 +5137,11 @@ shaka.Player = class extends shaka.util.FakeEventTarget {
external: true,
};

this.manifest_.imageStreams.push(stream);
if (this.loadMode_ == shaka.Player.LoadMode.SRC_EQUALS) {
this.externalSrcEqualsThumbnailsStreams_.push(stream);
} else {
this.manifest_.imageStreams.push(stream);
}
this.onTracksChanged_();
return shaka.util.StreamUtils.imageStreamToTrack(stream);
}
Expand Down
6 changes: 1 addition & 5 deletions lib/util/error.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -360,11 +360,7 @@ shaka.util.Error.Code = {
*/
'CHAPTERS_TRACK_FAILED': 2015,

/**
* External thumbnails tracks cannot be added in src= because native platform
* doesn't support it.
*/
'CANNOT_ADD_EXTERNAL_THUMBNAILS_TO_SRC_EQUALS': 2016,
// RETIRED: 'CANNOT_ADD_EXTERNAL_THUMBNAILS_TO_SRC_EQUALS': 2016,

/**
* Only external urls of WebVTT type are supported.
Expand Down
66 changes: 66 additions & 0 deletions test/player_src_equals_integration.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -422,6 +422,72 @@ describe('Player Src Equals', () => {
expect(player.getManifest()).toBeFalsy();
});

describe('addThumbnailsTrack', () => {
it('appends thumbnails for external thumbnails with sprites',
async () => {
await loadWithSrcEquals(SMALL_MP4_CONTENT_URI, /* startTime= */ null);
const locationUri = new goog.Uri(location.href);
const partialUri =
new goog.Uri('/base/test/test/assets/thumbnails-sprites.vtt');
const absoluteUri = locationUri.resolve(partialUri);
const newTrack =
await player.addThumbnailsTrack(absoluteUri.toString());

expect(player.getImageTracks()).toEqual([newTrack]);

const thumbnail1 = await player.getThumbnails(newTrack.id, 0);
expect(thumbnail1.startTime).toBe(0);
expect(thumbnail1.duration).toBe(5);
expect(thumbnail1.height).toBe(90);
expect(thumbnail1.positionX).toBe(0);
expect(thumbnail1.positionY).toBe(0);
expect(thumbnail1.width).toBe(160);
const thumbnail2 = await player.getThumbnails(newTrack.id, 10);
expect(thumbnail2.startTime).toBe(5);
expect(thumbnail2.duration).toBe(25);
expect(thumbnail2.height).toBe(90);
expect(thumbnail2.positionX).toBe(160);
expect(thumbnail2.positionY).toBe(0);
expect(thumbnail2.width).toBe(160);
const thumbnail3 = await player.getThumbnails(newTrack.id, 40);
expect(thumbnail3.startTime).toBe(30);
expect(thumbnail3.duration).toBe(30);
expect(thumbnail3.height).toBe(90);
expect(thumbnail3.positionX).toBe(160);
expect(thumbnail3.positionY).toBe(90);
expect(thumbnail3.width).toBe(160);

const thumbnails = await player.getAllThumbnails(newTrack.id);
expect(thumbnails.length).toBe(3);
});

it('appends thumbnails for external thumbnails without sprites',
async () => {
await loadWithSrcEquals(SMALL_MP4_CONTENT_URI, /* startTime= */ null);
const locationUri = new goog.Uri(location.href);
const partialUri =
new goog.Uri('/base/test/test/assets/thumbnails.vtt');
const absoluteUri = locationUri.resolve(partialUri);
const newTrack =
await player.addThumbnailsTrack(absoluteUri.toString());

expect(player.getImageTracks()).toEqual([newTrack]);

const thumbnail1 = await player.getThumbnails(newTrack.id, 0);
expect(thumbnail1.startTime).toBe(0);
expect(thumbnail1.duration).toBe(5);
const thumbnail2 = await player.getThumbnails(newTrack.id, 10);
expect(thumbnail2.startTime).toBe(5);
expect(thumbnail2.duration).toBe(25);
const thumbnail3 = await player.getThumbnails(newTrack.id, 40);
expect(thumbnail3.startTime).toBe(30);
expect(thumbnail3.duration).toBe(30);

const thumbnails = await player.getAllThumbnails(newTrack.id);
expect(thumbnails.length).toBe(3);
});
}); // describe('addThumbnailsTrack')

/**
* @param {string} contentUri
* @param {?number} startTime
Expand Down

0 comments on commit 88e4cd4

Please sign in to comment.