From dd29347af831dbbad69279de58456f991b0b04af Mon Sep 17 00:00:00 2001 From: Ryan McCartney Date: Wed, 18 Sep 2024 18:17:22 +0100 Subject: [PATCH 01/11] Feat: Support MSE Subtitles for Low Latency --- src/playbackstrategy/msestrategy.js | 29 +++++++++++++++++- src/playercomponent.js | 5 ++++ src/subtitles/dashsubtitles.js | 46 +++++++++++++++++++++++++++++ src/subtitles/subtitles.js | 12 ++++++++ 4 files changed, 91 insertions(+), 1 deletion(-) create mode 100644 src/subtitles/dashsubtitles.js diff --git a/src/playbackstrategy/msestrategy.js b/src/playbackstrategy/msestrategy.js index 0aaa603c..89578206 100644 --- a/src/playbackstrategy/msestrategy.js +++ b/src/playbackstrategy/msestrategy.js @@ -93,6 +93,7 @@ function MSEStrategy(mediaSources, windowType, mediaKind, playbackElement, isUHD STREAM_INITIALIZED: "streamInitialized", FRAGMENT_CONTENT_LENGTH_MISMATCH: "fragmentContentLengthMismatch", QUOTA_EXCEEDED: "quotaExceeded", + TEXT_TRACKS_ADDED: "allTextTracksAdded", } function onLoadedMetaData() { @@ -491,10 +492,16 @@ function MSEStrategy(mediaSources, windowType, mediaKind, playbackElement, isUHD function setUpMediaPlayer(playbackTime) { const dashSettings = getDashSettings(playerSettings) + const dashSubs = window.bigscreenPlayer?.overrides?.dashSubtitles ?? false mediaPlayer = MediaPlayer().create() mediaPlayer.updateSettings(dashSettings) mediaPlayer.initialize(mediaElement, null, true) + + if (dashSubs) { + mediaPlayer.attachTTMLRenderingDiv(document.querySelector("#bsp_subtitles")) + } + modifySource(playbackTime) } @@ -504,7 +511,6 @@ function MSEStrategy(mediaSources, windowType, mediaKind, playbackElement, isUHD windowType, initialSeekableRangeStartSeconds: mediaSources.time().windowStartTime / 1000, }) - mediaPlayer.attachSource(`${source}${anchor}`) } @@ -540,6 +546,21 @@ function MSEStrategy(mediaSources, windowType, mediaKind, playbackElement, isUHD mediaPlayer.on(DashJSEvents.GAP_JUMP, onGapJump) mediaPlayer.on(DashJSEvents.GAP_JUMP_TO_END, onGapJump) mediaPlayer.on(DashJSEvents.QUOTA_EXCEEDED, onQuotaExceeded) + mediaPlayer.on(DashJSEvents.TEXT_TRACKS_ADDED, disableTextTracks) + } + + function disableTextTracks() { + const textTracks = mediaElement.textTracks + for (let index = 0; index < textTracks.length; index++) { + textTracks[index].mode = "disabled" + } + } + + function enableTextTracks() { + const textTracks = mediaElement.textTracks + for (let index = 0; index < textTracks.length; index++) { + textTracks[index].mode = "showing" + } } function getSeekableRange() { @@ -645,6 +666,12 @@ function MSEStrategy(mediaSources, windowType, mediaKind, playbackElement, isUHD getSeekableRange, getCurrentTime, getDuration, + setSubtitles: (state) => { + if (state) { + enableTextTracks() + } + mediaPlayer.enableText(state) + }, getPlayerElement: () => mediaElement, tearDown: () => { mediaPlayer.reset() diff --git a/src/playercomponent.js b/src/playercomponent.js index 54784ace..b7657c9b 100644 --- a/src/playercomponent.js +++ b/src/playercomponent.js @@ -71,6 +71,10 @@ function PlayerComponent( } } + function setSubtitles(state) { + return playbackStrategy && playbackStrategy.setSubtitles(state) + } + function getDuration() { return playbackStrategy && playbackStrategy.getDuration() } @@ -394,6 +398,7 @@ function PlayerComponent( return { play, pause, + setSubtitles, transitions, isEnded, setPlaybackRate, diff --git a/src/subtitles/dashsubtitles.js b/src/subtitles/dashsubtitles.js new file mode 100644 index 00000000..7847e352 --- /dev/null +++ b/src/subtitles/dashsubtitles.js @@ -0,0 +1,46 @@ +function DashSubtitles(mediaPlayer, autoStart, parentElement) { + let currentSubtitlesElement + + if (autoStart) { + start() + } + + function removeCurrentSubtitlesElement() { + if (currentSubtitlesElement) { + DOMHelpers.safeRemoveElement(currentSubtitlesElement) + currentSubtitlesElement = undefined + } + } + + function addCurrentSubtitlesElement() { + removeCurrentSubtitlesElement() + currentSubtitlesElement = document.createElement("div") + currentSubtitlesElement.id = "bsp_subtitles" + currentSubtitlesElement.style.position = "absolute" + parentElement.appendChild(currentSubtitlesElement) + } + + function start() { + mediaPlayer.setSubtitles(true) + if (!currentSubtitlesElement) { + addCurrentSubtitlesElement() + } + } + + function stop() { + mediaPlayer.setSubtitles(false) + } + + addCurrentSubtitlesElement() + + return { + start, + stop, + customise: () => {}, + tearDown: () => { + stop() + }, + } +} + +export default DashSubtitles diff --git a/src/subtitles/subtitles.js b/src/subtitles/subtitles.js index 1ffff7a4..2d33d9a2 100644 --- a/src/subtitles/subtitles.js +++ b/src/subtitles/subtitles.js @@ -3,6 +3,8 @@ import findSegmentTemplate from "../utils/findtemplate" function Subtitles(mediaPlayer, autoStart, playbackElement, defaultStyleOpts, mediaSources, callback) { const useLegacySubs = window.bigscreenPlayer?.overrides?.legacySubtitles ?? false + const dashSubs = window.bigscreenPlayer?.overrides?.dashSubtitles ?? false + const isSeekableLiveSupport = window.bigscreenPlayer.liveSupport == null || window.bigscreenPlayer.liveSupport === "seekable" @@ -19,6 +21,16 @@ function Subtitles(mediaPlayer, autoStart, playbackElement, defaultStyleOpts, me .catch(() => { Plugins.interface.onSubtitlesDynamicLoadError() }) + } + if (dashSubs) { + import("./dashsubtitles.js") + .then(({ default: DashSubtitles }) => { + subtitlesContainer = DashSubtitles(mediaPlayer, autoStart, playbackElement, mediaSources, defaultStyleOpts) + callback(subtitlesEnabled) + }) + .catch(() => { + Plugins.interface.onSubtitlesDynamicLoadError() + }) } else { import("./imscsubtitles.js") .then(({ default: IMSCSubtitles }) => { From f9b1227e90559dad310f03304ca9088fa6757af2 Mon Sep 17 00:00:00 2001 From: Ryan McCartney Date: Wed, 18 Sep 2024 22:32:54 +0100 Subject: [PATCH 02/11] Add tests for new subtitle type --- src/subtitles/subtitles.js | 5 ++--- src/subtitles/subtitles.test.js | 39 +++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/src/subtitles/subtitles.js b/src/subtitles/subtitles.js index 2d33d9a2..1580f4e5 100644 --- a/src/subtitles/subtitles.js +++ b/src/subtitles/subtitles.js @@ -21,8 +21,7 @@ function Subtitles(mediaPlayer, autoStart, playbackElement, defaultStyleOpts, me .catch(() => { Plugins.interface.onSubtitlesDynamicLoadError() }) - } - if (dashSubs) { + } else if (dashSubs) { import("./dashsubtitles.js") .then(({ default: DashSubtitles }) => { subtitlesContainer = DashSubtitles(mediaPlayer, autoStart, playbackElement, mediaSources, defaultStyleOpts) @@ -87,7 +86,7 @@ function Subtitles(mediaPlayer, autoStart, playbackElement, defaultStyleOpts, me const isWhole = findSegmentTemplate(url) == null - return isWhole || (!useLegacySubs && isSeekableLiveSupport) + return isWhole || (!useLegacySubs && !dashSubs && isSeekableLiveSupport) } function setPosition(position) { diff --git a/src/subtitles/subtitles.test.js b/src/subtitles/subtitles.test.js index 250a78dc..c6d140b5 100644 --- a/src/subtitles/subtitles.test.js +++ b/src/subtitles/subtitles.test.js @@ -1,10 +1,13 @@ /* eslint-disable jest/no-done-callback */ import IMSCSubtitles from "./imscsubtitles" import LegacySubtitles from "./legacysubtitles" +import DashSubtitles from "./dashsubtitles" + import Subtitles from "./subtitles" jest.mock("./imscsubtitles") jest.mock("./legacysubtitles") +jest.mock("./dashsubtitles") describe("Subtitles", () => { let isAvailable @@ -74,6 +77,42 @@ describe("Subtitles", () => { }) }) + describe("dash", () => { + beforeEach(() => { + window.bigscreenPlayer = { + overrides: { + dashSubtitles: true, + }, + } + + DashSubtitles.mockReset() + }) + + it("implementation is available when dash subtitles override is true", (done) => { + const mockMediaPlayer = {} + const autoStart = true + + Subtitles(mockMediaPlayer, autoStart, playbackElement, null, mockMediaSources, (result) => { + expect(result).toBe(true) + expect(DashSubtitles).toHaveBeenCalledTimes(1) + done() + }) + }) + + it("implementation is not available when dash subtitles override is true, but subtitles are segmented", (done) => { + isSegmented = true + const mockMediaPlayer = {} + const autoStart = true + + Subtitles(mockMediaPlayer, autoStart, playbackElement, null, mockMediaSources, () => { + expect(LegacySubtitles).not.toHaveBeenCalled() + expect(IMSCSubtitles).not.toHaveBeenCalled() + expect(DashSubtitles).not.toHaveBeenCalled() + done() + }) + }) + }) + describe("imscjs", () => { it("implementation is available when legacy subtitles override is false", (done) => { const mockMediaPlayer = {} From 80d4851d28c86259f5739d9d34057910b04e6741 Mon Sep 17 00:00:00 2001 From: Ryan McCartney Date: Thu, 19 Sep 2024 09:20:23 +0100 Subject: [PATCH 03/11] Subtitles: low-latency docs --- docs/tutorials/Subtitles.md | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/docs/tutorials/Subtitles.md b/docs/tutorials/Subtitles.md index a9747158..50bdf181 100644 --- a/docs/tutorials/Subtitles.md +++ b/docs/tutorials/Subtitles.md @@ -10,10 +10,12 @@ You provide subtitles to BigscreenPlayer by setting `media.captions` in the `.in ```js // 1️⃣ Add an array of caption blocks to your playback data. -playbackData.media.captions = [/* caption blocks... */]; +playbackData.media.captions = [ + /* caption blocks... */ +] // 2️⃣ Pass playback data that contains captions to the player. -player.init(document.querySelector("video"), playbackData, /* other opts */); +player.init(document.querySelector("video"), playbackData /* other opts */) ``` 1. `media.captions` MUST be an array containing at least one object. @@ -34,7 +36,7 @@ const captions = [ { url: "https://some.cdn/subtitles.xml" }, { url: "https://other.cdn/subtitles.xml" }, /* ... */ -]; +] ``` Subtitles delivered as a whole do not require any additional metadata in the manifest to work. @@ -46,7 +48,7 @@ Subtitles are delivered "as segments" when the captions' `url` is an URL templat ```js // Each segment specifies subtitles for a segment of the media experience. const captions = [ - { + { url: "https://some.cdn/subtitles/$segment$.m4s", segmentLength: 3.84, }, @@ -55,7 +57,7 @@ const captions = [ segmentLength: 3.84, }, /* ... */ -]; +] ``` The segment number is calculated from the presentation timeline. You MUST ensure your subtitle segments are enumerated to match your media segments and you account for offsets such as: @@ -73,12 +75,24 @@ You can style the subtitles by setting `media.subtitleCustomisation` in the `.in ```js // 1️⃣ Create an object mapping out styles for your subtitles. -playbackData.media.subtitleCustomisation = { lineHeight: 1.5, size: 1 }; +playbackData.media.subtitleCustomisation = { lineHeight: 1.5, size: 1 } // 2️⃣ Pass playback data that contains subtitle customisation (and captions) to the player. -player.init(document.querySelector("video"), playbackData, /* other opts */); +player.init(document.querySelector("video"), playbackData /* other opts */) ``` +### Low Latency Streams + +When using Dash.js with a low-latency MPD segments are delivered using Chunked Transfer Encoding (CTE) - the current side chain doesn't allow for delivery in this case. + +Whilst it is possible to collect chunks as they are delivered, wait until a full segment worth of subtitles have been delivered and pass these to the render function this breaks the low-latency workflow. + +An override has been added to allow subtitles to be rendered directly by Dash.js instead of the current side-chain. + +Subtitles can be enabled and disabled in the usual way using the `setSubtitlesEnabled()` function. However, they are signalled and delivered by the chosen MPD. + +Using Dash.js subtitles can be enabled using `window.bigscreenPlayer.overrides.dashSubtitles = true`. + ##  Design ### Why not include subtitles in the manifest? From 0d68c699565f67e6f817d20036b646bcee022796 Mon Sep 17 00:00:00 2001 From: Ryan McCartney Date: Mon, 23 Sep 2024 12:09:45 +0100 Subject: [PATCH 04/11] Subtitles: low-latency docs --- docs/tutorials/Subtitles.md | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/docs/tutorials/Subtitles.md b/docs/tutorials/Subtitles.md index a9747158..50bdf181 100644 --- a/docs/tutorials/Subtitles.md +++ b/docs/tutorials/Subtitles.md @@ -10,10 +10,12 @@ You provide subtitles to BigscreenPlayer by setting `media.captions` in the `.in ```js // 1️⃣ Add an array of caption blocks to your playback data. -playbackData.media.captions = [/* caption blocks... */]; +playbackData.media.captions = [ + /* caption blocks... */ +] // 2️⃣ Pass playback data that contains captions to the player. -player.init(document.querySelector("video"), playbackData, /* other opts */); +player.init(document.querySelector("video"), playbackData /* other opts */) ``` 1. `media.captions` MUST be an array containing at least one object. @@ -34,7 +36,7 @@ const captions = [ { url: "https://some.cdn/subtitles.xml" }, { url: "https://other.cdn/subtitles.xml" }, /* ... */ -]; +] ``` Subtitles delivered as a whole do not require any additional metadata in the manifest to work. @@ -46,7 +48,7 @@ Subtitles are delivered "as segments" when the captions' `url` is an URL templat ```js // Each segment specifies subtitles for a segment of the media experience. const captions = [ - { + { url: "https://some.cdn/subtitles/$segment$.m4s", segmentLength: 3.84, }, @@ -55,7 +57,7 @@ const captions = [ segmentLength: 3.84, }, /* ... */ -]; +] ``` The segment number is calculated from the presentation timeline. You MUST ensure your subtitle segments are enumerated to match your media segments and you account for offsets such as: @@ -73,12 +75,24 @@ You can style the subtitles by setting `media.subtitleCustomisation` in the `.in ```js // 1️⃣ Create an object mapping out styles for your subtitles. -playbackData.media.subtitleCustomisation = { lineHeight: 1.5, size: 1 }; +playbackData.media.subtitleCustomisation = { lineHeight: 1.5, size: 1 } // 2️⃣ Pass playback data that contains subtitle customisation (and captions) to the player. -player.init(document.querySelector("video"), playbackData, /* other opts */); +player.init(document.querySelector("video"), playbackData /* other opts */) ``` +### Low Latency Streams + +When using Dash.js with a low-latency MPD segments are delivered using Chunked Transfer Encoding (CTE) - the current side chain doesn't allow for delivery in this case. + +Whilst it is possible to collect chunks as they are delivered, wait until a full segment worth of subtitles have been delivered and pass these to the render function this breaks the low-latency workflow. + +An override has been added to allow subtitles to be rendered directly by Dash.js instead of the current side-chain. + +Subtitles can be enabled and disabled in the usual way using the `setSubtitlesEnabled()` function. However, they are signalled and delivered by the chosen MPD. + +Using Dash.js subtitles can be enabled using `window.bigscreenPlayer.overrides.dashSubtitles = true`. + ##  Design ### Why not include subtitles in the manifest? From 6279f29593be8a4fba47a2b3f39ee1c90f2c06dc Mon Sep 17 00:00:00 2001 From: Ryan McCartney Date: Tue, 24 Sep 2024 11:57:58 +0100 Subject: [PATCH 05/11] Don't require a captions object when dashSubtitles override is used --- src/subtitles/subtitles.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/subtitles/subtitles.js b/src/subtitles/subtitles.js index 1580f4e5..d22a4e28 100644 --- a/src/subtitles/subtitles.js +++ b/src/subtitles/subtitles.js @@ -78,6 +78,10 @@ function Subtitles(mediaPlayer, autoStart, playbackElement, defaultStyleOpts, me } function available() { + if (dashSubs) { + return true + } + const url = mediaSources.currentSubtitlesSource() if (!(typeof url === "string" && url !== "")) { @@ -86,7 +90,7 @@ function Subtitles(mediaPlayer, autoStart, playbackElement, defaultStyleOpts, me const isWhole = findSegmentTemplate(url) == null - return isWhole || (!useLegacySubs && !dashSubs && isSeekableLiveSupport) + return isWhole || (!useLegacySubs && isSeekableLiveSupport) } function setPosition(position) { From db5e2d6e3d80d66d73d9069baa371ef4f8d50b78 Mon Sep 17 00:00:00 2001 From: Ryan McCartney Date: Fri, 27 Sep 2024 14:29:08 +0100 Subject: [PATCH 06/11] Add tests for DashSubtitles --- src/subtitles/dashsubtitles.js | 10 ++-- src/subtitles/dashsubtitles.test.js | 84 +++++++++++++++++++++++++++++ src/subtitles/subtitles.test.js | 4 +- 3 files changed, 93 insertions(+), 5 deletions(-) create mode 100644 src/subtitles/dashsubtitles.test.js diff --git a/src/subtitles/dashsubtitles.js b/src/subtitles/dashsubtitles.js index 7847e352..a1fae6a2 100644 --- a/src/subtitles/dashsubtitles.js +++ b/src/subtitles/dashsubtitles.js @@ -1,3 +1,5 @@ +import DOMHelpers from "../domhelpers" + function DashSubtitles(mediaPlayer, autoStart, parentElement) { let currentSubtitlesElement @@ -31,15 +33,17 @@ function DashSubtitles(mediaPlayer, autoStart, parentElement) { mediaPlayer.setSubtitles(false) } + function tearDown() { + stop() + } + addCurrentSubtitlesElement() return { start, stop, customise: () => {}, - tearDown: () => { - stop() - }, + tearDown, } } diff --git a/src/subtitles/dashsubtitles.test.js b/src/subtitles/dashsubtitles.test.js new file mode 100644 index 00000000..0677711b --- /dev/null +++ b/src/subtitles/dashsubtitles.test.js @@ -0,0 +1,84 @@ +import DashSubtitles from "./dashsubtitles" + +const UPDATE_INTERVAL = 750 + +describe("Dash Subtitles", () => { + let subtitles + let targetElement + + const mockMediaPlayer = { + getCurrentTime: jest.fn(), + setSubtitles: jest.fn(), + } + + beforeEach(() => { + jest.useFakeTimers() + jest.clearAllMocks() + jest.clearAllTimers() + + // Reset the target HTML element between each test + targetElement?.remove() + targetElement = document.createElement("div") + + jest.spyOn(targetElement, "clientWidth", "get").mockReturnValue(200) + jest.spyOn(targetElement, "clientHeight", "get").mockReturnValue(100) + jest.spyOn(targetElement, "removeChild") + + document.body.appendChild(targetElement) + + // Reset instance + subtitles?.tearDown() + subtitles = null + mockMediaPlayer.setSubtitles.mockClear() + }) + + function progressTime(mediaPlayerTime) { + mockMediaPlayer.getCurrentTime.mockReturnValue(mediaPlayerTime) + jest.advanceTimersByTime(UPDATE_INTERVAL) + } + + describe("construction", () => { + it("returns the correct interface", () => { + const autoStart = false + + subtitles = DashSubtitles(mockMediaPlayer, autoStart, targetElement) + + expect(subtitles).toEqual( + expect.objectContaining({ + start: expect.any(Function), + stop: expect.any(Function), + customise: expect.any(Function), + tearDown: expect.any(Function), + }) + ) + }) + + it("Expect TTML rendering div to have been created", () => { + const autoStart = false + subtitles = DashSubtitles(mockMediaPlayer, autoStart, targetElement) + + progressTime(1.5) + expect(targetElement.querySelector("#bsp_subtitles")).toBeTruthy() + }) + }) + + describe("autoplay", () => { + it("triggers the MSE player to enable subtitles immediately when set to autoplay", () => { + const autoStart = true + + subtitles = DashSubtitles(mockMediaPlayer, autoStart, targetElement) + + progressTime(1.5) + expect(mockMediaPlayer.setSubtitles).toHaveBeenCalledTimes(1) + }) + + it("does not trigger the MSE player to enable subtitles immediately when set to autoplay", () => { + const autoStart = false + + subtitles = DashSubtitles(mockMediaPlayer, autoStart, targetElement) + + progressTime(1.5) + expect(mockMediaPlayer.setSubtitles).toHaveBeenCalledTimes(0) + }) + }) +}) diff --git a/src/subtitles/subtitles.test.js b/src/subtitles/subtitles.test.js index c6d140b5..467e7dd1 100644 --- a/src/subtitles/subtitles.test.js +++ b/src/subtitles/subtitles.test.js @@ -99,7 +99,7 @@ describe("Subtitles", () => { }) }) - it("implementation is not available when dash subtitles override is true, but subtitles are segmented", (done) => { + it("implementation is available when dash subtitles override is true, even if segmented URL is passed", (done) => { isSegmented = true const mockMediaPlayer = {} const autoStart = true @@ -107,7 +107,7 @@ describe("Subtitles", () => { Subtitles(mockMediaPlayer, autoStart, playbackElement, null, mockMediaSources, () => { expect(LegacySubtitles).not.toHaveBeenCalled() expect(IMSCSubtitles).not.toHaveBeenCalled() - expect(DashSubtitles).not.toHaveBeenCalled() + expect(DashSubtitles).toHaveBeenCalledTimes(1) done() }) }) From 3f656961c6979bdf7494f965d2ba65a8b38ddce8 Mon Sep 17 00:00:00 2001 From: Ryan McCartney Date: Thu, 10 Oct 2024 13:34:38 +0100 Subject: [PATCH 07/11] Rename DashSubs to EmbeddedSubs --- docs/tutorials/Subtitles.md | 6 ++++-- src/playbackstrategy/msestrategy.js | 4 ++-- .../{dashsubtitles.js => embeddedsubtitles.js} | 4 ++-- ...itles.test.js => embeddedsubtitles.test.js} | 12 ++++++------ src/subtitles/subtitles.js | 18 ++++++++++++------ src/subtitles/subtitles.test.js | 18 +++++++++--------- 6 files changed, 35 insertions(+), 27 deletions(-) rename src/subtitles/{dashsubtitles.js => embeddedsubtitles.js} (90%) rename src/subtitles/{dashsubtitles.test.js => embeddedsubtitles.test.js} (83%) diff --git a/docs/tutorials/Subtitles.md b/docs/tutorials/Subtitles.md index 50bdf181..79499413 100644 --- a/docs/tutorials/Subtitles.md +++ b/docs/tutorials/Subtitles.md @@ -51,10 +51,12 @@ const captions = [ { url: "https://some.cdn/subtitles/$segment$.m4s", segmentLength: 3.84, + cdn: "default", }, { url: "https://other.cdn/subtitles/$segment$.m4s", segmentLength: 3.84, + cdn: "default", }, /* ... */ ] @@ -83,7 +85,7 @@ player.init(document.querySelector("video"), playbackData /* other opts */) ### Low Latency Streams -When using Dash.js with a low-latency MPD segments are delivered using Chunked Transfer Encoding (CTE) - the current side chain doesn't allow for delivery in this case. +When using Dash.js with a low-latency MPD segments are delivered using Chunked Transfer Encoding (CTE) - the default side chain doesn't allow for delivery in this case. Whilst it is possible to collect chunks as they are delivered, wait until a full segment worth of subtitles have been delivered and pass these to the render function this breaks the low-latency workflow. @@ -91,7 +93,7 @@ An override has been added to allow subtitles to be rendered directly by Dash.js Subtitles can be enabled and disabled in the usual way using the `setSubtitlesEnabled()` function. However, they are signalled and delivered by the chosen MPD. -Using Dash.js subtitles can be enabled using `window.bigscreenPlayer.overrides.dashSubtitles = true`. +Using Dash.js subtitles can be enabled using `window.bigscreenPlayer.overrides.embeddedSubtitles = true`. ##  Design diff --git a/src/playbackstrategy/msestrategy.js b/src/playbackstrategy/msestrategy.js index 7f5601d8..d0633ba9 100644 --- a/src/playbackstrategy/msestrategy.js +++ b/src/playbackstrategy/msestrategy.js @@ -500,13 +500,13 @@ function MSEStrategy(mediaSources, windowType, mediaKind, playbackElement, isUHD function setUpMediaPlayer(playbackTime) { const dashSettings = getDashSettings(playerSettings) - const dashSubs = window.bigscreenPlayer?.overrides?.dashSubtitles ?? false + const embeddedSubs = window.bigscreenPlayer?.overrides?.embeddedSubtitles ?? false mediaPlayer = MediaPlayer().create() mediaPlayer.updateSettings(dashSettings) mediaPlayer.initialize(mediaElement, null, true) - if (dashSubs) { + if (embeddedSubs) { mediaPlayer.attachTTMLRenderingDiv(document.querySelector("#bsp_subtitles")) } diff --git a/src/subtitles/dashsubtitles.js b/src/subtitles/embeddedsubtitles.js similarity index 90% rename from src/subtitles/dashsubtitles.js rename to src/subtitles/embeddedsubtitles.js index a1fae6a2..7dd4772f 100644 --- a/src/subtitles/dashsubtitles.js +++ b/src/subtitles/embeddedsubtitles.js @@ -1,6 +1,6 @@ import DOMHelpers from "../domhelpers" -function DashSubtitles(mediaPlayer, autoStart, parentElement) { +function EmbeddedSubtitles(mediaPlayer, autoStart, parentElement) { let currentSubtitlesElement if (autoStart) { @@ -47,4 +47,4 @@ function DashSubtitles(mediaPlayer, autoStart, parentElement) { } } -export default DashSubtitles +export default EmbeddedSubtitles diff --git a/src/subtitles/dashsubtitles.test.js b/src/subtitles/embeddedsubtitles.test.js similarity index 83% rename from src/subtitles/dashsubtitles.test.js rename to src/subtitles/embeddedsubtitles.test.js index 0677711b..d5051c62 100644 --- a/src/subtitles/dashsubtitles.test.js +++ b/src/subtitles/embeddedsubtitles.test.js @@ -1,8 +1,8 @@ -import DashSubtitles from "./dashsubtitles" +import EmbeddedSubtitles from "./embeddedsubtitles" const UPDATE_INTERVAL = 750 -describe("Dash Subtitles", () => { +describe("Embedded Subtitles", () => { let subtitles let targetElement @@ -41,7 +41,7 @@ describe("Dash Subtitles", () => { it("returns the correct interface", () => { const autoStart = false - subtitles = DashSubtitles(mockMediaPlayer, autoStart, targetElement) + subtitles = EmbeddedSubtitles(mockMediaPlayer, autoStart, targetElement) expect(subtitles).toEqual( expect.objectContaining({ @@ -55,7 +55,7 @@ describe("Dash Subtitles", () => { it("Expect TTML rendering div to have been created", () => { const autoStart = false - subtitles = DashSubtitles(mockMediaPlayer, autoStart, targetElement) + subtitles = EmbeddedSubtitles(mockMediaPlayer, autoStart, targetElement) progressTime(1.5) expect(targetElement.querySelector("#bsp_subtitles")).toBeTruthy() @@ -66,7 +66,7 @@ describe("Dash Subtitles", () => { it("triggers the MSE player to enable subtitles immediately when set to autoplay", () => { const autoStart = true - subtitles = DashSubtitles(mockMediaPlayer, autoStart, targetElement) + subtitles = EmbeddedSubtitles(mockMediaPlayer, autoStart, targetElement) progressTime(1.5) expect(mockMediaPlayer.setSubtitles).toHaveBeenCalledTimes(1) @@ -75,7 +75,7 @@ describe("Dash Subtitles", () => { it("does not trigger the MSE player to enable subtitles immediately when set to autoplay", () => { const autoStart = false - subtitles = DashSubtitles(mockMediaPlayer, autoStart, targetElement) + subtitles = EmbeddedSubtitles(mockMediaPlayer, autoStart, targetElement) progressTime(1.5) expect(mockMediaPlayer.setSubtitles).toHaveBeenCalledTimes(0) diff --git a/src/subtitles/subtitles.js b/src/subtitles/subtitles.js index d22a4e28..b1a102f9 100644 --- a/src/subtitles/subtitles.js +++ b/src/subtitles/subtitles.js @@ -3,7 +3,7 @@ import findSegmentTemplate from "../utils/findtemplate" function Subtitles(mediaPlayer, autoStart, playbackElement, defaultStyleOpts, mediaSources, callback) { const useLegacySubs = window.bigscreenPlayer?.overrides?.legacySubtitles ?? false - const dashSubs = window.bigscreenPlayer?.overrides?.dashSubtitles ?? false + const embeddedSubs = window.bigscreenPlayer?.overrides?.embeddedSubtitles ?? false const isSeekableLiveSupport = window.bigscreenPlayer.liveSupport == null || window.bigscreenPlayer.liveSupport === "seekable" @@ -21,10 +21,16 @@ function Subtitles(mediaPlayer, autoStart, playbackElement, defaultStyleOpts, me .catch(() => { Plugins.interface.onSubtitlesDynamicLoadError() }) - } else if (dashSubs) { - import("./dashsubtitles.js") - .then(({ default: DashSubtitles }) => { - subtitlesContainer = DashSubtitles(mediaPlayer, autoStart, playbackElement, mediaSources, defaultStyleOpts) + } else if (embeddedSubs) { + import("./embeddedsubtitles.js") + .then(({ default: EmbeddedSubtitles }) => { + subtitlesContainer = EmbeddedSubtitles( + mediaPlayer, + autoStart, + playbackElement, + mediaSources, + defaultStyleOpts + ) callback(subtitlesEnabled) }) .catch(() => { @@ -78,7 +84,7 @@ function Subtitles(mediaPlayer, autoStart, playbackElement, defaultStyleOpts, me } function available() { - if (dashSubs) { + if (embeddedSubs) { return true } diff --git a/src/subtitles/subtitles.test.js b/src/subtitles/subtitles.test.js index 467e7dd1..39dac1b6 100644 --- a/src/subtitles/subtitles.test.js +++ b/src/subtitles/subtitles.test.js @@ -1,13 +1,13 @@ /* eslint-disable jest/no-done-callback */ import IMSCSubtitles from "./imscsubtitles" import LegacySubtitles from "./legacysubtitles" -import DashSubtitles from "./dashsubtitles" +import EmbeddedSubtitles from "./embeddedsubtitles" import Subtitles from "./subtitles" jest.mock("./imscsubtitles") jest.mock("./legacysubtitles") -jest.mock("./dashsubtitles") +jest.mock("./embeddedsubtitles") describe("Subtitles", () => { let isAvailable @@ -77,29 +77,29 @@ describe("Subtitles", () => { }) }) - describe("dash", () => { + describe("embedded", () => { beforeEach(() => { window.bigscreenPlayer = { overrides: { - dashSubtitles: true, + embeddedSubtitles: true, }, } - DashSubtitles.mockReset() + EmbeddedSubtitles.mockReset() }) - it("implementation is available when dash subtitles override is true", (done) => { + it("implementation is available when embedded subtitles override is true", (done) => { const mockMediaPlayer = {} const autoStart = true Subtitles(mockMediaPlayer, autoStart, playbackElement, null, mockMediaSources, (result) => { expect(result).toBe(true) - expect(DashSubtitles).toHaveBeenCalledTimes(1) + expect(EmbeddedSubtitles).toHaveBeenCalledTimes(1) done() }) }) - it("implementation is available when dash subtitles override is true, even if segmented URL is passed", (done) => { + it("implementation is available when embedded subtitles override is true, even if segmented URL is passed", (done) => { isSegmented = true const mockMediaPlayer = {} const autoStart = true @@ -107,7 +107,7 @@ describe("Subtitles", () => { Subtitles(mockMediaPlayer, autoStart, playbackElement, null, mockMediaSources, () => { expect(LegacySubtitles).not.toHaveBeenCalled() expect(IMSCSubtitles).not.toHaveBeenCalled() - expect(DashSubtitles).toHaveBeenCalledTimes(1) + expect(EmbeddedSubtitles).toHaveBeenCalledTimes(1) done() }) }) From 4c6edf7b71cd89fc43ca6e1642905c990ba3a398 Mon Sep 17 00:00:00 2001 From: Tom Coward <34864926+tom-coward@users.noreply.github.com> Date: Wed, 23 Oct 2024 14:10:27 +0100 Subject: [PATCH 08/11] Fix merge (missing }) --- src/playbackstrategy/msestrategy.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/playbackstrategy/msestrategy.js b/src/playbackstrategy/msestrategy.js index 687ae6cb..159e98e1 100644 --- a/src/playbackstrategy/msestrategy.js +++ b/src/playbackstrategy/msestrategy.js @@ -583,6 +583,7 @@ function MSEStrategy(mediaSources, windowType, mediaKind, playbackElement, isUHD const textTracks = mediaElement.textTracks for (let index = 0; index < textTracks.length; index++) { textTracks[index].mode = "showing" + } } function manifestLoadingFinished(event) { From 485ca684a149d6ad0b5f28c47228acedd7db632d Mon Sep 17 00:00:00 2001 From: Tom Coward <34864926+tom-coward@users.noreply.github.com> Date: Tue, 29 Oct 2024 11:47:47 +0000 Subject: [PATCH 09/11] Wait for media player to be ready before enabling embedded subtitles --- src/subtitles/embeddedsubtitles.js | 5 +++++ src/subtitles/embeddedsubtitles.test.js | 3 ++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/subtitles/embeddedsubtitles.js b/src/subtitles/embeddedsubtitles.js index 7dd4772f..bc0238be 100644 --- a/src/subtitles/embeddedsubtitles.js +++ b/src/subtitles/embeddedsubtitles.js @@ -4,7 +4,12 @@ function EmbeddedSubtitles(mediaPlayer, autoStart, parentElement) { let currentSubtitlesElement if (autoStart) { + mediaPlayer.addEventCallback(this, onMediaPlayerReady) + } + + function onMediaPlayerReady() { start() + mediaPlayer.removeEventCallback(this, onMediaPlayerReady) } function removeCurrentSubtitlesElement() { diff --git a/src/subtitles/embeddedsubtitles.test.js b/src/subtitles/embeddedsubtitles.test.js index d5051c62..a66910f3 100644 --- a/src/subtitles/embeddedsubtitles.test.js +++ b/src/subtitles/embeddedsubtitles.test.js @@ -9,6 +9,7 @@ describe("Embedded Subtitles", () => { const mockMediaPlayer = { getCurrentTime: jest.fn(), setSubtitles: jest.fn(), + addEventCallback: jest.fn(), } beforeEach(() => { @@ -63,7 +64,7 @@ describe("Embedded Subtitles", () => { }) describe("autoplay", () => { - it("triggers the MSE player to enable subtitles immediately when set to autoplay", () => { + it.skip("triggers the MSE player to enable subtitles immediately when set to autoplay", () => { const autoStart = true subtitles = EmbeddedSubtitles(mockMediaPlayer, autoStart, targetElement) From 591a5537b3c8749b21dce0572e5091cccb2303db Mon Sep 17 00:00:00 2001 From: Ryan McCartney Date: Thu, 6 Feb 2025 18:04:46 +0000 Subject: [PATCH 10/11] Pass IMSCjs style options through BSP to embeddedsubtitles --- src/playbackstrategy/msestrategy.js | 5 +++++ src/playercomponent.js | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/src/playbackstrategy/msestrategy.js b/src/playbackstrategy/msestrategy.js index 159e98e1..30309642 100644 --- a/src/playbackstrategy/msestrategy.js +++ b/src/playbackstrategy/msestrategy.js @@ -608,6 +608,10 @@ function MSEStrategy(mediaSources, windowType, mediaKind, playbackElement, isUHD } } + function customiseSubtitles(options) { + return mediaPlayer && mediaPlayer.updateSettings({ streaming: { text: { imsc: { options } } } }) + } + function getDuration() { return mediaPlayer && mediaPlayer.isReady() ? mediaPlayer.duration() : 0 } @@ -777,6 +781,7 @@ function MSEStrategy(mediaSources, windowType, mediaKind, playbackElement, isUHD startAutoResumeTimeout() } }, + customiseSubtitles, play: () => mediaPlayer.play(), setCurrentTime: (time) => { publishedSeekEvent = false diff --git a/src/playercomponent.js b/src/playercomponent.js index b7657c9b..a32c8854 100644 --- a/src/playercomponent.js +++ b/src/playercomponent.js @@ -75,6 +75,10 @@ function PlayerComponent( return playbackStrategy && playbackStrategy.setSubtitles(state) } + function customiseSubtitles(styleOpts) { + return playbackStrategy && playbackStrategy.customiseSubtitles(styleOpts) + } + function getDuration() { return playbackStrategy && playbackStrategy.getDuration() } @@ -399,6 +403,7 @@ function PlayerComponent( play, pause, setSubtitles, + customiseSubtitles, transitions, isEnded, setPlaybackRate, From 56c838689f80dd8c7354f545b38ecc21a0f1b897 Mon Sep 17 00:00:00 2001 From: Ryan McCartney Date: Sun, 9 Feb 2025 16:52:09 +0000 Subject: [PATCH 11/11] Add customisation to embedded subtitles --- src/subtitles/embeddedsubtitles.js | 41 ++++++++++++++-- src/subtitles/embeddedsubtitles.test.js | 62 +++++++++++++++++++++++-- 2 files changed, 95 insertions(+), 8 deletions(-) diff --git a/src/subtitles/embeddedsubtitles.js b/src/subtitles/embeddedsubtitles.js index bc0238be..2171f8bd 100644 --- a/src/subtitles/embeddedsubtitles.js +++ b/src/subtitles/embeddedsubtitles.js @@ -1,14 +1,17 @@ import DOMHelpers from "../domhelpers" +import Utils from "../utils/playbackutils" -function EmbeddedSubtitles(mediaPlayer, autoStart, parentElement) { +function EmbeddedSubtitles(mediaPlayer, autoStart, parentElement, mediaSources, defaultStyleOpts) { let currentSubtitlesElement + let imscRenderOpts = transformStyleOptions(defaultStyleOpts) + if (autoStart) { + start() mediaPlayer.addEventCallback(this, onMediaPlayerReady) } function onMediaPlayerReady() { - start() mediaPlayer.removeEventCallback(this, onMediaPlayerReady) } @@ -29,6 +32,7 @@ function EmbeddedSubtitles(mediaPlayer, autoStart, parentElement) { function start() { mediaPlayer.setSubtitles(true) + customise(imscRenderOpts) if (!currentSubtitlesElement) { addCurrentSubtitlesElement() } @@ -42,12 +46,43 @@ function EmbeddedSubtitles(mediaPlayer, autoStart, parentElement) { stop() } + function customise(styleOpts) { + const customStyleOptions = transformStyleOptions(styleOpts) + imscRenderOpts = Utils.merge(imscRenderOpts, customStyleOptions) + mediaPlayer.customiseSubtitles(imscRenderOpts) + } + + // Opts: { backgroundColour: string (css colour, hex), fontFamily: string , size: number, lineHeight: number } + function transformStyleOptions(opts) { + if (opts === undefined) return + + const customStyles = {} + + if (opts.backgroundColour) { + customStyles.spanBackgroundColorAdjust = { transparent: opts.backgroundColour } + } + + if (opts.fontFamily) { + customStyles.fontFamily = opts.fontFamily + } + + if (opts.size > 0) { + customStyles.sizeAdjust = opts.size + } + + if (opts.lineHeight) { + customStyles.lineHeightAdjust = opts.lineHeight + } + + return customStyles + } + addCurrentSubtitlesElement() return { start, stop, - customise: () => {}, + customise, tearDown, } } diff --git a/src/subtitles/embeddedsubtitles.test.js b/src/subtitles/embeddedsubtitles.test.js index a66910f3..e7d315bc 100644 --- a/src/subtitles/embeddedsubtitles.test.js +++ b/src/subtitles/embeddedsubtitles.test.js @@ -10,6 +10,7 @@ describe("Embedded Subtitles", () => { getCurrentTime: jest.fn(), setSubtitles: jest.fn(), addEventCallback: jest.fn(), + customiseSubtitles: jest.fn(), } beforeEach(() => { @@ -42,7 +43,7 @@ describe("Embedded Subtitles", () => { it("returns the correct interface", () => { const autoStart = false - subtitles = EmbeddedSubtitles(mockMediaPlayer, autoStart, targetElement) + subtitles = EmbeddedSubtitles(mockMediaPlayer, autoStart, targetElement, null, {}) expect(subtitles).toEqual( expect.objectContaining({ @@ -56,7 +57,7 @@ describe("Embedded Subtitles", () => { it("Expect TTML rendering div to have been created", () => { const autoStart = false - subtitles = EmbeddedSubtitles(mockMediaPlayer, autoStart, targetElement) + subtitles = EmbeddedSubtitles(mockMediaPlayer, autoStart, targetElement, null, {}) progressTime(1.5) expect(targetElement.querySelector("#bsp_subtitles")).toBeTruthy() @@ -64,10 +65,10 @@ describe("Embedded Subtitles", () => { }) describe("autoplay", () => { - it.skip("triggers the MSE player to enable subtitles immediately when set to autoplay", () => { + it("triggers the MSE player to enable subtitles immediately when set to autoplay", () => { const autoStart = true - subtitles = EmbeddedSubtitles(mockMediaPlayer, autoStart, targetElement) + subtitles = EmbeddedSubtitles(mockMediaPlayer, autoStart, targetElement, null, {}) progressTime(1.5) expect(mockMediaPlayer.setSubtitles).toHaveBeenCalledTimes(1) @@ -76,10 +77,61 @@ describe("Embedded Subtitles", () => { it("does not trigger the MSE player to enable subtitles immediately when set to autoplay", () => { const autoStart = false - subtitles = EmbeddedSubtitles(mockMediaPlayer, autoStart, targetElement) + subtitles = EmbeddedSubtitles(mockMediaPlayer, autoStart, targetElement, null, {}) progressTime(1.5) expect(mockMediaPlayer.setSubtitles).toHaveBeenCalledTimes(0) }) }) + + describe("customisation", () => { + it("overrides the subtitles styling metadata with supplied defaults when rendering", () => { + const expectedStyles = { spanBackgroundColorAdjust: { transparent: "black" }, fontFamily: "Arial" } + + subtitles = EmbeddedSubtitles(mockMediaPlayer, false, targetElement, null, { + backgroundColour: "black", + fontFamily: "Arial", + }) + + subtitles.start() + + progressTime(1) + + expect(mockMediaPlayer.customiseSubtitles).toHaveBeenCalledWith(expectedStyles) + }) + + it("overrides the subtitles styling metadata with supplied custom styles when rendering", () => { + subtitles = EmbeddedSubtitles(mockMediaPlayer, false, targetElement, null, {}) + + const styleOpts = { size: 0.7, lineHeight: 0.9 } + const expectedOpts = { sizeAdjust: 0.7, lineHeightAdjust: 0.9 } + + mockMediaPlayer.getCurrentTime.mockReturnValueOnce(1) + + subtitles.start() + subtitles.customise(styleOpts) + + expect(mockMediaPlayer.customiseSubtitles).toHaveBeenCalledWith(expectedOpts) + }) + + it("merges the current subtitles styling metadata with new supplied custom styles when rendering", () => { + const defaultStyleOpts = { backgroundColour: "black", fontFamily: "Arial" } + const customStyleOpts = { size: 0.7, lineHeight: 0.9 } + const expectedOpts = { + spanBackgroundColorAdjust: { transparent: "black" }, + fontFamily: "Arial", + sizeAdjust: 0.7, + lineHeightAdjust: 0.9, + } + + subtitles = EmbeddedSubtitles(mockMediaPlayer, false, targetElement, null, defaultStyleOpts) + + mockMediaPlayer.getCurrentTime.mockReturnValueOnce(1) + + subtitles.start() + subtitles.customise(customStyleOpts) + + expect(mockMediaPlayer.customiseSubtitles).toHaveBeenCalledWith(expectedOpts) + }) + }) })