Skip to content

Commit

Permalink
Add SMPTE-TT Subtitle Images (#1859)
Browse files Browse the repository at this point in the history 
This change adds support for images via SMPTE-TT by adding background image support to the TTML text parser. This required changing region parsing to respect global extends.

Issue #840
  • Loading branch information
@vaage
Álvaro Velad Galván authored and vaage committed Apr 4, 2019
1 parent 7ea43bc commit 6f3241e
Show file tree
Hide file tree
Showing 7 changed files with 123 additions and 19 deletions.
16 changes: 16 additions & 0 deletions demo/common/assets.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1109,6 +1109,22 @@ shakaAssets.testAssets = [
shakaAssets.Feature.SEGMENT_TEMPLATE_DURATION,
],
},
{
name: 'Live sim TTML Image Subtitles embedded (VoD)',
manifestUri: 'https://livesim.dashif.org/dash/vod/testpic_2s/img_subs.mpd',

iconUri: 'https://storage.googleapis.com/shaka-asset-icons/dash_if_test_pattern.png',

encoder: shakaAssets.Encoder.UNKNOWN,
source: shakaAssets.Source.DASH_IF,
drm: [],
features: [
shakaAssets.Feature.DASH,
shakaAssets.Feature.MP4,
shakaAssets.Feature.SUBTITLES,
shakaAssets.Feature.TTML,
],
},
// }}}

// Wowza assets {{{
Expand Down
10 changes: 10 additions & 0 deletions externs/shaka/text.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -283,6 +283,16 @@ shaka.extern.Cue.prototype.color;
shaka.extern.Cue.prototype.backgroundColor;


/**
* Image background represented by any string that would be
* accepted in image HTML element.
* E. g. 'data:[mime type];base64,[data]'.
* @type {!string}
* @exportDoc
*/
shaka.extern.Cue.prototype.backgroundImage;


/**
* Text font size in px or em (e.g. '100px'/'100em').
* @type {string}
Expand Down
6 changes: 6 additions & 0 deletions lib/text/cue.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,12 @@ shaka.text.Cue = function(startTime, endTime, payload) {
*/
this.backgroundColor = '';

/**
* @override
* @exportInterface
*/
this.backgroundImage = null;

/**
* @override
* @exportInterface
Expand Down
92 changes: 73 additions & 19 deletions lib/text/ttml_text_parser.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,7 @@ shaka.text.TtmlTextParser.prototype.parseMedia = function(data, time) {
let frameRateMultiplier = null;
let tickRate = null;
let spaceStyle = null;
let extent = null;
let tts = xml.getElementsByTagName('tt');
let tt = tts[0];
// TTML should always have tt element.
Expand All @@ -97,6 +98,7 @@ shaka.text.TtmlTextParser.prototype.parseMedia = function(data, time) {
XmlUtils.getAttributeNS(tt, ttpNs, 'frameRateMultiplier');
tickRate = XmlUtils.getAttributeNS(tt, ttpNs, 'tickRate');
spaceStyle = tt.getAttribute('xml:space') || 'default';
extent = tt.getAttribute('tts:extent');
}

if (spaceStyle != 'default' && spaceStyle != 'preserve') {
Expand All @@ -110,14 +112,16 @@ shaka.text.TtmlTextParser.prototype.parseMedia = function(data, time) {
let rateInfo = new TtmlTextParser.RateInfo_(
frameRate, subFrameRate, frameRateMultiplier, tickRate);

const metadataElements = TtmlTextParser.getLeafNodes_(
tt.getElementsByTagName('metadata')[0]);
let styles = TtmlTextParser.getLeafNodes_(
tt.getElementsByTagName('styling')[0]);
let regionElements = TtmlTextParser.getLeafNodes_(
tt.getElementsByTagName('layout')[0]);
let cueRegions = [];
for (let i = 0; i < regionElements.length; i++) {
let cueRegion = TtmlTextParser.parseCueRegion_(
regionElements[i], styles);
regionElements[i], styles, extent);
if (cueRegion) {
cueRegions.push(cueRegion);
}
Expand All @@ -130,6 +134,7 @@ shaka.text.TtmlTextParser.prototype.parseMedia = function(data, time) {
let cue = TtmlTextParser.parseCue_(textNodes[i],
time.periodStart,
rateInfo,
metadataElements,
styles,
regionElements,
cueRegions,
Expand Down Expand Up @@ -319,6 +324,7 @@ shaka.text.TtmlTextParser.addNewLines_ = function(element, whitespaceTrim) {
* @param {!Element} cueElement
* @param {number} offset
* @param {!shaka.text.TtmlTextParser.RateInfo_} rateInfo
* @param {!Array.<!Element>} metadataElements
* @param {!Array.<!Element>} styles
* @param {!Array.<!Element>} regionElements
* @param {!Array.<!shaka.text.CueRegion>} cueRegions
Expand All @@ -327,8 +333,8 @@ shaka.text.TtmlTextParser.addNewLines_ = function(element, whitespaceTrim) {
* @private
*/
shaka.text.TtmlTextParser.parseCue_ = function(
cueElement, offset, rateInfo, styles, regionElements,
cueRegions, whitespaceTrim) {
cueElement, offset, rateInfo, metadataElements, styles,
regionElements, cueRegions, whitespaceTrim) {
// Disregard empty elements:
// TTML allows for empty elements like <div></div>.
// If cueElement has neither time attributes, nor
Expand Down Expand Up @@ -368,15 +374,22 @@ shaka.text.TtmlTextParser.parseCue_ = function(

// Get other properties if available.
let regionElement = shaka.text.TtmlTextParser.getElementFromCollection_(
cueElement, 'region', regionElements);
cueElement, 'region', regionElements, /* prefix= */ '');
if (regionElement && regionElement.getAttribute('xml:id')) {
let regionId = regionElement.getAttribute('xml:id');
let regionsWithId = cueRegions.filter(function(region) {
return region.id == regionId;
});
cue.region = regionsWithId[0];
}
shaka.text.TtmlTextParser.addStyle_(cue, cueElement, regionElement, styles);
const imageElement = shaka.text.TtmlTextParser.getElementFromCollection_(
cueElement, 'smpte:backgroundImage', metadataElements, '#');
shaka.text.TtmlTextParser.addStyle_(
cue,
cueElement,
regionElement,
imageElement,
styles);

return cue;
};
Expand All @@ -386,11 +399,14 @@ shaka.text.TtmlTextParser.parseCue_ = function(
* Parses an Element into a TextTrackCue or VTTCue.
*
* @param {!Element} regionElement
* @param {!Array.<!Element>} styles
* @param {!Array.<!Element>} styles Defined in the top of tt element and
* used principally for images.
* @param {string} globalExtent
* @return {shaka.text.CueRegion}
* @private
*/
shaka.text.TtmlTextParser.parseCueRegion_ = function(regionElement, styles) {
shaka.text.TtmlTextParser.parseCueRegion_ = function(regionElement, styles,
globalExtent) {
const TtmlTextParser = shaka.text.TtmlTextParser;
let region = new shaka.text.CueRegion();
let id = regionElement.getAttribute('xml:id');
Expand All @@ -401,6 +417,14 @@ shaka.text.TtmlTextParser.parseCueRegion_ = function(regionElement, styles) {
}
region.id = id;

let globalResults = null;
if (globalExtent) {
globalResults = TtmlTextParser.percentValues_.exec(globalExtent) ||
TtmlTextParser.pixelValues_.exec(globalExtent);
}
const globalWidth = globalResults ? Number(globalResults[1]) : null;
const globalHeight = globalResults ? Number(globalResults[2]) : null;

let results = null;
let percentage = null;
let extent = TtmlTextParser.getStyleAttributeFromRegion_(
Expand All @@ -409,13 +433,21 @@ shaka.text.TtmlTextParser.parseCueRegion_ = function(regionElement, styles) {
percentage = TtmlTextParser.percentValues_.exec(extent);
results = percentage || TtmlTextParser.pixelValues_.exec(extent);
if (results != null) {
region.width = Number(results[1]);
region.height = Number(results[2]);
region.widthUnits = percentage ?
if (globalWidth != null) {
region.width = Number(results[1] * 100 / globalWidth);
} else {
region.width = Number(results[1]);
}
if (globalHeight != null) {
region.height = Number(results[2] * 100 / globalHeight);
} else {
region.height = Number(results[2]);
}
region.widthUnits = percentage || globalWidth != null ?
shaka.text.CueRegion.units.PERCENTAGE :
shaka.text.CueRegion.units.PX;

region.heightUnits = percentage ?
region.heightUnits = percentage || globalHeight != null ?
shaka.text.CueRegion.units.PERCENTAGE :
shaka.text.CueRegion.units.PX;
}
Expand All @@ -427,9 +459,17 @@ shaka.text.TtmlTextParser.parseCueRegion_ = function(regionElement, styles) {
percentage = TtmlTextParser.percentValues_.exec(origin);
results = percentage || TtmlTextParser.pixelValues_.exec(origin);
if (results != null) {
region.viewportAnchorX = Number(results[1]);
region.viewportAnchorY = Number(results[2]);
region.viewportAnchorUnits = percentage ?
if (globalHeight != null) {
region.viewportAnchorX = Number(results[1] * 100 / globalHeight);
} else {
region.viewportAnchorX = Number(results[1]);
}
if (globalWidth != null) {
region.viewportAnchorY = Number(results[2] * 100 / globalWidth);
} else {
region.viewportAnchorY = Number(results[2]);
}
region.viewportAnchorUnits = percentage || globalWidth != null ?
shaka.text.CueRegion.units.PERCENTAGE :
shaka.text.CueRegion.units.PX;
}
Expand All @@ -444,11 +484,12 @@ shaka.text.TtmlTextParser.parseCueRegion_ = function(regionElement, styles) {
* @param {!shaka.text.Cue} cue
* @param {!Element} cueElement
* @param {Element} region
* @param {Element} imageElement
* @param {!Array.<!Element>} styles
* @private
*/
shaka.text.TtmlTextParser.addStyle_ = function(
cue, cueElement, region, styles) {
cue, cueElement, region, imageElement, styles) {
const TtmlTextParser = shaka.text.TtmlTextParser;
const Cue = shaka.text.Cue;

Expand Down Expand Up @@ -550,6 +591,17 @@ shaka.text.TtmlTextParser.addStyle_ = function(
cue.fontStyle = Cue.fontStyle[fontStyle.toUpperCase()];
}

if (imageElement) {
const backgroundImageType = imageElement.getAttribute('imagetype');
const backgroundImageEncoding = imageElement.getAttribute('encoding');
const backgroundImageData = imageElement.textContent;
if (backgroundImageType == 'PNG' &&
backgroundImageEncoding == 'Base64' &&
backgroundImageData) {
cue.backgroundImage = 'data:image/png;base64,' + backgroundImageData;
}
}

// Text decoration is an array of values which can come both from the
// element's style or be inherited from elements' parent nodes. All of those
// values should be applied as long as they don't contradict each other. If
Expand Down Expand Up @@ -669,7 +721,7 @@ shaka.text.TtmlTextParser.getStyleAttributeFromRegion_ = function(
}

let style = shaka.text.TtmlTextParser.getElementFromCollection_(
region, 'style', styles);
region, 'style', styles, /* prefix= */ '');
if (style) {
return XmlUtils.getAttributeNS(style, ttsNs, attribute);
}
Expand All @@ -694,7 +746,8 @@ shaka.text.TtmlTextParser.getStyleAttributeFromElement_ = function(

let getElementFromCollection_ =
shaka.text.TtmlTextParser.getElementFromCollection_;
let style = getElementFromCollection_(cueElement, 'style', styles);
let style = getElementFromCollection_(
cueElement, 'style', styles, /* prefix= */ '');
if (style) {
return XmlUtils.getAttributeNS(style, ttsNs, attribute);
}
Expand All @@ -709,11 +762,12 @@ shaka.text.TtmlTextParser.getStyleAttributeFromElement_ = function(
* @param {Element} element
* @param {string} attributeName
* @param {!Array.<Element>} collection
* @param {string} prefixName
* @return {Element}
* @private
*/
shaka.text.TtmlTextParser.getElementFromCollection_ = function(
element, attributeName, collection) {
element, attributeName, collection, prefixName) {
if (!element || collection.length < 1) {
return null;
}
Expand All @@ -722,7 +776,7 @@ shaka.text.TtmlTextParser.getElementFromCollection_ = function(
element, attributeName);
if (itemName) {
for (let i = 0; i < collection.length; i++) {
if (collection[i].getAttribute('xml:id') == itemName) {
if ((prefixName + collection[i].getAttribute('xml:id')) == itemName) {
item = collection[i];
break;
}
Expand Down
3 changes: 3 additions & 0 deletions ui/less/buttons.less
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,9 @@
/* A background color behind the play arrow. */
background-color: rgba(255, 255, 255, .9);

/* Overlay the captions */
z-index: 1;

/* Actual icon images for the two states this could be in.
* These will be inlined as data URIs when compiled, and so do not need to be
* deployed separately from the compiled CSS.
Expand Down
1 change: 1 addition & 0 deletions ui/less/containers.less
View file
Original file line number Diff line number Diff line change
Expand Up @@ -192,6 +192,7 @@

.shaka-text-container {
bottom: 12%;
top: 0%;
width: 100%;
min-width: 48px;

Expand Down
14 changes: 14 additions & 0 deletions ui/text_displayer.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -204,6 +204,20 @@ shaka.ui.TextDisplayer = class {
captionsStyle.color = cue.color;
captionsStyle.direction = cue.direction;

if (cue.backgroundImage) {
captionsStyle.backgroundImage = 'url(\'' + cue.backgroundImage + '\')';
captionsStyle.backgroundRepeat = 'no-repeat';
captionsStyle.backgroundSize = 'contain';
captionsStyle.backgroundPosition = 'center';
}
if (cue.backgroundImage && cue.region) {
const percentageUnit = shaka.text.CueRegion.units.PERCENTAGE;
const heightUnit = cue.region.heightUnits == percentageUnit ? '%' : 'px';
const widthUnit = cue.region.widthUnits == percentageUnit ? '%' : 'px';
captionsStyle.height = cue.region.height + heightUnit;
captionsStyle.width = cue.region.width + widthUnit;
}

// The displayAlign attribute specifys the vertical alignment of the
// captions inside the text container. Before means at the top of the
// text container, and after means at the bottom.
Expand Down

0 comments on commit 6f3241e

Please sign in to comment.