Skip to content

Commit

Permalink
[android] Improve multi-encrypted-video demo (#1837)
Browse files Browse the repository at this point in the history 
In Cobalt:
* Removed tunnel mode check on audio decoder in Cobalt.
The check prevents tunnel mode from being enabled via mime attribute,
which is required to enable tunnel mode on the primary video of the
demo, as mentioned below. The previous implementation checks whether the
audio decoder supports tunnel mode before enabling it. However, none of
them does, and the audio will be decoded to PCM before sending to the
AudioTrack, so whether the audio decoder supports tunnel mode is
irrelevant.

In the demo:
1. Now the demo tries to play the primary video using tunnel mode, by
passing "tunnelmode=true" as an extra mime attribute when supported.
2. Added more streams with different codecs and resolutions, and allow
specifying them at runtime.
3. Try to create com.widevine.alpha when creation of
com.youtube.widevine.l3 fails. This allows the demo to run on browsers
without Widevine L3 support, including Chrome.
4. Download media data at startup to reduce CPU usage durnig playback.

b/175883701

Change-Id: Id1e6c3f19a509f20b8f6413840b5ad223e5437ee
  • Loading branch information
@xiaomings
xiaomings authored Oct 25, 2023
1 parent 751da5b commit 6b6abea
Show file tree
Hide file tree
Showing 8 changed files with 233 additions and 138 deletions.
30 changes: 16 additions & 14 deletions cobalt/demos/content/multi-encrypted-video/multi-encrypted-video.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
margin: 0;
}

#player-layer {
#primary-player-layer {
width: 100%;
height: 100%;
}
Expand All @@ -33,13 +33,14 @@
height: 100%;
}

#ui-layer {
#secondary-player-layer {
position: absolute;
top: 15%;
height: 85%;
top: 60%;
height: 40%;
width: 100%;
background-color: rgba(33, 33, 33, .75);
padding: 24px;
display: flex;
justify-content: center;
}

.item {
Expand All @@ -48,22 +49,23 @@
display: inline-block;
margin: 24px;
vertical-align: middle;
background-color: rgba(33, 33, 33, .75);
}
</style>
</head>
<body>
<div id="player-layer">
<video class="primary" id="primary-video" muted="1" autoplay="1"></video>
<div id="primary-player-layer">
<video id="primary-video" muted="1" autoplay="1"></video>
</div>
<div id="ui-layer">
<div class="item" style="background-color: #D44">
<video class="secondary" id="secondary-video-1" muted="1" autoplay="1"></video>
<div id="secondary-player-layer">
<div class="item">
<video id="secondary-video-1" muted="1" autoplay="1"></video>
</div>
<div class="item" style="background-color: #4D4">
<video class="secondary" id="secondary-video-2" muted="1" autoplay="1"></video>
<div class="item">
<video id="secondary-video-2" muted="1" autoplay="1"></video>
</div>
<div class="item" style="background-color: #44D">
<video class="secondary" id="secondary-video-3" muted="1" autoplay="1"></video>
<div class="item">
<video id="secondary-video-3" muted="1" autoplay="1"></video>
</div>
</div>
<script src="multi-encrypted-video.js"></script>
Expand Down
274 changes: 206 additions & 68 deletions cobalt/demos/content/multi-encrypted-video/multi-encrypted-video.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,58 @@
// See the License for the specific language governing permissions and
// limitations under the License.


// Dictionary mapping string descriptions to media file descriptions in the
// form of [contentType, url, maxVideoCapabilities (for videos only), licenseUrl]
const MEDIA_FILES = {
'av1_720p_60fps_drm': {
contentType: 'video/mp4; codecs="av01.0.05M.08"',
url: 'https://storage.googleapis.com/ytlr-cert.appspot.com/test-materials/media/av1-senc/sdr_720p60.mp4',
maxVideoCapabilities: 'width=1280; height=720',
licenseUrl: 'https://dash-mse-test.appspot.com/api/drm/widevine?drm_system=widevine&source=YOUTUBE&ip=0.0.0.0&ipbits=0&expire=19000000000&key=ik0&sparams=ip,ipbits,expire,drm_system,source,video_id&video_id=6508f99557a8385f&signature=5153900DAC410803EC269D252DAAA82BA6D8B825.495E631E406584A8EFCB4E9C9F3D45F6488B94E4',
},
// 40 MB
'h264_720p_24fps_drm': {
contentType: 'video/mp4; codecs="avc1.640028"',
url: 'http://yt-dash-mse-test.commondatastorage.googleapis.com/media/oops_cenc-20121114-145-no-clear-start.mp4',
maxVideoCapabilities: 'width=1280; height=720',
licenseUrl: 'https://dash-mse-test.appspot.com/api/drm/widevine?drm_system=widevine&source=YOUTUBE&ip=0.0.0.0&ipbits=0&expire=19000000000&key=test_key1&sparams=ip,ipbits,expire,drm_system,source,video_id&video_id=03681262dc412c06&signature=9C4BE99E6F517B51FED1F0B3B31966D3C5DAB9D6.6A1F30BB35F3A39A4CA814B731450D4CBD198FFD',
},
// 38 MB
'h264_720p_60fps_drm': {
contentType: 'video/mp4; codecs="avc1.640028"',
url: 'https://storage.googleapis.com/ytlr-cert.appspot.com/test-materials/media/drml3NoHdcp_h264_720p_60fps_cenc.mp4',
maxVideoCapabilities: 'width=1280; height=720',
licenseUrl: 'https://dash-mse-test.appspot.com/api/drm/widevine?drm_system=widevine&source=YOUTUBE&ip=0.0.0.0&ipbits=0&expire=19000000000&key=test_key1&sparams=ip,ipbits,expire,drm_system,source,video_id&video_id=03681262dc412c06&signature=9C4BE99E6F517B51FED1F0B3B31966D3C5DAB9D6.6A1F30BB35F3A39A4CA814B731450D4CBD198FFD',
},
// 32 MB
'vp9_720p_60fps_drm': {
contentType: 'video/webm; codecs="vp9"',
url: 'https://storage.googleapis.com/ytlr-cert.appspot.com/test-materials/media/drml3NoHdcp_vp9_720p_60fps_enc.webm',
maxVideoCapabilities: 'width=1280; height=720',
licenseUrl: 'https://dash-mse-test.appspot.com/api/drm/widevine?drm_system=widevine&source=YOUTUBE&ip=0.0.0.0&ipbits=0&expire=19000000000&key=test_key1&sparams=ip,ipbits,expire,drm_system,source,video_id&video_id=f320151fa3f061b2&signature=81E7B33929F9F35922F7D2E96A5E7AC36F3218B2.673F553EE51A48438AE5E707AEC87A071B4FEF65'
},
// 1 MB
// Mono won't work with tunnel mode on many devices.
'aac_mono_drm': {
contentType: 'audio/mp4; codecs="mp4a.40.2"',
url: 'http://yt-dash-mse-test.commondatastorage.googleapis.com/media/oops_cenc-20121114-148.mp4',
licenseUrl: 'https://dash-mse-test.appspot.com/api/drm/widevine?drm_system=widevine&source=YOUTUBE&ip=0.0.0.0&ipbits=0&expire=19000000000&key=test_key1&sparams=ip,ipbits,expire,drm_system,source,video_id&video_id=03681262dc412c06&signature=9C4BE99E6F517B51FED1F0B3B31966D3C5DAB9D6.6A1F30BB35F3A39A4CA814B731450D4CBD198FFD',
},
// 2.8 MB
'aac_clear': {
contentType: 'audio/mp4; codecs="mp4a.40.2"',
url: 'http://yt-dash-mse-test.commondatastorage.googleapis.com/media/car-20120827-8c.mp4',
},
// 1.7 MB
'opus_clear': {
contentType: 'audio/webm; codecs="opus"',
url: 'https://storage.googleapis.com/ytlr-cert.appspot.com/test-materials/media/car_opus_med.webm',
},
};

mediaCache = {}

function fetchArrayBuffer(method, url, body, callback) {
var xhr = new XMLHttpRequest();
xhr.responseType = 'arraybuffer';
Expand All @@ -22,6 +74,16 @@ function fetchArrayBuffer(method, url, body, callback) {
xhr.send(body);
}

async function fetchMediaData(mediaFileId) {
if (mediaFileId in mediaCache) {
return mediaCache[mediaFileId];
}

const response = await fetch(MEDIA_FILES[mediaFileId].url);
mediaCache[mediaFileId] = await response.arrayBuffer();
return mediaCache[mediaFileId];
}

function extractLicense(licenseArrayBuffer) {
var licenseArray = new Uint8Array(licenseArrayBuffer);
var licenseStartIndex = licenseArray.length - 2;
Expand All @@ -37,80 +99,156 @@ function extractLicense(licenseArrayBuffer) {
return licenseArray.subarray(licenseStartIndex);
}

var videoContentType = 'video/mp4; codecs="avc1.640028"';
var audioContentType = 'audio/mp4; codecs="mp4a.40.2"';

function play(videoElementId, keySystem) {
navigator.requestMediaKeySystemAccess(keySystem, [{
'initDataTypes': ['cenc'],
'videoCapabilities': [{'contentType': videoContentType}],
'audioCapabilities': [{'contentType': audioContentType}]
}]).then(function(mediaKeySystemAccess) {
return mediaKeySystemAccess.createMediaKeys();
}).then(function(mediaKeys) {
var videoElement = document.getElementById(videoElementId);

if (videoElementId != 'primary-video') {
videoElement.setMaxVideoCapabilities('width=1280; height=720');
async function createMediaKeySystem(isPrimaryVideo, audioContentType, videoContentType) {
const keySystems = isPrimaryVideo ? ['com.widevine.alpha'] : ['com.youtube.widevine.l3', 'com.widevine.alpha'];
for (keySystem of keySystems) {
try {
mediaKeySystemAccess = await navigator.requestMediaKeySystemAccess(keySystem, [{
'initDataTypes': ['cenc', 'webm'],
'audioCapabilities': [{'contentType': audioContentType}],
'videoCapabilities': [{'contentType': videoContentType}]}]);
return mediaKeySystemAccess.createMediaKeys();
} catch {
console.log('create keySystem ' + keySystem + ' failed.')
continue;
}
}
}

if (mediaKeys.getMetrics) {
console.log('Found getMetrics(), calling it ...');
try {
mediaKeys.getMetrics();
console.log('Calling getMetrics() succeeded.');
} catch(e) {
console.log('Calling getMetrics() failed.');
}
}
function createTunnelModeContentType(videoContentType, tunnelModeAttributeValue) {
return videoContentType + '; tunnelmode=' + tunnelModeAttributeValue;
}

videoElement.setMediaKeys(mediaKeys);

mediaKeySession = mediaKeys.createSession();
mediaKeySession.addEventListener('message', function(messageEvent) {
var licenseServerUrl = 'https://dash-mse-test.appspot.com/api/drm/widevine?drm_system=widevine&source=YOUTUBE&ip=0.0.0.0&ipbits=0&expire=19000000000&key=test_key1&sparams=ip,ipbits,expire,drm_system,source,video_id&video_id=03681262dc412c06&signature=9C4BE99E6F517B51FED1F0B3B31966D3C5DAB9D6.6A1F30BB35F3A39A4CA814B731450D4CBD198FFD';
fetchArrayBuffer('POST', licenseServerUrl, messageEvent.message,
function(licenseArrayBuffer) {
mediaKeySession.update(extractLicense(licenseArrayBuffer));
});
});

videoElement.addEventListener('encrypted', function(encryptedEvent) {
mediaKeySession.generateRequest(
encryptedEvent.initDataType, encryptedEvent.initData);
});

var mediaSource = new MediaSource();
mediaSource.addEventListener('sourceopen', function() {
var videoSourceBuffer = mediaSource.addSourceBuffer(videoContentType);
fetchArrayBuffer('GET',
'http://yt-dash-mse-test.commondatastorage.googleapis.com/media/oops_cenc-20121114-145-no-clear-start.mp4',
null,
function(videoArrayBuffer) {
videoSourceBuffer.appendBuffer(videoArrayBuffer);
});
function isTunnelModeSupported(videoContentType) {
if (!MediaSource.isTypeSupported(videoContentType)) {
// If the content type isn't supported at all, it won't be supported in
// tunnel mode.
return false;
}
if (MediaSource.isTypeSupported(createTunnelModeContentType(videoContentType, 'invalid'))) {
// The implementation doesn't understand the `tunnelmode` attribute.
return false;
}
return MediaSource.isTypeSupported(createTunnelModeContentType(videoContentType, 'true'));
}

async function play(videoElementId, videoFileId, optionalAudioFileId) {
const isPrimaryVideo = videoElementId == 'primary-video';

videoContentType = MEDIA_FILES[videoFileId].contentType;
if (isTunnelModeSupported(videoContentType)) {
videoContentType = createTunnelModeContentType(videoContentType, 'true');
}

var mediaKeys = await createMediaKeySystem(isPrimaryVideo, optionalAudioFileId ? MEDIA_FILES[optionalAudioFileId].contentType : MEDIA_FILES['opus_clear'].contentType, videoContentType);
var videoElement = document.getElementById(videoElementId);

if (!isPrimaryVideo && videoElement.setMaxVideoCapabilities) {
videoElement.setMaxVideoCapabilities(MEDIA_FILES[videoFileId].maxVideoCapabilities);
}

var audioSourceBuffer = mediaSource.addSourceBuffer(audioContentType);
fetchArrayBuffer('GET',
'http://yt-dash-mse-test.commondatastorage.googleapis.com/media/oops_cenc-20121114-148.mp4',
null,
function(audioArrayBuffer) {
audioSourceBuffer.appendBuffer(audioArrayBuffer);
videoElement.setMediaKeys(mediaKeys);

mediaKeySession = mediaKeys.createSession();
var licenseServerUrl = MEDIA_FILES[videoFileId].licenseUrl;
mediaKeySession.addEventListener('message', function(messageEvent) {
fetchArrayBuffer('POST', licenseServerUrl, messageEvent.message,
function(licenseArrayBuffer) {
mediaKeySession.update(extractLicense(licenseArrayBuffer));
});
});
});

videoElement.addEventListener('encrypted', function(encryptedEvent) {
mediaKeySession.generateRequest(
encryptedEvent.initDataType, encryptedEvent.initData);
});

var mediaSource = new MediaSource();
mediaSource.addEventListener('sourceopen', async function() {
var videoSourceBuffer = mediaSource.addSourceBuffer(videoContentType);
var audioSourceBuffer;

if (optionalAudioFileId) {
audioSourceBuffer = mediaSource.addSourceBuffer(MEDIA_FILES[optionalAudioFileId].contentType);
}

var videoArrayBuffer = await fetchMediaData(videoFileId);
videoSourceBuffer.appendBuffer(videoArrayBuffer);

videoElement.src = URL.createObjectURL(mediaSource);
videoElement.play();
if (audioSourceBuffer) {
var audioArrayBuffer = await fetchMediaData(optionalAudioFileId);
audioSourceBuffer.appendBuffer(audioArrayBuffer);
}
});

videoElement.src = URL.createObjectURL(mediaSource);
videoElement.play();
}

function getGetParameters() {
var parsedParameters = {};

const urlComponents = window.location.href.split('?');
if (urlComponents.length < 2) {
return parsedParameters;
}

const query = urlComponents[1];
const parameters = query.split('&');

for (parameter of parameters) {
const split = parameter.split('=');
if (split.length == 0) {
continue;
}
if (split.length == 1) {
parsedParameters[split[0]] = '';
} else {
parsedParameters[split[0]] = split[1];
}
}

return parsedParameters;
}

play('primary-video', 'com.widevine.alpha');
window.setTimeout(function() {
play('secondary-video-1', 'com.youtube.widevine.l3');
}, 10000);
window.setTimeout(function() {
play('secondary-video-2', 'com.youtube.widevine.l3');
}, 20000);
window.setTimeout(function() {
play('secondary-video-3', 'com.youtube.widevine.l3');
}, 30000);
function populateMediaFileIds() {
var mediaFileIds = [];
const getParameters = getGetParameters();

mediaFileIds['video0'] = getParameters['video0'] ?? 'vp9_720p_60fps_drm';
mediaFileIds['video1'] = getParameters['video1'] ?? 'h264_720p_24fps_drm';
mediaFileIds['video2'] = getParameters['video2'] ?? 'vp9_720p_60fps_drm';
mediaFileIds['video3'] = getParameters['video3'] ?? 'h264_720p_24fps_drm';
mediaFileIds['audio'] = getParameters['audio'] ?? 'opus_clear';

return mediaFileIds;
}

async function prefetchMediaData(mediaFileIds) {
for (mediaFileId of Object.keys(mediaFileIds)) {
await fetchMediaData(mediaFileIds[mediaFileId]);
}
}

async function main() {
if (window.h5vcc && window.h5vcc.settings) {
h5vcc.settings.set('MediaSource.EnableAvoidCopyingArrayBuffer', 1);
}

const mediaFileIds = populateMediaFileIds();
await prefetchMediaData(mediaFileIds);

play('primary-video', mediaFileIds['video0'], mediaFileIds['audio']);
window.setTimeout(function() {
play('secondary-video-1', mediaFileIds['video1']);
}, 10000);
window.setTimeout(function() {
play('secondary-video-2', mediaFileIds['video2']);
}, 20000);
window.setTimeout(function() {
play('secondary-video-3', mediaFileIds['video3']);
}, 30000);
}


main();
11 changes: 1 addition & 10 deletions starboard/android/apk/app/src/main/java/dev/cobalt/media/MediaCodecUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -718,8 +718,7 @@ public static String findVideoDecoder(
* "" otherwise.
*/
@UsedByNative
public static String findAudioDecoder(
String mimeType, int bitrate, boolean mustSupportTunnelMode) {
public static String findAudioDecoder(String mimeType, int bitrate) {
// Note: MediaCodecList is sorted by the framework such that the best decoders come first.
for (MediaCodecInfo info : new MediaCodecList(MediaCodecList.ALL_CODECS).getCodecInfos()) {
if (info.isEncoder()) {
Expand All @@ -737,14 +736,6 @@ public static String findAudioDecoder(
if (!bitrateRange.contains(bitrate)) {
continue;
}
if (mustSupportTunnelMode
&& !codecCapabilities.isFeatureSupported(CodecCapabilities.FEATURE_TunneledPlayback)) {
continue;
}
// TODO: Determine if we can safely check if an audio codec requires the tunneled playback
// feature. i.e., reject when |mustSupportTunnelMode| == false
// and codecCapabilities.isFeatureRequired(CodecCapabilities.FEATURE_TunneledPlayback) ==
// true.
return name;
}
}
Expand Down
Loading

0 comments on commit 6b6abea

Please sign in to comment.