Skip to content

Commit

Permalink
Track frames
Browse files Browse the repository at this point in the history
  • Loading branch information
@friendlymatthew
friendlymatthew committed Oct 9, 2024
1 parent 99b288c commit d3360d1
Show file tree
Hide file tree
Showing 3 changed files with 170 additions and 73 deletions.
136 changes: 84 additions & 52 deletions index.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,12 @@
font-family: "Times New Roman", Times, serif;
height: 100vh;
width: 100vw;
padding: 1px;
padding: 2px;
}

body {
display: flex;
flex-direction: column;
}

video {
Expand All @@ -26,33 +31,43 @@
display: none;
}

#videos {
flex-shrink: 0;
ul {
list-style-type: none;
padding-left: 0;
margin: 0;
}

#environment {
display: flex;
width: 100vw;
li {
margin: 0;
padding: 2px;
}

#divider {
width: 1px;
background-color: slategray;
margin: 0 16px;
height: auto;
li p {
margin: 0;
padding: 2px;
}

#decoder {
flex-grow: 1;
.video-header {
height: 25px;
}

#decoder-svc {
#environment {
display: flex;
gap: 20px;
width: 100vw;
background-color: antiquewhite;
}

#decoder-svc:disabled {
display: none;

#layers {
flex-grow: 1;
overflow-y: scroll;
margin: 0;
padding: 0;
}

#l3t3-entries {
margin: 0;
padding: 0;
}

</style>
Expand All @@ -73,51 +88,68 @@
</p>
</div>

<div id="buttons">
<button id="startButton">Start (Press 1)</button>
<button disabled id="callButton">Call (Press 2)</button>
<button disabled id="hangupButton">Hang Up (Press 3)</button>
</div>

<div id="environment">
<div id="videos">
<div>
<p>WebRTC Sender (Computer)</p>
<video autoplay id="video1" muted playsinline></video>
<div id="sender">
<div class="video-header">
<button id="startButton">Start (Press 1)</button>
<button disabled id="callButton">Call (Press 2)</button>
<button disabled id="hangupButton">Hang Up (Press 3)</button>
</div>
<div>WebRTC Sender (Computer)</div>
<video autoplay id="video1" muted playsinline></video>
</div>

<div>
<p>WebRTC Receiver (Remote Peer)</p>
<video autoplay id="video2" playsinline></video>
<div>
<div style="height: 25px;">
<label for="svc-mode">Scalability mode:</label>

<select id="svc-mode" name="svc-layers">
<option value="true">L3T3</option>
<option value="false">S3T3</option>
</select>
</div>
<div>WebRTC Receiver (Remote Peer)</div>
<video autoplay id="video2" playsinline></video>
<video autoplay id="video2a" playsinline></video>
</div>
<div>
<div class="video-header">
<label for="spatial">Spatial Layer:</label>

<select id="spatial" name="spatial-layers">
<option value="3">Layer 3</option>
<option value="2">Layer 2</option>
<option value="1">Base Layer</option>
</select>

<label for="temporal">Temporal Layer:</label>

<select id="temporal" name="temporal-layers">
<option value="3">Layer 3</option>
<option value="2">Layer 2</option>
<option value="1">Base Layer</option>
</select>
</div>
<div>Frames Decoded by WebCodec's VideoDecoder</div>
<video autoplay id="video3" playsinline></video>
</div>
<div id="divider"></div>
<div id="decoder">
</div>

<div id="layers">
<p>
*There are actually two remote peers with different scalability modes. Above the WebRTC receiver, you can toggle
video elements.
</p>

<div>
<div>
<p>Frames Decoded by WebCodec's VideoDecoder</p>
<video autoplay id="video3" playsinline></video>
<p>L3T3 Encoded Frames</p>
</div>
<div id="decoder-svc">
<div>
<label for="spatial">Spatial Layer:</label>

<select id="spatial" name="spatial-layers">
<option value="3">Layer 3</option>
<option value="2">Layer 2</option>
<option value="1">Base Layer</option>
</select>
</div>
<div>
<label for="temporal">Temporal Layer:</label>

<select id="temporal" name="temporal-layers">
<option value="3">Layer 3</option>
<option value="2">Layer 2</option>
<option value="1">Base Layer</option>
</select>
</div>
<div id="l3t3-entries">
</div>
</div>

</div>

<script src="https://webrtc.github.io/adapter/adapter-latest.js"></script>
Expand Down
75 changes: 75 additions & 0 deletions js/main.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@

const video1 = document.querySelector('video#video1');
const video2 = document.querySelector('video#video2');
const video2a = document.querySelector('video#video2a')
const video3 = document.querySelector('video#video3');

video2.addEventListener('resize', () => {
Expand Down Expand Up @@ -216,12 +217,84 @@ worker.postMessage({
operation: 'init', writable
}, [writable]);

const encodedL3T3Frames = new Map();

worker.onmessage = ({data}) => {
if (data.operation === 'track-ready') {
video3.srcObject = new MediaStream([mediaStreamTrackGenerator]);
}

if (data.operation === "encoded-frame") {
const {timestamp, spatialIndex, temporalIndex, size, type} = data;

if (encodedL3T3Frames.has(timestamp)) {
const layers = encodedL3T3Frames.get(timestamp);
layers.push({
spatialIndex,
temporalIndex,
size,
type,
});

encodedL3T3Frames.set(timestamp, layers);
updateEncodedFrame(timestamp, layers);

} else {
encodedL3T3Frames.set(timestamp, [{
spatialIndex,
temporalIndex,
size,
type,
}]);

appendEncodedFrame(timestamp, [{
spatialIndex,
temporalIndex,
size,
type
}])
}
}
};

function updateEncodedFrame(timestamp, frames) {
const entry = document.querySelector(`#entry-${timestamp} ul`);
if (entry) {
entry.innerHTML = frames.map(f => `
<li style="padding: 2px; background-color: ${f.type === 'delta' ? 'yellow' : 'lawngreen'};">
<p>${JSON.stringify({
spatialIndex: f.spatialIndex,
temporalIndex: f.temporalIndex,
size: f.size,
}, null, 2)}</p>
</li>
`).join('')
}
}

function appendEncodedFrame(timestamp, frames) {
const container = document.getElementById('l3t3-entries');
const frameEntry = document.createElement('div');
frameEntry.setAttribute('id', `entry-${timestamp}`);
frameEntry.innerHTML = `
<div><strong>Timestamp ${timestamp}:</strong></div>
<ul>
${frames.map(f => `
<li style="background-color: ${f.type === 'delta' ? 'yellow' : 'lawngreen'};">
<p>${JSON.stringify({
spatialIndex: f.spatialIndex,
temporalIndex: f.temporalIndex,
size: f.size,
}, null, 2)}</p>
</li>
`).join('')}
</ul>
`;

container.appendChild(frameEntry);
}

async function call() {
callButton.disabled = true;
hangupButton.disabled = false;
Expand All @@ -247,6 +320,8 @@ async function call() {

function hangup() {
// console.log('Ending call');

console.log(encodedL3T3Frames);
startToEnd.close();
hangupButton.disabled = true;
callButton.disabled = false;
Expand Down
32 changes: 11 additions & 21 deletions js/worker.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,33 +68,24 @@ initializeDecoder();
let highestSpatialLayer = 3, highestTemporalLayer = 3;


const encoded = new Map();


async function handleTransform(operation, readable, writable) {
if (operation === 'encode') {
const transformer = new TransformStream({
async transform(encodedFrame, controller) {
const {temporalIndex, spatialIndex, width, height} = encodedFrame.getMetadata();
const {timestamp, data, type} = encodedFrame;

if (encoded.has(timestamp)) {
const layers = encoded.get(timestamp);
layers.push({
spatialIndex,
temporalIndex,
size: data.byteLength
});

encoded.set(timestamp, layers);

} else {
encoded.set(timestamp, [{
spatialIndex,
temporalIndex,
size: data.byteLength
}]);
}
const size = data.byteLength;

postMessage({
operation: 'encoded-frame',
timestamp,
spatialIndex,
temporalIndex,
size,
type,
});


controller.enqueue(encodedFrame);
}
Expand Down Expand Up @@ -123,7 +114,6 @@ async function handleTransform(operation, readable, writable) {
await videoDecoder.decode(chunk);
}

console.log(`Encoded catalog: `, encoded);
},
});
await readable
Expand Down

0 comments on commit d3360d1

Please sign in to comment.