问题
I am using MediaRecorder API to record videos in web applications. The application has the option to switch between the camera and screen. I am using Canvas to augment stream recording. The logic involves capturing stream from the camera and redirecting it to the video element. This video is then rendered on canvas and the stream from canvas is passed to MediaRecorder.
What I noticed is that switching from screen to video (and vice-versa) works fine as long as the user doesn't switch/minimize the chrome window. The canvas rendering uses requestAnimationFrame and it freezes after the tab loses its focus.
Is there any way to instruct chrome not to pause the execution of requestAnimationFrame? Is there any alternate way to switch streams without impacting MediaRecorder recording?
Update: After reading through the documentation, tabs which play audio or having active websocket connection are not throttled. This is something which we are not doing at this moment. This might be a workaround, but hoping for any alternative solution from community. (setTimeout or setInterval are too throttled and hence not using that, plus it impacts rendering quality)
Update 2: I could able to fix this problem using Worker. Instead of using Main UI Thread for requestAnimationFrame, the worker is invoking the API and the notification is sent to Main Thread via postMessage. Upon completion of rendering by UI Thread, a message is sent back to Worker. There is also a delta period calculation to throttle overwhelming messages from worker.
回答1:
There is an ongoing proposal to add a .replaceTrack() method to the MediaRecorder API, but for the time being, the specs still read
If at any point, a track is added to or removed from stream’s track set, the UA MUST immediately stop gathering data, discard any data that it has gathered [...]
And that's what is implemented.
So we still have to rely on ugly hacks to make this by ourselves...
Here is one, which seems to correctly work only in Firefox for I still don't know what reasons, using a MediaSource as mixer.
This works this way:
- capture your videos streams,
- record them all using a MediaRecorder per video
- catch the
dataavailableof these MediaRecorders and feed a MediaSource with their chunks - capture the stream of a video element that plays this MediaSource
- record this mixed stream
However this whole setup adds a significant delay (don't be surprised if you have to wait a few seconds before the switching of sources is visible), and it's crazy heavy on the CPU...
Remember below demo will work correctly when page is blurred only in Firefox
{ // remap unstable FF version
const proto = HTMLMediaElement.prototype;
if( !proto.captureStream ) { proto.captureStream = proto.mozCaptureStream; }
}
waitForEvent( document.getElementById( 'starter' ), 'click' )
.then( (evt) => evt.target.parentNode.remove() )
.then( (async() => {
const urls = [
"2/22/Volcano_Lava_Sample.webm",
"/a/a4/BBH_gravitational_lensing_of_gw150914.webm"
].map( (suffix) => "https://upload.wikimedia.org/wikipedia/commons/" + suffix );
const switcher_btn = document.getElementById( 'switcher' );
const stop_btn = document.getElementById( 'stopper' );
const video_out = document.getElementById( 'out' );
const type = 'video/webm; codecs="vp8"';
if( !MediaSource.isTypeSupported( type ) ) {
throw new Error( 'Not Supported' );
}
let stopped = false;
let current = 0;
switcher_btn.onclick = (evt) => { current = +!current; };
console.log( 'loading videos, please wait' );
// see below for 'recordVid'
const recorders = await Promise.all( urls.map( (url, index) => recordVid( url, type ) ) );
const source = new MediaSource();
// create an offscreen video so it doesn't get paused when hidden
const mixed_vid = document.createElement( 'video' );
mixed_vid.autoplay = true;
mixed_vid.muted = true;
mixed_vid.src = URL.createObjectURL( source );
await waitForEvent( source, 'sourceopen' );
const buffer = source.addSourceBuffer( type );
buffer.mode = "sequence";
// init our requestData loop
appendBuffer();
mixed_vid.play();
await waitForEvent( mixed_vid, 'playing' );
console.clear();
// final recording part below
const mixed_stream = mixed_vid.captureStream();
// only for demo, so we can see what happens now
video_out.srcObject = mixed_stream;
const rec = new MediaRecorder( mixed_stream );
const chunks = [];
rec.ondataavailable = (evt) => chunks.push( evt.data );
rec.onstop = (evt) => {
stopped = true;
const final_file = new Blob( chunks );
recorders.forEach( (rec) => rec.stop() );
// only for demo, since we did set its srcObject
video_out.srcObject = null;
video_out.src = URL.createObjectURL( final_file );
switcher_btn.remove();
stop_btn.remove();
};
stop_btn.onclick = (evt) => rec.stop();
rec.start();
// requestData loop
async function appendBuffer() {
if( stopped ) { return; }
const chunks = await Promise.all( recorders.map( rec => rec.requestData() ) );
const chunk = chunks[ current ];
// first iteration is generally empty
if( !chunk.byteLength ) { setTimeout( appendBuffer, 100 ); return; }
buffer.appendBuffer( chunk );
await waitForEvent( buffer, 'update' );
appendBuffer();
};
}))
.catch( console.error )
// some helpers below
// returns a video loaded to given url
function makeVid( url ) {
const vid = document.createElement('video');
vid.crossOrigin = true;
vid.loop = true;
vid.muted = true;
vid.src = url;
return vid.play()
.then( (_) => vid );
}
/* Records videos from given url
** returns an object which exposes two method
** 'requestData()' returns a Promise resolved by the latest available chunk of data
** 'stop()' stops the video element and the recorder
*/
async function recordVid( url, type ) {
const player = await makeVid( url );
const stream = videoStream( player.captureStream() );
// const stream = await navigator.mediaDevices.getUserMedia({ video: true });
const recorder = new MediaRecorder( stream, { mimeType: type } );
const chunks = [];
recorder.start( );
return {
requestData() {
recorder.requestData();
const data_prom = waitForEvent( recorder, "dataavailable" )
.then( (evt) => evt.data.arrayBuffer() );
return data_prom;
},
stop() { recorder.stop(); player.pause(); }
};
}
// removes the audio tracks from a MediaStream
function videoStream( mixed ) {
return new MediaStream( mixed.getVideoTracks() );
}
// Promisifies EventTarget.addEventListener
function waitForEvent( target, type ) {
return new Promise( (res) => target.addEventListener( type, res, { once: true } ) );
}video { max-height: 100vh; max-width: 100vw; vertical-align: top; }
.overlay {
background: #ded;
position: fixed;
z-index: 999;
height: 100vh;
width: 100vw;
top: 0;
left: 0;
display: flex;
align-items: center;
justify-content: center;
}<div class="overlay">
<button id="starter">start demo</button>
</div>
<button id="switcher">switch source</button>
<button id="stopper">stop recording</button>
<video id="out" muted controls autoplay></video>An other such hack is to create a local RTC connection, and to record the receiving end.
Though, while on paper this should have worked, my Firefox will just weirdly mix up both streams in something I would advise epileptic readers to avoid, and Chrome recorders produce a single frame video, possibly because the video size does change...
So, this currently doesn't seem to work anywhere, but here it is in case browsers fix their bugs before implementing the MediaRecorder.replaceTrack.
{ // remap unstable FF version
const proto = HTMLMediaElement.prototype;
if( !proto.captureStream ) { proto.captureStream = proto.mozCaptureStream; }
}
waitForEvent( document.getElementById( 'starter' ), 'click' )
.then( (evt) => evt.target.parentNode.remove() )
.then( (async() => {
const urls = [
"2/22/Volcano_Lava_Sample.webm",
"/a/a4/BBH_gravitational_lensing_of_gw150914.webm"
].map( (suffix) => "https://upload.wikimedia.org/wikipedia/commons/" + suffix );
const switcher_btn = document.getElementById( 'switcher' );
const stop_btn = document.getElementById( 'stopper' );
const video_out = document.getElementById( 'out' );
let current = 0;
// see below for 'recordVid'
const video_tracks = await Promise.all( urls.map( (url, index) => getVideoTracks( url ) ) );
const mixable_stream = await mixableStream( video_tracks[ current ].track );
switcher_btn.onclick = async (evt) => {
current = +!current;
await mixable_stream.replaceTrack( video_tracks[ current ].track );
};
// final recording part below
// only for demo, so we can see what happens now
video_out.srcObject = mixable_stream.stream;
const rec = new MediaRecorder( mixable_stream.stream );
const chunks = [];
rec.ondataavailable = (evt) => chunks.push( evt.data );
rec.onerror = console.log;
rec.onstop = (evt) => {
const final_file = new Blob( chunks );
video_tracks.forEach( (track) => track.stop() );
// only for demo, since we did set its srcObject
video_out.srcObject = null;
video_out.src = URL.createObjectURL( final_file );
switcher_btn.remove();
stop_btn.remove();
const anchor = document.createElement( 'a' );
anchor.download = 'file.webm';
anchor.textContent = 'download';
anchor.href = video_out.src;
document.body.prepend( anchor );
};
stop_btn.onclick = (evt) => rec.stop();
rec.start();
}))
.catch( console.error )
// some helpers below
// creates a mixable stream
async function mixableStream( initial_track ) {
const source_stream = new MediaStream( [] );
const pc1 = new RTCPeerConnection();
const pc2 = new RTCPeerConnection();
pc1.onicecandidate = (evt) => pc2.addIceCandidate( evt.candidate );
pc2.onicecandidate = (evt) => pc1.addIceCandidate( evt.candidate );
const wait_for_stream = waitForEvent( pc2, 'track')
.then( evt => new MediaStream( [ evt.track ] ) );
pc1.addTrack( initial_track, source_stream );
await waitForEvent( pc1, 'negotiationneeded' );
try {
await pc1.setLocalDescription(await pc1.createOffer());
await pc2.setRemoteDescription(pc1.localDescription);
await pc2.setLocalDescription(await pc2.createAnswer());
await pc1.setRemoteDescription(pc2.localDescription);
} catch (e) {
console.error(e);
}
return {
stream: await wait_for_stream,
async replaceTrack( new_track ) {
const sender = pc1.getSenders().find( ( { track } ) => track.kind == new_track.kind );
console.log( new_track );
return sender && sender.replaceTrack( new_track ) ||
Promise.reject('no such track');
}
}
}
// returns a video loaded to given url
function makeVid( url ) {
const vid = document.createElement('video');
vid.crossOrigin = true;
vid.loop = true;
vid.muted = true;
vid.src = url;
return vid.play()
.then( (_) => vid );
}
/* Records videos from given url
** @method stop() ::pauses the linked <video>
** @property track ::the video track
*/
async function getVideoTracks( url ) {
const player = await makeVid( url );
const track = player.captureStream().getVideoTracks()[ 0 ];
return {
track,
stop() { player.pause(); }
};
}
// Promisifies EventTarget.addEventListener
function waitForEvent( target, type ) {
return new Promise( (res) => target.addEventListener( type, res, { once: true } ) );
}video { max-height: 100vh; max-width: 100vw; vertical-align: top; }
.overlay {
background: #ded;
position: fixed;
z-index: 999;
height: 100vh;
width: 100vw;
top: 0;
left: 0;
display: flex;
align-items: center;
justify-content: center;
}<div class="overlay">
<button id="starter">start demo</button>
</div>
<button id="switcher">switch source</button>
<button id="stopper">stop recording</button>
<video id="out" muted controls autoplay></video>Then, currently the best is probably to still go the canvas way, with the Web Audio Timer I made for when the page is blurred, even though this will not work on Firefox.
回答2:
I had the same problem and trying to figure it out without too much complexity such as Canvas or SourceBuffer.
I used the PeerConnection for same page to make a connection. Once the connection is made you can use a rtpSender via peerconnection.addTrack And from there you can easily switch.
I just made a library and a demo that you can find: https://github.com/meething/StreamSwitcher/
来源：https://stackoverflow.com/questions/59846230/mediarecorder-switch-video-tracks