Skip to content

Commit

Permalink
ScarletsAudioStreamer, ScarletsMediaPresenter:
Browse files Browse the repository at this point in the history 
- onStop callback for more granulated flow control

ScarletsMediaPresenter:
- reset the buffer header on every play
- check if passed mimeType is a supported mimeType
- MediaRecorder only supports 1 parameters. this throw error on firefox

FIX latency NaN in realtimeBufferPlay and refactor local variables for coherence

I've fixed a bad calc of the latency that causes NaN values at the method 'realtimeBufferPlay' . Also I've refactored some local variables for better coherence

Refactor local variables for coherence

I've refactored some local variables for better coherence

FIXED header latency when is a video

More fixes. Running on chrome, firefox, safari
  • Loading branch information
@orlandolaycos
orlandolaycos committed Aug 27, 2021
1 parent 5934628 commit 3e6757c
Show file tree
Hide file tree
Showing 6 changed files with 92 additions and 48 deletions.
64 changes: 42 additions & 22 deletions dist/SFMediaStream.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,8 +91,6 @@ var userInteracted = false;
})();
// Minimum 3 bufferElement
var ScarletsAudioStreamer = function(chunksDuration){
var bufferElement = 3;

if(!chunksDuration) chunksDuration = 1000;
var chunksSeconds = chunksDuration/1000;

Expand All @@ -104,6 +102,8 @@ var ScarletsAudioStreamer = function(chunksDuration){
scope.mimeType = null;
scope.bufferElement = [];

scope.onStop = null;

scope.audioContext = ScarletsMedia.audioContext;
scope.outputNode = false; // Set this to a connectable Audio Node

Expand Down Expand Up @@ -145,6 +145,7 @@ var ScarletsAudioStreamer = function(chunksDuration){
mediaBuffer.stop();
scope.playing = false;
scope.buffering = false;
if (scope.onStop) scope.onStop();
}

scope.setBufferHeader = function(packet){
Expand Down Expand Up @@ -227,14 +228,16 @@ var ScarletsAudioStreamer = function(chunksDuration){
}

var bufferElementIndex = 0;
scope.realtimeBufferPlay = function(arrayBuffer){
scope.realtimeBufferPlay = function(packet){
if(scope.playing === false) return;

if(scope.debug) console.log("Receiving data", arrayBuffer[0].byteLength);
if(arrayBuffer[0].byteLength === 0) return;
arrayBuffer = arrayBuffer[0];
var arrayBuffer = packet[0];
var streamingTime = packet[1];

scope.latency = (Number(String(Date.now()).slice(-5, -3)) - arrayBuffer[1]) + chunksSeconds + scope.audioContext.baseLatency;
if(scope.debug) console.log("Receiving data", arrayBuffer.byteLength);
if(arrayBuffer.byteLength === 0) return;

scope.latency = (Number(String(Date.now()).slice(-5, -3)) - streamingTime) + chunksSeconds + scope.audioContext.baseLatency;

var index = bufferElementIndex;
bufferElementIndex++;
Expand All @@ -252,18 +255,22 @@ var ScarletsAudioStreamer = function(chunksDuration){
// ====== Synchronous Playing ======
// Play next audio when last audio was finished

scope.receiveBuffer = function(arrayBuffer){
scope.receiveBuffer = function(packet){
if(scope.playing === false || !mediaBuffer.append) return;

mediaBuffer.append(arrayBuffer[0]);
var arrayBuffer = packet[0];
var streamingTime = packet[1];

mediaBuffer.append(arrayBuffer);

if(audioElement.paused)
audioElement.play();

scope.latency = (Number(String(Date.now()).slice(-5, -3)) - arrayBuffer[1]) + scope.audioContext.baseLatency + chunksSeconds;
scope.latency = (Number(String(Date.now()).slice(-5, -3)) - streamingTime) + scope.audioContext.baseLatency + chunksSeconds;
if(scope.debug) console.log("Total latency: "+scope.latency);
}
}

var BufferHeader = {
"audio/webm;codecs=opus": "GkXfo59ChoEBQveBAULygQRC84EIQoKEd2VibUKHgQRChYECGFOAZwH/////////FUmpZpkq17GDD0JATYCGQ2hyb21lV0GGQ2hyb21lFlSua7+uvdeBAXPFh7o5nyc1kHqDgQKGhkFfT1BVU2Oik09wdXNIZWFkAQIAAIC7AAAAAADhjbWERzuAAJ+BAmJkgSAfQ7Z1Af/////////ngQCjjIEAAID/A//+//7//qM="
};
Expand Down Expand Up @@ -780,6 +787,7 @@ var ScarletsMediaPresenter = function(options, latency){

scope.onRecordingReady = null;
scope.onBufferProcess = null;
scope.onStop = null;

scope.mediaRecorder = null;
scope.recordingReady = false;
Expand All @@ -798,11 +806,14 @@ var ScarletsMediaPresenter = function(options, latency){
// Deprecated
scope.options = options;

scope.polyfill = void 0;

var mediaType = options.video ? 'video' : 'audio';

// Check supported mimeType and codecs for the recorder
if (options.mimeType && !MediaRecorder.isTypeSupported(options.mimeType)) {
console.log("MediaRecorder doesn't supports mimetype " + options.mimeType);
options.mimeType = null;
}

if(!options.mimeType){
var supportedMimeType = false;
var codecsList = mediaType === 'audio' ? audioCodecs : videoCodecs;
Expand All @@ -826,7 +837,8 @@ var ScarletsMediaPresenter = function(options, latency){
break;
}
options.mimeType = supportedMimeType;
console.log("mimeType: "+supportedMimeType);

if (scope.debug) console.log("mimeType: "+supportedMimeType);
}

var mediaGranted = function(mediaStream) {
Expand All @@ -850,18 +862,19 @@ var ScarletsMediaPresenter = function(options, latency){
scope.bufferHeader = null;
var bufferHeaderLength = false;

scope.mediaRecorder = new MediaRecorder(mediaStream, options, scope.polyfill);
scope.mediaRecorder = new MediaRecorder(mediaStream);

if(scope.debug) console.log("MediaRecorder obtained");
scope.mediaRecorder.onstart = function(e) {
scope.recording = true;
};

const isVideo = options.video !== void 0;
const headerLatency = isVideo ? 565 : 100;

scope.mediaRecorder.ondataavailable = function(e){
// Stream segments after the header was obtained
if(bufferHeaderLength !== false){
if (scope.bufferHeader && bufferHeaderLength !== false){
var streamingTime = Number(String(Date.now()).slice(-5, -3));
scope.onBufferProcess([e.data, streamingTime]);
return;
Expand Down Expand Up @@ -897,7 +910,7 @@ var ScarletsMediaPresenter = function(options, latency){

scope.recordingReady = true;

if(latency === 100) return;
if(latency === headerLatency) return;

// Record with the custom latency
scope.mediaRecorder.stop();
Expand All @@ -907,7 +920,7 @@ var ScarletsMediaPresenter = function(options, latency){
};

// Get first header
scope.mediaRecorder.start(isVideo ? 565 : 100);
scope.mediaRecorder.start(headerLatency);
}

var pendingConnect = [];
Expand Down Expand Up @@ -1003,9 +1016,12 @@ var ScarletsMediaPresenter = function(options, latency){

// scope.mediaRecorder.ondataavailable = null;
// scope.mediaRecorder.onstart = null;
// scope.bufferHeader = null;

scope.bufferHeader = null;

afterStop = true;

if (scope.onStop) scope.onStop();
};
}

Expand All @@ -1016,6 +1032,7 @@ ScarletsMediaPresenter.isTypeSupported = function(mimeType){
return "MediaRecorder is not supporting this type";
return "Maybe supported";
}

ScarletsMediaEffect.chorus = function(sourceNode){
var context = ScarletsMedia.audioContext;
var output = context.createGain();
Expand Down Expand Up @@ -2353,26 +2370,29 @@ var ScarletsVideoStreamer = function(videoElement, chunksDuration){

mediaBuffer = new MediaBuffer(scope.mimeType, chunksDuration, arrayBuffer);

console.log(mediaBuffer);
videoElement.src = scope.objectURL = mediaBuffer.objectURL;
}

scope.playStream = function(){
scope.playing = true;
}

scope.receiveBuffer = function(arrayBuffer){
scope.receiveBuffer = function(packet){
if(scope.playing === false || !mediaBuffer.append) return;

mediaBuffer.append(arrayBuffer[0]);
var arrayBuffer = packet[0];
var streamingTime = packet[1];

mediaBuffer.append(arrayBuffer[);

if(videoElement.paused)
videoElement.play();

scope.latency = (Number(String(Date.now()).slice(-5, -3)) - arrayBuffer[1]) + scope.audioContext.baseLatency + chunksSeconds;
scope.latency = (Number(String(Date.now()).slice(-5, -3)) - streamingTime) + scope.audioContext.baseLatency + chunksSeconds;
if(scope.debug) console.log("Total latency: "+scope.latency);
}
}

ScarletsMedia.extra = new function(){
var self = this;
self.isMobile = function(){
Expand Down
28 changes: 17 additions & 11 deletions src/AudioStreamer.js
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
// Minimum 3 bufferElement
var ScarletsAudioStreamer = function(chunksDuration){
var bufferElement = 3;

if(!chunksDuration) chunksDuration = 1000;
var chunksSeconds = chunksDuration/1000;

Expand All @@ -13,6 +11,8 @@ var ScarletsAudioStreamer = function(chunksDuration){
scope.mimeType = null;
scope.bufferElement = [];

scope.onStop = null;

scope.audioContext = ScarletsMedia.audioContext;
scope.outputNode = false; // Set this to a connectable Audio Node

Expand Down Expand Up @@ -54,6 +54,7 @@ var ScarletsAudioStreamer = function(chunksDuration){
mediaBuffer.stop();
scope.playing = false;
scope.buffering = false;
if (scope.onStop) scope.onStop();
}

scope.setBufferHeader = function(packet){
Expand Down Expand Up @@ -136,14 +137,16 @@ var ScarletsAudioStreamer = function(chunksDuration){
}

var bufferElementIndex = 0;
scope.realtimeBufferPlay = function(arrayBuffer){
scope.realtimeBufferPlay = function(packet){
if(scope.playing === false) return;

if(scope.debug) console.log("Receiving data", arrayBuffer[0].byteLength);
if(arrayBuffer[0].byteLength === 0) return;
arrayBuffer = arrayBuffer[0];
var arrayBuffer = packet[0];
var streamingTime = packet[1];

scope.latency = (Number(String(Date.now()).slice(-5, -3)) - arrayBuffer[1]) + chunksSeconds + scope.audioContext.baseLatency;
if(scope.debug) console.log("Receiving data", arrayBuffer.byteLength);
if(arrayBuffer.byteLength === 0) return;

scope.latency = (Number(String(Date.now()).slice(-5, -3)) - streamingTime) + chunksSeconds + scope.audioContext.baseLatency;

var index = bufferElementIndex;
bufferElementIndex++;
Expand All @@ -161,15 +164,18 @@ var ScarletsAudioStreamer = function(chunksDuration){
// ====== Synchronous Playing ======
// Play next audio when last audio was finished

scope.receiveBuffer = function(arrayBuffer){
scope.receiveBuffer = function(packet){
if(scope.playing === false || !mediaBuffer.append) return;

mediaBuffer.append(arrayBuffer[0]);
var arrayBuffer = packet[0];
var streamingTime = packet[1];

mediaBuffer.append(arrayBuffer);

if(audioElement.paused)
audioElement.play();

scope.latency = (Number(String(Date.now()).slice(-5, -3)) - arrayBuffer[1]) + scope.audioContext.baseLatency + chunksSeconds;
scope.latency = (Number(String(Date.now()).slice(-5, -3)) - streamingTime) + scope.audioContext.baseLatency + chunksSeconds;
if(scope.debug) console.log("Total latency: "+scope.latency);
}
}
}
5 changes: 5 additions & 0 deletions src/BufferHeader.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,11 @@ var BufferHeader = {
};

function getBufferHeader(type) {
if (!window.chrome && type === "audio/webm;codecs=opus" ) {
// this header is only for chrome based brosers
return false;
}

var buff = BufferHeader[type];
if(buff === void 0) return false;

Expand Down
4 changes: 3 additions & 1 deletion src/MediaBuffer.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,9 @@ var MediaBuffer = function(mimeType, chunksDuration, bufferHeader){
if(sourceBuffer === null)
return false;

if(sourceBuffer.buffered.length === 2)
if (!sourceBuffer.updating && sourceBuffer.buffered.length === 2)
// The problem of accessing to 'sourceBuffer.buffered' is that after you append data, the SourceBuffer instance becomes temporarily unusable while it's working.
// During this time, the SourceBuffer's updating property will be set to true, so it's easy to check for.
console.log('something wrong');

if(totalTime >= 20000)
Expand Down
27 changes: 18 additions & 9 deletions src/MediaPresenter.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ var ScarletsMediaPresenter = function(options, latency){

scope.onRecordingReady = null;
scope.onBufferProcess = null;
scope.onStop = null;

scope.mediaRecorder = null;
scope.recordingReady = false;
Expand All @@ -45,11 +46,14 @@ var ScarletsMediaPresenter = function(options, latency){
// Deprecated
scope.options = options;

scope.polyfill = void 0;

var mediaType = options.video ? 'video' : 'audio';

// Check supported mimeType and codecs for the recorder
if (options.mimeType && !MediaRecorder.isTypeSupported(options.mimeType)) {
console.log("MediaRecorder doesn't supports mimetype " + options.mimeType);
options.mimeType = null;
}

if(!options.mimeType){
var supportedMimeType = false;
var codecsList = mediaType === 'audio' ? audioCodecs : videoCodecs;
Expand All @@ -73,7 +77,8 @@ var ScarletsMediaPresenter = function(options, latency){
break;
}
options.mimeType = supportedMimeType;
console.log("mimeType: "+supportedMimeType);

if (scope.debug) console.log("mimeType: "+supportedMimeType);
}

var mediaGranted = function(mediaStream) {
Expand All @@ -97,18 +102,19 @@ var ScarletsMediaPresenter = function(options, latency){
scope.bufferHeader = null;
var bufferHeaderLength = false;

scope.mediaRecorder = new MediaRecorder(mediaStream, options, scope.polyfill);
scope.mediaRecorder = new MediaRecorder(mediaStream, options);

if(scope.debug) console.log("MediaRecorder obtained");
scope.mediaRecorder.onstart = function(e) {
scope.recording = true;
};

const isVideo = options.video !== void 0;
const headerLatency = isVideo ? 565 : 100;

scope.mediaRecorder.ondataavailable = function(e){
// Stream segments after the header was obtained
if(bufferHeaderLength !== false){
if (bufferHeaderLength !== false){
var streamingTime = Number(String(Date.now()).slice(-5, -3));
scope.onBufferProcess([e.data, streamingTime]);
return;
Expand Down Expand Up @@ -144,7 +150,7 @@ var ScarletsMediaPresenter = function(options, latency){

scope.recordingReady = true;

if(latency === 100) return;
if(latency === headerLatency) return;

// Record with the custom latency
scope.mediaRecorder.stop();
Expand All @@ -154,7 +160,7 @@ var ScarletsMediaPresenter = function(options, latency){
};

// Get first header
scope.mediaRecorder.start(isVideo ? 565 : 100);
scope.mediaRecorder.start(headerLatency);
}

var pendingConnect = [];
Expand Down Expand Up @@ -250,9 +256,12 @@ var ScarletsMediaPresenter = function(options, latency){

// scope.mediaRecorder.ondataavailable = null;
// scope.mediaRecorder.onstart = null;
// scope.bufferHeader = null;

scope.bufferHeader = null;

afterStop = true;

if (scope.onStop) scope.onStop();
};
}

Expand All @@ -262,4 +271,4 @@ ScarletsMediaPresenter.isTypeSupported = function(mimeType){
if(!MediaRecorder.isTypeSupported(mimeType))
return "MediaRecorder is not supporting this type";
return "Maybe supported";
}
}
Loading

0 comments on commit 3e6757c

Please sign in to comment.