ALPS-700 - Omnitone crash

README.md

@@ -34,7 +34,7 @@ ForgeJS has the following dependencies:
 
 - [three.js](https://threejs.org/) r86 ([MIT license](https://github.com/mrdoob/three.js/blob/dev/LICENSE))
 - [hammer.js](http://hammerjs.github.io/) 2.0.8 ([MIT license](https://github.com/hammerjs/hammer.js/blob/master/LICENSE.md))
-- [omnitone](http://googlechrome.github.io/omnitone/#home) 1.0.1 ([Apache 2.0 license](https://github.com/GoogleChrome/omnitone/blob/master/LICENSE))
+- [omnitone](http://googlechrome.github.io/omnitone/#home) 1.0.6 ([Apache 2.0 license](https://github.com/GoogleChrome/omnitone/blob/master/LICENSE))
 - [dash.js](https://github.com/Dash-Industry-Forum/dash.js) 2.6.0 ([BSD license](https://github.com/Dash-Industry-Forum/dash.js/blob/development/LICENSE.md))
 
 > NOTE: We made a custom build of three.js with some classes concatenated to it. These classes are included in the original three.js repository but not concatenated in the main build. We added EffectComposer, RenderPass, ClearPass, MaskPass, ShaderPass, TexturePass and CopyShader in our three.custom.min.js.

externs/lib/omnitone.ext.js

@@ -9,51 +9,151 @@
 var Omnitone = {};
 
 /**
- * Create a singleton FOADecoder instance.
+ * Create a FOARenderer, the first-order ambisonic decoder and the optimized binaural renderer.
  * @param {AudioContext} context - Associated AudioContext.
- * @param {HTMLMediaElement} element - Video or Audio DOM element to be streamed.
- * @param {FOADecoderOptions} options - Options for FOA decoder.
- * @return {FOADecoder}
+ * @param {FOARendererConfig} config
+ * @return {FOARenderer}
  */
-Omnitone.createFOADecoder = function (context, element, options) {};
+Omnitone.createFOARenderer = function(context, config) {};
 
 /**
- * @typedef {{HRTFSetUrl:(string|undefined), postGainDB:(number|undefined), channelMap:(Array<number>|undefined)}}
- * @name FOADecoderOptions
- * @property {string} HRTFSetUrl - Base URL for the cube HRTF sets.
- * @property {number} postGainDB - Post-decoding gain compensation in dB.
- * @property {Array<number>} channelMap -  Custom channel map.
+ * @typedef {{channelMap:(Array|undefined), hrirPathList:(Array|undefined), renderingMode:(string|undefined)}}
+ * @name FOARendererConfig
+ * @property {Array} channelMap - Custom channel routing map. Useful for handling the inconsistency in browser's multichannel audio decoding.
+ * @property {Array} hrirPathList - A list of paths to HRIR files. It overrides the internal HRIR list if given.
+ * @property {string} renderingMode - Rendering mode.
  */
-var FOADecoderOptions;
+var FOARendererConfig;
 
 /**
+ * Creates HOARenderer for higher-order ambisonic decoding and the optimized binaural rendering.
+ * @param {AudioContext} context - Associated AudioContext.
+ * @param {HOARendererConfig} config
+ * @return {HOARenderer}
+ */
+Omnitone.createHOARenderer = function(context, config) {};
+
+/**
+ * @typedef {{ambisonicOrder:(number|undefined), hrirPathList:(Array|undefined), renderingMode:(string|undefined)}}
+ * @name HOARendererConfig
+ * @property {number} ambisonicOrder - Ambisonic order.
+ * @property {Array} hrirPathList - A list of paths to HRIR files. It overrides the internal HRIR list if given.
+ * @property {string} renderingMode - Rendering mode.
+ */
+var HOARendererConfig;
+
+/**
+ * Omnitone FOA renderer class. Uses the optimized convolution technique.
  * @constructor
  * @param {AudioContext} context - Associated AudioContext.
- * @param {HTMLMediaElement} element - Target video or audio element for streaming.
- * @param {FOADecoderOptions} options
- * @return {!FOADecoder}
+ * @param {FOARendererConfig} config
+ * @return {!FOARenderer}
  */
-function FOADecoder(context, element, options) {};
+function FOARenderer(context, config) {};
 
 /**
- * @param  {Object} cameraMatrix - The Matrix4 object of the Three.js camera.
+ * @const
  */
-FOADecoder.prototype.setRotationMatrixFromCamera = function(cameraMatrix) {};
+Omnitone.FOARenderer = FOARenderer;
 
 /**
- * @param {string} mode - Decoding mode.
- * When the mode is 'bypass' the decoder is disabled and bypass the input stream to the output.
- * Setting the mode to 'ambisonic' activates the decoder.
- * When the mode is 'off', all the processing is completely turned off saving the CPU power.
+ * Initializes and loads the resource for the renderer.
+ */
+FOARenderer.prototype.initialize = function() {};
+
+/**
+ * Set the channel map.
+ * @param {Array<number>} channelMap - Custom channel routing for FOA stream.
+ */
+FOARenderer.prototype.setChannelMap = function(channelMap) {};
+
+/**
+ * Updates the rotation matrix with 3x3 matrix.
+ * @param {Array<number>} rotationMatrix3 - A 3x3 rotation matrix. (column-major)
  */
-FOADecoder.prototype.setMode = function(mode) {};
+FOARenderer.prototype.setRotationMatrix3 = function(rotationMatrix3) {};
 
 /**
- *
+ * Updates the rotation matrix with 4x4 matrix.
+ * @param {Array<number>} rotationMatrix4 - A 4x4 rotation matrix. (column-major)
  */
-FOADecoder.prototype.initialize = function() {};
+FOARenderer.prototype.setRotationMatrix4 = function(rotationMatrix4) {};
+
+/**
+ * Set the rotation matrix from a Three.js camera object. Deprecated in V1, and this exists only for the backward compatiblity. Instead, use |setRotatationMatrix4()| with Three.js |camera.worldMatrix.elements|.
+ * @deprecated
+ * @param {Object} cameraMatrix - Matrix4 from Three.js |camera.matrix|.
+ */
+FOARenderer.prototype.setRotationMatrixFromCamera = function(cameraMatrix) {};
+
+/**
+ * Set the rendering mode.
+ * @param {string} mode - Rendering mode.
+ *  - 'ambisonic': activates the ambisonic decoding/binaurl rendering.
+ *  - 'bypass': bypasses the input stream directly to the output. No ambisonic
+ *    decoding or encoding.
+ *  - 'off': all the processing off saving the CPU power.
+ */
+FOARenderer.prototype.setRenderingMode = function(mode) {};
+
+/**
+ * @type {GainNode}
+ */
+FOARenderer.prototype.input;
+
+/**
+ * @type {GainNode}
+ */
+FOARenderer.prototype.output;
+
+/**
+ * Omnitone HOA renderer class. Uses the optimized convolution technique.
+ * @constructor
+ * @param {AudioContext} context - Associated AudioContext.
+ * @param {HOARendererConfig} config
+ * @return {!HOARenderer}
+ */
+function HOARenderer(context, config) {};
 
 /**
  * @const
  */
-Omnitone.FOADecoder = FOADecoder;
+Omnitone.HOARenderer = HOARenderer;
+
+/**
+ * Initializes and loads the resource for the renderer.
+ * @return {Promise}
+ */
+HOARenderer.prototype.initialize = function() {};
+
+/**
+ * Updates the rotation matrix with 3x3 matrix.
+ * @param {Array<number>} rotationMatrix3 - A 3x3 rotation matrix. (column-major)
+ */
+HOARenderer.prototype.setRotationMatrix3 = function(rotationMatrix3) {};
+
+/**
+ * Updates the rotation matrix with 4x4 matrix.
+ * @param {Array<number>} rotationMatrix4 - A 4x4 rotation matrix. (column-major)
+ */
+HOARenderer.prototype.setRotationMatrix4 = function(rotationMatrix4) {};
+
+/**
+ * Set the decoding mode.
+ * @param {string} mode - Decoding mode.
+ *  - 'ambisonic': activates the ambisonic decoding/binaurl rendering.
+ *  - 'bypass': bypasses the input stream directly to the output. No ambisonic
+ *    decoding or encoding.
+ *  - 'off': all the processing off saving the CPU power.
+ */
+HOARenderer.prototype.setRenderingMode = function(mode) {};
+
+/**
+ * @type {GainNode}
+ */
+HOARenderer.prototype.input;
+
+/**
+ * @type {GainNode}
+ */
+HOARenderer.prototype.output;

src/audio/Sound.js

@@ -5,14 +5,14 @@
  * @param {FORGE.Viewer} viewer - The {@link FORGE.Viewer} reference.
  * @param {string} key - The sound file id reference.
  * @param {string} url - The sound file url.
- * @param {boolean=} ambisonic - Is the sound ambisonic and need binaural rendering?
+ * @param {number=} ambisonicOrder - Is it an ambisonic order sound?
  * @extends {FORGE.BaseObject}
  *
  * @todo  Ability to force audio type into config
  * @todo  Make a test plugin that creates sound, add sound to the PluginObjectFactory
  * @todo  Loop during x steps (parameter) only if loop is true
  */
-FORGE.Sound = function(viewer, key, url, ambisonic)
+FORGE.Sound = function(viewer, key, url, ambisonicOrder)
 {
     /**
      * The viewer reference.
@@ -240,29 +240,44 @@ FORGE.Sound = function(viewer, key, url, ambisonic)
     this._z = 0;
 
     /**
-     * FOADecoder is a ready-made FOA decoder and binaural renderer.
-     * @name  FORGE.Sound#_decoder
-     * @type {?FOADecoder}
+     * Ambisonic renderer is a ready-made FOA or HOA decoder and binaural renderer.
+     * @name  FORGE.Sound#_ambisonicsRenderer
+     * @type {?(FOARenderer|HOARenderer)}
      * @private
      */
-    this._decoder = null;
+    this._ambisonicsRenderer = null;
 
     /**
-     * Is it an ambisonical sound?
-     * @name  FORGE.Sound#_ambisonic
-     * @type {boolean}
+     * Media Element Audio souce node element.
+     * @name  FORGE.Sound#_soundElementSource
+     * @type {?MediaElementAudioSourceNode}
      * @private
      */
-    this._ambisonic = ambisonic || false;
+    this._soundElementSource = null;
 
     /**
-     * Default channel map for ambisonic sound.
+     * Is it a an ambisonic order soundtrack?
+     * @name  FORGE.Sound#_ambisonicOrder
+     * @type {number}
+     * @private
+     */
+    this._ambisonicOrder = ambisonicOrder || 0;
+
+    /**
+     * Default channel map for FOA ambisonic sound.
      * @name  FORGE.Sound#_defaultChannelMap
      * @type {Array<number>}
      * @private
      */
     this._defaultChannelMap = [0, 1, 2, 3]; //AMBIX
-    // this._defaultChannelMap = [0, 3, 1, 2]; //FUMA
+
+    /**
+     * Default ambisonics order for HOA renderer.
+     * @name  FORGE.VideoHTML5#_defaultAmbisonicOrder
+     * @type {number}
+     * @private
+     */
+    this._defaultAmbisonicOrder = 3;
 
     /**
      * To save the pending state to be applied after the sound object will be ready.
@@ -424,10 +439,10 @@ FORGE.Sound.prototype.constructor = FORGE.Sound;
  */
 FORGE.Sound.prototype._boot = function()
 {
-    if (this._ambisonic === true && this._isAmbisonic() === false)
+    if (this._isAmbisonic() === false)
     {
         this.log("FORGE.Sound: can't manage ambisonic sound without Google Chrome Omnitone library and WebAudio API.");
-        this._ambisonic = false;
+        this._ambisonicOrder = 0;
     }
 
     //register the uid
@@ -452,11 +467,15 @@ FORGE.Sound.prototype._boot = function()
         this._gainNode.gain.value = this._volume * this._viewer.audio.volume;
         this._gainNode.connect(this._inputNode);
     }
+
+    var loaded = false;
     if (this._viewer.audio.useAudioTag === true || this._isAmbisonic() === true)
     {
         if (this._viewer.cache.has(FORGE.Cache.types.SOUND, this._key) === true)
         {
-            this._loadComplete(this._viewer.cache.get(FORGE.Cache.types.SOUND, this._key));
+            // wait long enough so that a frame has passed (here at 24fps)
+            window.setTimeout(this._loadComplete.bind(this, this._viewer.cache.get(FORGE.Cache.types.SOUND, this._key)), 40);
+            loaded = true;
         }
 
         //Listen to the main volume change to adapt the sound volume accordingly.
@@ -467,7 +486,10 @@ FORGE.Sound.prototype._boot = function()
 
     if (this._url !== "")
     {
-        this._viewer.load.sound(this._key, this._url, this._loadComplete, this, this._isAmbisonic());
+        if (loaded === false)
+        {
+            this._viewer.load.sound(this._key, this._url, this._loadComplete, this, this._isAmbisonic());
+        }
 
         if (this._onLoadStart !== null)
         {
@@ -492,6 +514,11 @@ FORGE.Sound.prototype._loadComplete = function(file)
         return;
     }
 
+    if (this._isAmbisonic() === true)
+    {
+        file.data = file.data.cloneNode(true);
+    }
+
     this._soundFile = file;
 
     this._ready = true;
@@ -531,16 +558,30 @@ FORGE.Sound.prototype._decode = function(file)
 
             if (this._isAmbisonic() === true)
             {
-                // FOA decoder and binaural renderer
-                this._decoder = Omnitone.createFOADecoder(this._context, file.data,
+                // Source
+                this._soundElementSource = this._context.createMediaElementSource(file.data);
+
+                if (this._ambisonicOrder > 1)
+                {
+                    //HOA decoder and binaural renderer (for 2nd and 3rd order)
+                    this._ambisonicsRenderer = Omnitone.createHOARenderer(this._context, {
+                        ambisonicOrder: (this._ambisonicOrder > 1 ? this._ambisonicOrder : this._defaultAmbisonicOrder), // can be 2 or 3
+                        hrirPathList: null,
+                        renderingMode: 'ambisonic'
+                    });
+                }
+                else
                 {
-                    channelMap: this._defaultChannelMap
-                    // HRTFSetUrl: 'YOUR_HRTF_SET_URL', //Base URL for the cube HRTF sets.
-                    // postGainDB: 0, //Post-decoding gain compensation in dB.
-                });
+                    //FOA decoder and binaural renderer (for 1st order)
+                    this._ambisonicsRenderer = Omnitone.createFOARenderer(this._context, {
+                        channelMap: this._defaultChannelMap, // [0, 1, 2, 3]; for AMBIX & [0, 3, 1, 2] for FUMA
+                        hrirPathList: null,
+                        renderingMode: 'ambisonic'
+                    });
+                }
 
-                // Initialize the decoder
-                this._decoder.initialize().then(this._decodeCompleteBind, this._decodeErrorBind);
+                // Initialize the ambisonics renderer
+                this._ambisonicsRenderer.initialize().then(this._decodeCompleteBind, this._decodeErrorBind);
             }
             else
             {
@@ -591,11 +632,10 @@ FORGE.Sound.prototype._dispatchDecodedEvents = function()
  * Event handler for decode complete event, it stores decoding data into the sound file object.
  * @method FORGE.Sound#_decodeComplete
  * @private
- * @param  {?AudioBuffer} buffer - The raw binary data buffer.
  */
 FORGE.Sound.prototype._decodeComplete = function(buffer)
 {
-    if (this._soundFile === null)
+    if (this._ambisonicsRenderer === null)
     {
         this.log("FORGE.Sound._decodeComplete error, sound file is null");
         return;
@@ -606,6 +646,12 @@ FORGE.Sound.prototype._decodeComplete = function(buffer)
         this._soundFile.data = buffer;
     }
 
+    if (this._ambisonicsRenderer !== null)
+    {
+        this._soundElementSource.connect(this._ambisonicsRenderer.input);
+        this._ambisonicsRenderer.output.connect(this._context.destination);
+    }
+
     this._decoded = true;
 
     this._dispatchDecodedEvents();
@@ -736,7 +782,7 @@ FORGE.Sound.prototype._applyPanner = function(connect)
  */
 FORGE.Sound.prototype._isAmbisonic = function()
 {
-    return (this._ambisonic === true && this._viewer.audio.useWebAudio === true && typeof Omnitone !== "undefined");
+    return (this._ambisonicOrder > 0 && this._viewer.audio.useWebAudio === true && typeof Omnitone !== "undefined");
 };
 
 /**
@@ -775,11 +821,11 @@ FORGE.Sound.prototype.update = function()
         }
     }
 
-    if (this._decoder !== null && this._playing === true)
+    if (this._ambisonicsRenderer !== null && this._playing === true)
     {
         // Rotate the binaural renderer based on a Three.js camera object.
-        var m4 = this._viewer.renderer.camera.modelViewInverse;
-        this._decoder.setRotationMatrixFromCamera(m4);
+        var m4 = this._viewer.renderer.camera.modelView;
+        this._ambisonicsRenderer.setRotationMatrix4(m4.elements);
     }
 };
 
@@ -1173,6 +1219,14 @@ FORGE.Sound.prototype.destroy = function()
         this._viewer.audio.onVolumeChange.remove(this._mainVolumeChangeHandler, this);
     }
 
+    if (this._isAmbisonic() === true)
+    {
+        this._ambisonicsRenderer.output.disconnect();
+        this._soundElementSource.disconnect();
+        this._ambisonicsRenderer = null;
+        this._soundElementSource = null;
+    }
+
     this._viewer.audio.onDisable.remove(this._disableSoundHandler, this);
 
     this._viewer.audio.remove(this);
@@ -1363,7 +1417,7 @@ Object.defineProperty(FORGE.Sound.prototype, "ambisonic",
     /** @this {FORGE.Sound} */
     get: function()
     {
-        return this._ambisonic;
+        return this._isAmbisonic();
     }
 });
 

src/media/Media.js

@@ -252,7 +252,12 @@ FORGE.Media.prototype._parseConfig = function(config)
             var scene = this._viewer.story.scene;
 
             // check of the ambisonic state of the video sound prior to the video instanciation
-            this._displayObject = new FORGE.VideoHTML5(this._viewer, this._uid, null, null, (scene.hasSoundTarget(this._uid) === true && scene.isAmbisonic() === true ? true : false));
+            var ambisonicOrder = 0;
+            if (scene.hasSoundTarget(this._uid) === true && scene.isAmbisonic() > 0)
+            {
+                ambisonicOrder = scene.isAmbisonic();
+            }
+            this._displayObject = new FORGE.VideoHTML5(this._viewer, this._uid, null, null, ambisonicOrder);
         }
 
         // At this point, source.url is either a streaming address, a simple

src/plugin/PluginObjectFactory.js

@@ -234,10 +234,10 @@ FORGE.PluginObjectFactory.prototype.button = function(config)
  * @param {?(string|FORGE.VideoQuality|Array<(FORGE.VideoQuality|string)>)=} config - The video configuration object
  * @param {string=} streaming - The video streaming format. Can be "HTML5" or "DASH".
  * @param {string=} qualityMode - The video quality mode. Can be "auto" or "manual".
- * @param {boolean=} ambisonic - 3D sound including ambisonics. For "HTML5" video only.
+ * @param {number=} ambisonicOrder - Ambisonic order to use for FOA/HOA renderer. For "HTML5" video only.
  * @return {(FORGE.VideoHTML5|FORGE.VideoDash)} Returns the created FORGE.Video object.
  */
-FORGE.PluginObjectFactory.prototype.video = function(key, config, streaming, qualityMode, ambisonic)
+FORGE.PluginObjectFactory.prototype.video = function(key, config, streaming, qualityMode, ambisonicOrder)
 {
     var video;
 
@@ -247,7 +247,7 @@ FORGE.PluginObjectFactory.prototype.video = function(key, config, streaming, qua
     }
     else
     {
-        video = new FORGE.VideoHTML5(this._viewer, key, config, qualityMode, ambisonic);
+        video = new FORGE.VideoHTML5(this._viewer, key, config, qualityMode, ambisonicOrder);
     }
 
     video.onDestroy.addOnce(this._destroyObjectHandler, this);

src/render/RenderManager.js

@@ -357,8 +357,22 @@ FORGE.RenderManager.prototype._initSound = function(sceneConfig)
 
         if (typeof soundConfig.source.url !== "undefined" && soundConfig.source.url !== "")
         {
+            // check of the ambisonic state of the video sound prior to the video instanciation
+            var ambisonicOrder = 0;
+            if (sceneConfig.sound.type === FORGE.SoundType.AMBISONIC)
+            {
+                if (typeof sceneConfig.sound.order !== "undefined" && sceneConfig.sound.order !== null && sceneConfig.sound.order > 1)
+                {
+                    ambisonicOrder = sceneConfig.sound.order; // HOA
+                }
+                else
+                {
+                    ambisonicOrder = 1; // FOA
+                }
+            }
+
             // Warning : UID is not registered and applied to the FORGE.Sound object for registration
-            this._mediaSound = new FORGE.Sound(this._viewer, sceneConfig.sound.uid, sceneConfig.sound.source.url, (sceneConfig.sound.type === FORGE.SoundType.AMBISONIC));
+            this._mediaSound = new FORGE.Sound(this._viewer, sceneConfig.sound.uid, sceneConfig.sound.source.url, ambisonicOrder);
 
             if (typeof soundConfig.options !== "undefined" && soundConfig.options !== null)
             {

src/story/Scene.js

@@ -468,19 +468,24 @@ FORGE.Scene.prototype.hasSoundTarget = function(uid)
 };
 
 /**
- * Know if an ambisonic sound is attached to the scene?
+ * Know if an ambisonic order sound is attached to the scene?
  * @method FORGE.Scene#isAmbisonic
- * @return {boolean} Returns true if the scene has an ambisonic sound source, false if not.
+ * @return {number} Returns the ambisonic order number if the scene has an ambisonic sound source, 0 if not.
  */
 FORGE.Scene.prototype.isAmbisonic = function()
 {
     //@todo real check of the UID target object rather then the isAmbisonic method of the FORGE.Scene
     if (this.hasSoundSource() === true && this._config.sound.type === FORGE.SoundType.AMBISONIC)
     {
-        return true;
+        if (typeof this._config.sound.order !== "undefined" && this._config.sound.order !== null && this._config.sound.order > 1)
+        {
+            return this._config.sound.order;
+        }
+
+        return 1;
     }
 
-    return false;
+    return 0;
 };
 
 /**