Merge pull request #92 from make4all/vpotluri/issue87

feat(updated architecture): updated sonification architecture and partial bug fix for noise related demos

Author: venkateshpotluri

README.md

@@ -26,9 +26,9 @@ Runs the mocha test suite
 
 Start a compile watch of the library build and tests for iterative development.
 
-## yarn pretify
+## yarn prettify
 
-Run Prietter and format source files.
+Run Prettier and format source files.
 
 ## bundling
 

src/.vscode/launch.json

@@ -0,0 +1,15 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "type": "pwa-chrome",
+            "request": "launch",
+            "name": "Launch Chrome against localhost",
+            "url": "http://localhost:8080",
+            "webRoot": "${workspaceFolder}"
+        }
+    ]
+}

src/pages/Demo.tsx

@@ -14,10 +14,12 @@ import { IDemoView } from '../views/demos/IDemoView'
 import { DemoSimple } from '../views/demos/DemoSimple'
 import { DemoHighlightRegion } from '../views/demos/DemoHighlightRegion'
 import { op } from 'arquero'
+import { ExperimentalDemoHighlightRegion } from '../views/demos/ExperimentalDemoHighlightRegion'
 
 const DEMO_VIEW_MAP = {
     simple: { value: 'simple', label: 'Simple sonification', component: DemoSimple },
     highlightRegion: { value: 'highlightRegion', label: 'Highlight points for region', component: DemoHighlightRegion },
+    experimentalHighlightRegion: { value: 'experimentalHighlightRegion', label: 'experimental implementation of highlight points for region', component: ExperimentalDemoHighlightRegion },
 }
 
 let demoViewRef: React.RefObject<DemoSimple<DemoProps, DemoState> | DemoHighlightRegion> = React.createRef()

src/sonification/Sonifier.ts

@@ -26,11 +26,11 @@ export class Sonifier {
     /**
      * Every sonifier has an audio context used to play sounds
      */
-    private _audioCtx!: AudioContext
-    public get audioCtx(): AudioContext {
-        return this._audioCtx
+    private static _audioCtx = new AudioContext();
+    public static get audioCtx(): AudioContext {
+        return Sonifier._audioCtx
     }
-    private gainNode: GainNode
+    public static gainNode: GainNode
 
     /**
      * Whether or not audio is currently playing
@@ -140,14 +140,13 @@ export class Sonifier {
      */
     private constructor() {
         // super()
-        this._audioCtx = new AudioContext() // works without needing additional libraries. need to check this when we move away from react as it is currently pulling from react-dom.d.ts.
-        this._audioCtx.resume()
+        Sonifier._audioCtx.resume()
         //this.startTime = this.audioCtx.currentTime
         // Always begin in a "stopped" state since there is no data to play yet at construction time
         this._playbackState = PlaybackState.Stopped
         this.sources = new Map()
         this._displays = new Map()
-        this.gainNode = this._audioCtx.createGain()
+        Sonifier.gainNode = Sonifier._audioCtx.createGain()
     }
 
     /**
@@ -168,7 +167,7 @@ export class Sonifier {
         // The answer is yes if we ever want to handl control to a new/different audio context
         // maybe have an option for "halt" instead that ends everything?
 
-        this.audioCtx.suspend()
+        Sonifier.audioCtx.suspend()
         if (DEBUG) console.log('stopping. playback state is paused')
         this._playbackState = PlaybackState.Paused
         // this.audioCtx.close() -- gives everything up, should only be done at the very very end.
@@ -178,43 +177,41 @@ export class Sonifier {
     public onPlay() {
         // @todo do I need to do anything differently if was stopped instead of paused?
         // The answer is yes if we ever want to handl control to a new/different audio context
-        if (this.playbackState == PlaybackState.Playing && this.audioCtx.state == 'running') {
+        if (this.playbackState == PlaybackState.Playing && Sonifier.audioCtx.state == 'running') {
             if (DEBUG) console.log('playing')
         } else {
             if (DEBUG) console.log('setting up for playing')
-            this.audioCtx.resume()
-            this.gainNode.connect(this._audioCtx.destination)
-            this.startSources()
+            Sonifier.audioCtx.resume()
+            Sonifier.gainNode.connect(Sonifier.audioCtx.destination)
+            // this.startSources()
             this._playbackState = PlaybackState.Playing
         }
     }
 
-    /**
-     * Triggers all existing audio nodes to play.
-     *
-     * @todo if a new source is added after onPlay it won't get connected
-     * @todo what about visually displaying things
-     */
-    public startSources() {
-        if (DEBUG) console.log(`starting sources ${this.sources.size}`)
-        this.sources.forEach((source: DataSource, key: number) => {
-            source.displays().map((display) => {
-                if (DEBUG) console.log(`Source: ${source} Display: ${display.toString()}`)
-                let sonify = display as Sonify
-                let audioNode = sonify.getAudioNode(this)
-                if (audioNode != undefined) {
-                    audioNode.connect(this.gainNode)
-                }
-            })
-        })
-        this.gainNode.connect(this._audioCtx.destination)
-    }
+
+    // /**
+    //  * Triggers all existing audio nodes to play.
+    //  * 
+    //  * @todo if a new source is added after onPlay it won't get connected 
+    //  * @todo what about visually displaying things
+    //  */
+    // public startSources() {
+    //     if (DEBUG) console.log(`starting sources ${this.sources.size}`)
+    //     this.sources.forEach((source: DataSource, key: number) => {
+    //         source.displays().map((display) => {
+    //             if (DEBUG) console.log(`Source: ${source} Display: ${display.toString()}`)
+    //             display.show();
+    //         })
+    //     })
+    //     Sonifier.gainNode.connect(Sonifier._audioCtx.destination)
+    // }
+
 
     //needs extensive testing.
     public onPause() {
         if (DEBUG) console.log('Pausing. Playback state is paused')
-        this.audioCtx.suspend()
-        this.gainNode.disconnect()
+        Sonifier.audioCtx.suspend()
+        Sonifier.gainNode.disconnect()
         this._playbackState = PlaybackState.Paused
     }
 
@@ -262,7 +259,7 @@ export class Sonifier {
         else {
             setTimeout(() => {
                 this.pushPoint(point, sourceId)
-            }, time - d3.now())
+            }, time - Sonifier.audioCtx.currentTime*1000);
         }
     }
 }

src/sonification/displays/DatumDisplay.ts

@@ -22,7 +22,6 @@ export class DatumDisplay {
      */
     update(datum: Datum) {
         this.datum = datum
-        console.log(datum)
     }
 
     /**

src/sonification/displays/NoiseSonify.ts

@@ -1,57 +1,63 @@
 import { Datum } from '../Datum'
 import { Sonifier } from '../Sonifier'
-import { Sonify } from './Sonify'
+import { SonifyFixedDuration } from './SonifyFixedDuration'
+
+const DEBUG = false;
+
 
 /**
  * Class for sonifying a data point as a pitch.
  * @extends Sonify
  * @todo only plays noise once. investigate further. probably have to create new noise nodes for each point.
  *
  */
-export class NoiseSonify extends Sonify {
+export class NoiseSonify extends SonifyFixedDuration {
+
     /**
-     * @todo what do we do when this is live data? unlike audio nodes, noise nodes need a duration to calculate the buffer length.
+     * This is the node that knows how to play noise. It is connected to the this.audioNode
      */
+    private filter: BiquadFilterNode | undefined;
 
-    private noiseBufferSize = 20
-    private duration = 0.1
-
-    public getAudioNode(sonifier?: Sonifier) {
-        console.log('NoiseSonify:getAudioNode')
-        if (super.getAudioNode()) return super.getAudioNode()
-        if (sonifier) {
-            console.log('executing code in getAudioNode')
-            let sampleRate = sonifier.audioCtx.sampleRate
-            this.noiseBufferSize = sampleRate * this.duration
-            let buffer = sonifier.audioCtx.createBuffer(1, this.noiseBufferSize, sonifier.audioCtx.sampleRate)
-            let noiseNode = sonifier.audioCtx.createBufferSource()
-            noiseNode.buffer = buffer
-            let bandPassFilterNode = sonifier.audioCtx.createBiquadFilter()
-            bandPassFilterNode.type = 'bandpass'
-            bandPassFilterNode.frequency.value = 440
-            noiseNode.connect(bandPassFilterNode)
-            this.setAudioNode(noiseNode)
-        }
-        return super.getAudioNode()
+    protected extend(timeAdd: number) {
+        let noiseNode = this.outputNode as AudioBufferSourceNode
+        if (DEBUG) console.log('noiseSonify, getting noise node', noiseNode)
+        if (noiseNode) noiseNode.buffer = this.fillBuffer(timeAdd);
     }
 
-    public update(datum: Datum, duration?: number) {
-        console.log('NoiseSonify: updateDatum')
-        super.update(datum)
-        if (duration) this.duration = duration
-        let noiseNode = this.getAudioNode() as AudioBufferSourceNode
-        console.log('noiseSonify, getting noise node', noiseNode)
-        let buffer = noiseNode.buffer
-        if (buffer) {
-            let bufferData = buffer.getChannelData(0)
-            console.log('filling in buffer data')
-            for (let i = 0; i < this.noiseBufferSize; i++) {
-                bufferData[i] = Math.random() * 2 - 1
-            }
-            noiseNode.start()
+    /** 
+     * The length of the buffer for making noise
+     */
+    private noiseBufferSize = 20
+    /**
+     * create a buffer and fill it
+     * @param time Time to fill it for in seconds
+     */
+    private fillBuffer(length: number): AudioBuffer {
+        let sampleRate = Sonifier.audioCtx.sampleRate
+        let noiseBufferSize = sampleRate * length;
+        let buffer = Sonifier.audioCtx.createBuffer(1, noiseBufferSize, Sonifier.audioCtx.sampleRate)
+        let bufferData = buffer.getChannelData(0)
+        console.log('filling in buffer data');
+        for (let i = 0; i < noiseBufferSize; i++) {
+            bufferData[i] = Math.random() * 2 - 1
         }
+        return buffer;
     }
+    
+    public create(datum: Datum): AudioScheduledSourceNode {
+        let outputNode = Sonifier.audioCtx.createBufferSource()
+        this.filter = Sonifier.audioCtx.createBiquadFilter()
+        this.filter.type = 'bandpass'
+        this.filter.frequency.value = 440
+        this.filter.connect(Sonifier.gainNode);
 
+        outputNode.buffer = this.fillBuffer(this.duration);
+        outputNode.connect(this.filter);
+        this.outputNode = outputNode;
+        outputNode.start();
+        return outputNode;
+    }
+   
     public toString(): string {
         return `NoiseSonify`
     }

src/sonification/displays/NoteSonify.ts

@@ -2,6 +2,8 @@ import { Datum } from '../Datum'
 import { Sonify } from './Sonify'
 import { Sonifier } from '../Sonifier'
 
+const DEBUG = false;
+
 /**
  * Class for sonifying a data point as a pitch.
  * @extends Sonify
@@ -12,27 +14,6 @@ import { Sonifier } from '../Sonifier'
  * [note implementation not complete. Needs to handle scheduleSound still]
  */
 export class NoteSonify extends Sonify {
-    /**
-     * The start and end frequency for the note to play
-     */
-    private frequency: number | undefined
-
-    /**
-     * Returns an audio node that contains an oscillator
-     *
-     * @param sonifier
-     * @returns audio node that contains an oscillator
-     */
-    public getAudioNode(sonifier?: Sonifier) {
-        console.log('get audio node called')
-        if (super.getAudioNode()) return super.getAudioNode()
-        if (sonifier) {
-            console.log('created audio node')
-            let oscillator = sonifier.audioCtx.createOscillator()
-            super.setAudioNode(oscillator)
-        }
-        return super.getAudioNode()
-    }
 
     /**
      * Stores relevant information when a new datum arrives
@@ -43,33 +24,33 @@ export class NoteSonify extends Sonify {
      */
     public update(datum: Datum, duration = 200, volume?: number, smooth?: boolean) {
         super.update(datum)
-        console.log(`updating value  ${this.frequency}`)
-        let oscillator = this.getAudioNode() as OscillatorNode
-        if (this.frequency == undefined) {
-            // first data point
-            oscillator.frequency.value = datum.adjustedValue
-            this.frequency = datum.adjustedValue
-            oscillator.start()
-        } else {
-            oscillator.frequency.value = datum.adjustedValue
-            this.frequency = datum.adjustedValue
-        }
+        if (DEBUG) console.log(`updating value  ${this.datum.adjustedValue}`)
+        let oscillator = this.outputNode as OscillatorNode
+        oscillator.frequency.value = datum.adjustedValue
     }
 
+
     /**
      * Generates a new note sonifier
-     * @param duration The length of time the sound should play for
      * @param volume The volume the sound should play at
      * @param optionally include an audio node that can be played
-     * @param smooth optionall specify if sounds should transition smoothly between data points
      * @returns Returns an instance of specific subclass of SonificationType.
      */
     public constructor(volume?: number, audioNode?: AudioScheduledSourceNode) {
-        super(volume, audioNode)
+        
+        super( volume, Sonifier.audioCtx.createOscillator())
+
+        let oscillator = this.outputNode as OscillatorNode;
+        if (oscillator == undefined) {
+            // oscillator = Sonifier.audioCtx.createOscillator()
+            this.outputNode = oscillator;
+        }
+        oscillator.start();
     }
 
     public toString(): string {
-        //let oscillator = this.getAudioNode() as OscillatorNode;
-        return `NoteSonify`
+        let oscillator = this.outputNode as OscillatorNode;
+        if (oscillator) return `NoteSonify playing ${oscillator.frequency.value}`
+        else return `NoteSonify not currently playing`
     }
 }