phetsims / wave-interference

"Wave Interference" is an educational simulation in HTML5, by PhET Interactive Simulations.
MIT License
19 stars 5 forks source link

Sonification for the chart as a function of position #457

Open samreid opened 5 years ago

samreid commented 5 years ago

We would like to depict the sound from the position-based chart. Many notes are in the design doc.

samreid commented 5 years ago

Here's a patch from feasibilty prototype from today's discussion:

```diff Index: wave-interference/js/common/view/WaveAreaGraphNode.js IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- wave-interference/js/common/view/WaveAreaGraphNode.js (revision e0ddf9308b01444989aef2c543915f038a982f51) +++ wave-interference/js/common/view/WaveAreaGraphNode.js (date 1571779449268) @@ -9,19 +9,48 @@ 'use strict'; // modules + const BooleanProperty = require( 'AXON/BooleanProperty' ); const Bounds2 = require( 'DOT/Bounds2' ); + const ContinuousPropertySoundGenerator = require( 'TAMBO/sound-generators/ContinuousPropertySoundGenerator' ); const DashedLineNode = require( 'WAVE_INTERFERENCE/common/view/DashedLineNode' ); const Line = require( 'SCENERY/nodes/Line' ); const Node = require( 'SCENERY/nodes/Node' ); const Path = require( 'SCENERY/nodes/Path' ); + const Property = require( 'AXON/Property' ); + const Range = require( 'DOT/Range' ); const SceneToggleNode = require( 'WAVE_INTERFERENCE/common/view/SceneToggleNode' ); const Shape = require( 'KITE/Shape' ); + const soundManager = require( 'TAMBO/soundManager' ); const Util = require( 'DOT/Util' ); + const Vector2 = require( 'DOT/Vector2' ); const waveInterference = require( 'WAVE_INTERFERENCE/waveInterference' ); const WaveInterferenceConstants = require( 'WAVE_INTERFERENCE/common/WaveInterferenceConstants' ); const WaveInterferenceText = require( 'WAVE_INTERFERENCE/common/view/WaveInterferenceText' ); const WaveInterferenceUtils = require( 'WAVE_INTERFERENCE/common/WaveInterferenceUtils' ); + // sounds + // const sineSound = require( 'sound!TAMBO/ethereal-flute-for-meter-loop.mp3' ); + + // sounds + const etherealFluteSound = require( 'sound!WAVE_INTERFERENCE/ethereal-flute-for-meter-loop.mp3' ); + // const organ2Sound = require( 'sound!WAVE_INTERFERENCE/organ-v2-for-meter-loop.mp3' ); + // const organSound = require( 'sound!WAVE_INTERFERENCE/organ-for-meter-loop.mp3' ); + // const sineSound = require( 'sound!TAMBO/220hz-saturated-sine-loop.mp3' ); + // const sineSound2 = require( 'sound!WAVE_INTERFERENCE/220hz-saturated-sine-playback-rate-75.mp3' ); + // const stringSound1 = require( 'sound!TAMBO/strings-loop-middle-c-oscilloscope.mp3' ); + // const windSound1 = require( 'sound!TAMBO/winds-loop-middle-c-oscilloscope.mp3' ); + // const windSound2 = require( 'sound!TAMBO/winds-loop-c3-oscilloscope.mp3' ); + // const windyTone4 = require( 'sound!WAVE_INTERFERENCE/windy-tone-for-meter-loop-rate-75-pitch-matched-fixed.mp3' ); + // const windyToneSound = require( 'sound!WAVE_INTERFERENCE/windy-tone-for-meter-loop.mp3' ); + + // const sounds = [ sineSound2, windyTone4, stringSound1, sineSound, windSound1, windSound2, etherealFluteSound, organ2Sound, organSound, windyToneSound ]; + // const selectedSound=windyTone4; + // const selectedSound=stringSound1; + // const selectedSound=windSound1; + // const selectedSound=etherealFluteSound;// several votes! :) EM says it is a bit harsh, + // const selectedSound=organSound; + const selectedSound = etherealFluteSound; + // constants const TEXT_MARGIN_X = 8; const TEXT_MARGIN_Y = 6; @@ -200,16 +229,76 @@ } ); this.addChild( path ); + const derivativePath = new Path( new Shape(), { + stroke: 'red', + lineWidth: 2, + lineJoin: WaveInterferenceConstants.CHART_LINE_JOIN, // Prevents artifacts at the wave source + + // prevent the shape from going outside of the chart area + clipArea: pathClipArea, + + // prevent bounds computations during main loop + boundsMethod: 'none', + localBounds: pathClipArea.bounds + } ); + this.addChild( derivativePath ); + // Created once and reused to avoid allocations const sampleArray = []; + const soundPropertyGenerators = []; + for ( let i = 0; i < 20; i++ ) { + const p = new Property( 0 ); + const continuousPropertySoundGenerator = new ContinuousPropertySoundGenerator( p, selectedSound, new Range( 100, 500 ), new BooleanProperty( false ) ); + soundManager.addSoundGenerator( continuousPropertySoundGenerator ); + soundPropertyGenerators.push( continuousPropertySoundGenerator ); + } + // Manually tuned to center the line in the graph, dy must be synchronized with graphHeight const dx = -options.x; const dy = -options.centerY / 2 + 7.5; const updateShape = () => { const shape = getWaterSideShape( sampleArray, model.sceneProperty.value.lattice, waveAreaBounds, dx, dy ); - return path.setShape( shape ); + + let selectedIndex = 0; + const makeDerivative = s => { + const points = s.subpaths[ 0 ].points; + const derivative = new Shape(); + for ( let i = 0; i < points.length - 2; i++ ) { + const a = points[ i ]; + const b = points[ i + 1 ]; + const c = points[ i + 2 ]; + + let y = 100; + const x = ( a.x + b.x + c.x ) / 3; + if ( b.y < a.y && b.y < c.y && x > 50 ) { + y = 200; + assert && assert( !isNaN( x ) ); + const soundPropertyGenerator = soundPropertyGenerators[ selectedIndex ]; + + // console.log( x ); + soundPropertyGenerator.setOutputLevel( Util.linear( 1000, 100, 0, 1, x ), 0 ); + soundPropertyGenerator.property.value = x; + selectedIndex++; + } + + const p = new Vector2( x, y ); + derivative.lineToPoint( p ); + // console.log( x ); + + } + return derivative; + }; + + const derivative = makeDerivative( shape ); + + path.setShape( shape ); + derivativePath.setShape( derivative ); + + for ( let i = selectedIndex; i < soundPropertyGenerators.length; i++ ) { + soundPropertyGenerators[ i ].setOutputLevel( 0, 0 ); + } }; model.scenes.forEach( scene => scene.lattice.changedEmitter.addListener( updateShape ) ); Index: tambo/js/sound-generators/ContinuousPropertySoundGenerator.js IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- tambo/js/sound-generators/ContinuousPropertySoundGenerator.js (revision 340d72cec98a94e6959c96a7911193738bffc00a) +++ tambo/js/sound-generators/ContinuousPropertySoundGenerator.js (date 1571766568917) @@ -33,7 +33,7 @@ initialOutputLevel: 0.7, loop: true, trimSilence: false, - pitchRangeInSemitones: 36, + pitchRangeInSemitones: 18, pitchCenterOffset: 2, fadeStartDelay: 0.2, // in seconds, time to wait before starting fade fadeTime: 0.15, // in seconds, duration of fade out @@ -48,6 +48,8 @@ super( sound, options ); + this.property = property; + // @private {number} - see docs at options declaration this.fadeTime = options.fadeTime; @@ -61,21 +63,26 @@ this.remainingFadeTime = 0; // start with the output level at zero so that the initial sound generation has a bit of fade in - this.setOutputLevel( 0, 0 ); + // this.setOutputLevel( 0, 0 ); // function for starting the sound or adjusting the volume const listener = value => { if ( !resetInProgressProperty.value ) { + const r = { min: range.max, max: range.min }; // calculate the playback rate based on the value - const normalizedValue = Math.log( value / range.min ) / Math.log( range.max / range.min ); + const normalizedValue = Math.log( value / r.min ) / Math.log( r.max / r.min ); const centerValue = normalizedValue - 0.5; const midiNote = options.pitchRangeInSemitones / 2 * centerValue + options.pitchCenterOffset; const playbackRate = Math.pow( 2, midiNote / 12 ); - this.setPlaybackRate( playbackRate ); - this.setOutputLevel( this.nonFadedOutputLevel ); + // if ( playbackRate < 4 ) { + this.setPlaybackRate( playbackRate * 0.75, 1E-4 ); + // } + // this.setOutputLevel( this.nonFadedOutputLevel ); + + // console.log(playbackRate,this.nonFadedOutputLevel); if ( !this.playing ) { this.play(); } @@ -112,7 +119,7 @@ // the sound is fading out, adjust the output level const outputLevel = Math.max( ( this.remainingFadeTime - this.delayBeforeStop ) / this.fadeTime, 0 ); - this.setOutputLevel( outputLevel * this.nonFadedOutputLevel ); + // this.setOutputLevel( outputLevel * this.nonFadedOutputLevel ); } // fade out complete, stop playback ```
samreid commented 5 years ago

I cleaned up the prototype, softened the initial sound, improved the volume envelope and committed. I wouldn't call it polished, but at least it is ready for design discussion.

samreid commented 5 years ago

Here's a patch for playing discrete notes:

```diff Index: js/common/view/WaveAreaGraphNode.js IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- js/common/view/WaveAreaGraphNode.js (revision c42b2daa191896872e23df6c6ef87b4eaa06804e) +++ js/common/view/WaveAreaGraphNode.js (date 1573103660650) @@ -11,14 +11,15 @@ // modules const BooleanProperty = require( 'AXON/BooleanProperty' ); const Bounds2 = require( 'DOT/Bounds2' ); - const PeakToneGenerator = require( 'WAVE_INTERFERENCE/common/view/PeakToneGenerator' ); const DashedLineNode = require( 'WAVE_INTERFERENCE/common/view/DashedLineNode' ); const Line = require( 'SCENERY/nodes/Line' ); const Node = require( 'SCENERY/nodes/Node' ); const Path = require( 'SCENERY/nodes/Path' ); + const PeakToneGenerator = require( 'WAVE_INTERFERENCE/common/view/PeakToneGenerator' ); const Property = require( 'AXON/Property' ); const SceneToggleNode = require( 'WAVE_INTERFERENCE/common/view/SceneToggleNode' ); const Shape = require( 'KITE/Shape' ); + const SoundClip = require( 'TAMBO/sound-generators/SoundClip' ); const soundManager = require( 'TAMBO/soundManager' ); const Util = require( 'DOT/Util' ); const Vector2 = require( 'DOT/Vector2' ); @@ -31,7 +32,6 @@ // const sineSound = require( 'sound!TAMBO/ethereal-flute-for-meter-loop.mp3' ); // sounds - const etherealFluteSound = require( 'sound!WAVE_INTERFERENCE/ethereal-flute-for-meter-loop.mp3' ); // const organ2Sound = require( 'sound!WAVE_INTERFERENCE/organ-v2-for-meter-loop.mp3' ); // const organSound = require( 'sound!WAVE_INTERFERENCE/organ-for-meter-loop.mp3' ); // const sineSound = require( 'sound!TAMBO/220hz-saturated-sine-loop.mp3' ); @@ -41,6 +41,7 @@ // const windSound2 = require( 'sound!TAMBO/winds-loop-c3-oscilloscope.mp3' ); // const windyTone4 = require( 'sound!WAVE_INTERFERENCE/windy-tone-for-meter-loop-rate-75-pitch-matched-fixed.mp3' ); // const windyToneSound = require( 'sound!WAVE_INTERFERENCE/windy-tone-for-meter-loop.mp3' ); + const etherealFluteSound = require( 'sound!WAVE_INTERFERENCE/ethereal-flute-for-meter-loop.mp3' ); // const sounds = [ sineSound2, windyTone4, stringSound1, sineSound, windSound1, windSound2, etherealFluteSound, organ2Sound, organSound, windyToneSound ]; // const selectedSound=windyTone4; @@ -48,7 +49,18 @@ // const selectedSound=windSound1; // const selectedSound=etherealFluteSound;// several votes! :) EM says it is a bit harsh, // const selectedSound=organSound; - const selectedSound = etherealFluteSound; + const C3 = require( 'sound!WAVE_INTERFERENCE/bright_acoustic_piano-mp3/C3.mp3' ); + const G3 = require( 'sound!WAVE_INTERFERENCE/bright_acoustic_piano-mp3/G3.mp3' ); + const E3 = require( 'sound!WAVE_INTERFERENCE/bright_acoustic_piano-mp3/E3.mp3' ); + const C4 = require( 'sound!WAVE_INTERFERENCE/bright_acoustic_piano-mp3/C4.mp3' ); + const C5 = require( 'sound!WAVE_INTERFERENCE/bright_acoustic_piano-mp3/C5.mp3' ); + const selectedSound = C4; + + const C5clip = new SoundClip( C5 ); + const C4clip = new SoundClip( C4 ); + const C3clip = new SoundClip( C3 ); + const G3clip = new SoundClip( G3 ); + const E3clip = new SoundClip( E3 ); // constants const TEXT_MARGIN_X = 8; @@ -277,21 +289,39 @@ if ( b.y < a.y && b.y < c.y && x > 30 ) { y = 200; assert && assert( !isNaN( x ) ); - const soundPropertyGenerator = peakToneGenerators[ selectedIndex ]; + // const soundPropertyGenerator = peakToneGenerators[ selectedIndex ]; + // + // // const outputLevel = Util.linear( 30, 500, 0.2, -0.1, x ); + // // https://saylordotorg.github.io/text_intermediate-algebra/s10-03-logarithmic-functions-and-thei.html + // // fast exponential decay + // const outputLevel = 0.2 * Math.exp( -0.01 * x ); // larger coefficient means faster decay + // soundPropertyGenerator.setOutputLevel( outputLevel, 1 ); + // + // + // // Pixel coordinate maps to playback rate. + // const linear = Util.linear( 30, 500, 1.2, 0.2, x ); + // // const choices = [ 0.2, 0.4, 0.6, 0.8, 1.0, 1.2 ]; + // // _.minBy(choices,); + // soundPropertyGenerator.property.value = linear; + // selectedIndex++; - // const outputLevel = Util.linear( 30, 500, 0.2, -0.1, x ); - // https://saylordotorg.github.io/text_intermediate-algebra/s10-03-logarithmic-functions-and-thei.html - // fast exponential decay - const outputLevel = 0.2 * Math.exp( -0.01 * x ); // larger coefficient means faster decay - soundPropertyGenerator.setOutputLevel( outputLevel, 1 ); + [ C4clip, C5clip, C3clip, G3clip, E3clip ].forEach( clip => { + !soundManager.hasSoundGenerator( clip ) && soundManager.addSoundGenerator( clip ); + clip.setOutputLevel( 0.2 ); + } ); - - // Pixel coordinate maps to playback rate. - const linear = Util.linear( 30, 500, 1.2, 0.2, x ); - // const choices = [ 0.2, 0.4, 0.6, 0.8, 1.0, 1.2 ]; - // _.minBy(choices,); - soundPropertyGenerator.property.value = linear; - selectedIndex++; + if ( x > 100 && x < 105 ) { + C5clip.play(); + } + if ( x > 200 && x < 205 ) { + G3clip.play(); + } + if ( x > 300 && x < 305 ) { + E3clip.play(); + } + if ( x > 400 && x < 405 ) { + C3clip.play(); + } } const p = new Vector2( x, y ); ```
samreid commented 4 years ago

Based on discussion with @Ashton-Morris, I opened issue https://github.com/phetsims/tambo/issues/84 and will look into that for this issue.

samreid commented 4 years ago

We decided we will not include sonification of this graph for the next publication, and @Ashton-Morris wants to review this issue for discussion with Wanda.

Ashton-Morris commented 4 years ago

@samreid I have a saved a handful of links to your previous dev versions for eventual review with Wanda. Assuming they stay open to my access I have them bookmarked.

samreid commented 4 years ago

Tagging as deferred since this will not be in the next publication. @Ashton-Morris if you were using this issue to track discussion with Wanda, can it be moved to a separate issue?

Ashton-Morris commented 4 years ago

We can close this, I have collected the relevant sims and info and have saved it.

samreid commented 4 years ago

Leaving open since we may include this in a future release.