Building your own web synthesizer

jan.krutisch.de

for codebits 2012

/halfbyte

Agenda

History
Theory (a little)
Lotsa Demos
Outlook
Q & A
Bonus round: More of my stuff

History

Java Applet

Flash

Firefox

Audio Data API

+----------+
| <audio/> |
+-----+----+
      v
+----------+
|JavaScript|
+-----+----+
      v
+----------+
| <audio/> |
+----------+

I maded you some data

							
// ...

  var audio = new Audio();   audio.mozSetup(1, sampleRate); 
// ...

  written = audio.mozWriteAudio(tail);
// ...

Webkit

Web Audio API

Graphs!

Callbacks!

Effects!

Standards!

www.w3.org/TR/webaudio/

Web Audio API

Support

Chrome (also on Android 4)
Safari 6 (also iOS6 webkit)

Digital Audio Basics

Fourier

Nyquist

Sampling Theorem

Nyquist

Synthesis basics

Make Some Noise

Additive Synthesis

FM Synthesis

Subtractive Synthesis

Frequencies

Notes

Standard pitch: A

440 Hz

A

110,220,440,880,1760,...

(= Octave)

In Between

C-C#-D-D#-E-F-F#-G-G#-A-A#-B

(= Octave)

In Between

It's complicated

just, pythagorean, meantone, well, equal, syntonic

slendro, pelog

alpha, beta, gamma, delta

You Name It

Code!

AudioContext

Setup

							
  var ctx = new AudioContext();

Setup

							
  var ctx = new AudioContext();

Setup

							
  var ctx = new webkitAudioContext();

Setup

							
  var ctx = new mozAudioContext();

Setup

							
  var ctx = new mozAudioContext();

Properties

Reference time

(in seconds!)

							
  ctx.currentTime

Properties

Physical in/output

AudioDestinationNode

							
  ctx.destination

Properties

Sample Rate

(in Hz)

							
  ctx.sampleRate

OscillatorNode

Beep

	+-------+
	|  OSC  |
	+---+---+
	    v
	+-------+
	|  OUT  +
	+-------+

	            
	var osc = ctx.createOscillator();
	osc.connect(ctx.destination);
	osc.noteOn(0);
	osc.noteOff(ctx.currentTime + 0.2);

Square

	+-------+
	|  OSC  |
	+---+---+
	    v
	+-------+
	|  OUT  +
	+-------+

	            
	var osc = ctx.createOscillator();
	osc.type = osc.SQUARE;
	osc.connect(ctx.destination);
	osc.noteOn(0);
	osc.noteOff(ctx.currentTime + 1.0);

Sawtooth

	+-------+
	|  OSC  |
	+---+---+
	    v
	+-------+
	|  OUT  +
	+-------+

	            
	var osc = ctx.createOscillator();
	osc.type = osc.SAWTOOTH;
	osc.connect(ctx.destination);
	osc.noteOn(0);
	osc.noteOff(ctx.currentTime + 1.0);

Frequency

  +-------+
  |  OSC  |
  +---+---+
      v
  +-------+
  |  OUT  +
  +-------+

              
  var osc = ctx.createOscillator();
  osc.type = osc.SINE;
  osc.frequency.value = 880;
  osc.connect(ctx.destination);
  osc.noteOn(0);
  osc.noteOff(ctx.currentTime + 1.0);

Yo Mama so fat

Detune

  +-----+ +-----+
  | OSC | | OSC |
  +--+--+ +-----+
     +---+---+
         v
     +-------+
     |  OUT  +
     +-------+

              
var osc1 = ctx.createOscillator();
var osc2 = ctx.createOscillator();
osc1.type = osc1.SAWTOOTH;
osc2.type = osc2.SAWTOOTH;
osc1.detune.value = -20;
osc2.detune.value = 20;

Sample Playback

	+-------+
	|  ABS  |
	+---+---+
	    v
	+-------+
	|  OUT  +
	+-------+

	            
	var osc = ctx.createBufferSource();
	osc.buffer = AudioBuffer() // ...not actual code
	osc.connect(ctx.destination);
	osc.noteOn(0);
	osc.noteOff(ctx.currentTime + 0.2);

AudioBuffer

	            
var loadBuffer = function (context, url, callback) {
  var request = new XMLHttpRequest();
  request.open("GET", url, true);
  request.responseType = "arraybuffer";

  var onDecode = function(buffer) {
    console.log("decoded");
    callback(buffer);
  };
  var onErr = function(err) {
    console.log("error decoding", url)
  }
  request.onload = function() {
    context.decodeAudioData(request.response, onDecode, onErr);
  }
  request.send();
};

Grain Playback

	+-------+
	|  ABS  |
	+---+---+
	    v
	+-------+
	|  OUT  +
	+-------+

	            
	var osc = ctx.createBufferSource();
	osc.buffer = AudioBuffer() // ...not actual code
	osc.connect(ctx.destination);
	osc.noteGrainOn(0, 0.43, 0.23);
	osc.noteOff(ctx.currentTime + 1.0);

playbackRate

	+-------+
	|  ABS  |
	+---+---+
	    v
	+-------+
	|  OUT  +
	+-------+

	            
	var osc = ctx.createBufferSource();
	osc.buffer = AudioBuffer() // ...not actual code
	osc.connect(ctx.destination);
	osc.playbackRate.value = 1.5;
	osc.noteGrainOn(0, 0.43, 0.23);
	osc.noteOff(ctx.currentTime + 1.0);

JS Node

  +-------+
  |  JSN  |
  +---+---+
      v
  +-------+
  |  OUT  +
  +-------+

              
var noise = new NoiseGenNode(ctx);
noise.connect(ctx.destination)
noise.noteOn(0);
noise.noteOff(ctx.currentTime + 0.2);

NoiseGenNode

              
var NoiseGenNode = function(ctx) {
  var that = {};
  that.connect = function(dest) {
    that.destination = dest;
  }
  var jsNode = ctx.createJavaScriptNode(2048, 1, 1);
  var start = function() {
    var i,l;
    jsNode.onaudioprocess = function(ev) {
      var data = ev.outputBuffer.getChannelData(0);
      for(i=0,l=data.length;i<l;i++) {
        data[i] = (2.0 * Math.random()) - 1.0;
      }
    }
    jsNode.connect(that.destination);
  }
  var stop = function() {
    jsNode.disconnect(that.destination);
    jsNode.onaudioprocess = null;
  }
  that.noteOn = function(time) {
    if (time <= ctx.currentTime) {
      start();
    } else {
      setTimeout(function() {
        that.noteOn(time);
      }, 1);
    }
  }
  that.noteOff = function(time) {
    if (time <= ctx.currentTime) {
      stop();
    } else {
      setTimeout(function() {
        that.noteOff(time);
      }, 1);
    }
  }
  return that;
};

NoiseGenNode

              
var NoiseGenNode = function(ctx) {
  var that = {};
  [...]
  return that;
};

NoiseGenNode

              
  var that = {};
  that.connect = function(dest) {
    that.destination = dest;
  }
  var jsNode = ctx.createJavaScriptNode(2048, 1, 1);
  var start = function() {
    var i,l;
    jsNode.onaudioprocess = function(ev) {
      var data = ev.outputBuffer.getChannelData(0);
      for(i=0,l=data.length;i<l;i++) {
        data[i] = (2.0 * Math.random()) - 1.0;
      }
    }
    jsNode.connect(that.destination);
  }
  var stop = function() {
    jsNode.disconnect(that.destination);
    jsNode.onaudioprocess = null;
  }

NoiseGenNode

              
  that.noteOn = function(time) {
    if (time <= ctx.currentTime) {
      start();
    } else {
      setTimeout(function() {
        that.noteOn(time);
      }, 1);
    }
  }
  that.noteOff = function(time) {
    if (time <= ctx.currentTime) {
      stop();
    } else {
      setTimeout(function() {
        that.noteOff(time);
      }, 1);
    }
  }

GainNode

	+-------+
	|  OSC  |
	+---+---+
	    v
	+-------+
	| Gain  |
	+---+---+
	    v
	+-------+
	|  OUT  +
	+-------+

	            
  var osc = ctx.createOscillator();
  osc.type = osc.SQUARE;
  var gain = ctx.createGainNode();
  gain.gain.value = 0.3;
  osc.connect(gain);
  gain.connect(ctx.destination)

GainNode

	+-------+
	|  OSC  |
	+---+---+
	    v
	+-------+
	| Gain  |
	+---+---+
	    v
	+-------+
	|  OUT  +
	+-------+

	            
  gain.gain.setValueAtTime(0.0, ctx.currentTime, 0.0);
  gain.gain.linearRampToValueAtTime(1.0, ctx.currentTime + 1.0);

AudioParam

.value=
.setValueAtTime(value, time)
.linearRampToValueAtTime(value, time)
.exponentialRampToValueAtTime(value, time)
.setTargetValueAtTime(targetValue, time, timeConstant)
.setValueCurveAtTime(values, time, duration)
.cancelScheduledValues(startTime)

BiquadFilterNode

LOWPASS

	+-------+
	|  OSC  |
	+---+---+
	    v
	+-------+
	|Filter |
	+---+---+
	    v
	+-------+
	|  OUT  +
	+-------+

	            
var filter = ctx.createBiquadFilter();
filter.type = filter.LOWPASS;
filter.frequency.setValueAtTime(20000, ctx.currentTime);
filter.frequency.linearRampToValueAtTime(200, ctx.currentTime + 1.0);
filter.Q.value = 15;
osc.connect(filter);

Highpass

	+-------+
	|  OSC  |
	+---+---+
	    v
	+-------+
	|Filter |
	+---+---+
	    v
	+-------+
	|  OUT  +
	+-------+

	            
var filter = ctx.createBiquadFilter();
filter.frequency.setValueAtTime(20000, ctx.currentTime);
filter.frequency.linearRampToValueAtTime(200, ctx.currentTime + 1.0);
filter.type = filter.HIGHPASS;
filter.Q.value = 15;
osc.connect(filter);

Bandpass

	+-------+
	|  OSC  |
	+---+---+
	    v
	+-------+
	|Filter |
	+---+---+
	    v
	+-------+
	|  OUT  +
	+-------+

	            
var filter = ctx.createBiquadFilter();
filter.frequency.setValueAtTime(20000, ctx.currentTime);
filter.frequency.linearRampToValueAtTime(200, ctx.currentTime + 1.0);
filter.type = filter.BANDPASS;
filter.Q.value = 15;
osc.connect(filter);

Q

	+-------+
	|  OSC  |
	+---+---+
	    v
	+-------+
	|Filter |
	+---+---+
	    v
	+-------+
	|  OUT  +
	+-------+

	            
var filter = ctx.createBiquadFilter();
filter.frequency.value = 5000;
filter.Q.setValueAtTime(0, ctx.currentTime);
filter.Q.linearRampToValueAtTime(20, ctx.currentTime + 1.0);
filter.type = filter.LOWPASS;
osc.connect(filter);

Effects

Convolution

	+-------+
	|  OSC  |
	+---+---+
	    v
	+-------+
	| CONV  |
	+---+---+
	    v
	+-------+
	|  OUT  +
	+-------+

	            
var osc = ctx.createOscillator();
osc.type = osc.SQUARE;
var conv = ctx.createConvolver();
conv.buffer = loadedSamples['t600']
osc.connect(conv);
conv.connect(ctx.destination)

Convolution

  +-------+
+-+  OSC  |
| +---+---+
|     v
| +-------+ +------+
| | CONV  +>| GAIN |
| +-------+ +------+
|               |
| +-------+     |
+>|  OUT  +<----+
  +-------+

	            
var osc = ctx.createOscillator();
osc.type = osc.SQUARE;
var gain = ctx.createGainNode();
gain.gain.value = 0.5;
var conv = ctx.createConvolver();
conv.buffer = loadedSamples['t600']
osc.connect(conv);
conv.connect(gain);
osc.connect(ctx.destination)
gain.connect(ctx.destination)

Delay

  +-------+
+-+  OSC  |
| +---+---+
|     v
| +-------+ +------+
| | DELAY +>| GAIN |
| +-------+ +------+
|               |
| +-------+     |
+>|  OUT  +<----+
  +-------+

	            
var osc = ctx.createOscillator();
osc.type = osc.SQUARE;
var gain = ctx.createGainNode();
gain.gain.value = 0.5;
var conv = ctx.createConvolver();
conv.buffer = loadedSamples['t600']
osc.connect(conv);
conv.connect(gain);
osc.connect(ctx.destination)
gain.connect(ctx.destination)

Delay

  +-------+   +--------+
+-+  OSC  |   |  GAIN  |
| +---+---+   +--------+
|     |          |   ^
|     +----------+   |
|     v              |
| +-------+   +------+-+
| | DELAY +-->| FILTER |
| +---+---+   +--------+
|     v
| +-------+
| | GAIN  +
| +---+---+
|     v
| +-------+
+>|  OUT  +
  +-------+

	            
	            	// too much code!

Waveshaper

  +-------+
  |  OSC  |
  +---+---+
      v
  +-------+
  |SHAPER |
  +---+---+
      v
  +-------+
  |  OUT  |
  +-------+

              
var shaper = ctx.createWaveShaper();
shaper.curve = new Float32Array([-1, -1, -0.5, 0,0.5, 1, 1]);
osc.connect(shaper);

Waveshaper

  +-------+
  |  ABS  |
  +---+---+
      v
  +-------+
  |SHAPER |
  +---+---+
      v
  +-------+
  |  OUT  |
  +-------+

              
var shaper = ctx.createWaveShaper();
shaper.curve = new Float32Array([-1, -1, -0.5, 0,0.5, 1, 1]);
sample.connect(shaper);

Panner

  +-------+
  |  ABS  |
  +---+---+
      v
  +-------+
  |  PAN  |
  +---+---+
      v
  +-------+
  |  OUT  |
  +-------+

              
                var pan1 = ctx.createPanner();
                var pan2 = ctx.createPanner();
                pan1.setPosition(-5, 0,0);
                pan2.setPosition(-5, 0,0);

Putting it Together

AwesomeSound

  +-------+
  | STUFF |
  +---+---+
      v
  +-------+
  |  OUT  |
  +-------+

              
var osc1 = ctx.createOscillator();
var osc2 = ctx.createOscillator();
var filter = ctx.createBiquadFilter();
var gain = ctx.createGainNode();
var fxGain = ctx.createGainNode();
var shaperGain = ctx.createGainNode();
var shaper = ctx.createWaveShaper();
var conv = ctx.createConvolver();