AudioIO.jl

Spencer Russell

MIT Media Lab

@ssfrr

2014-06-26

How I Met Julia

• press 's' to pull up speaker notes
• introduced to Julia by Angel Pizarro in June 2012 at Philly Lambda

Some History

• MUSIC written in 1957 by Max Mathews at Bell Labs
• MUSIC 2-4 also at Bell Labs
• CSound still in wide use today

"Our sound isn't hardcore, breakcore or speedcore - it's multi-core and fully hyper-threaded."

• But what do these systems have in common?

UGens

Unit Generators

• Typically written in a lower-level language
• Who builds these UGens??

• CSound

• C/C++

• Max/MSP, PD

• C/C++

• SuperCollider

• C++

• Clojure (Overtone)

• C++

• I'm sure you're seeing a pattern here

• Julia

• Julia

AudioIO.jl

• Contributions from Howard Mao on File I/O
• Joris Kraak for audio Input

Exported Functions

af_open

play

stop

• Try to think in terms of conceptual methods
• In python you need different function names or a chain of "isinstance"
• don't add more methods than you have to

arr = my_synth_alg()::Array{Float32}
play(arr)

• Keeping this sort of use case simple is a big driving factor in AudioIO
• This is playing an array in native sample format
• What about playing an array in a different format?

arr = my_synth_alg()::Array{Int16}
play(arr)

• what is this wizardry?
• the magic of multiple dispatch!
• let's see how playing a Signed Int array is implemented

function play{T <: Signed}(arr::Array{T}, args...)
    arr = arr / typemax(T)
    play(arr, args...)
end

• we're just converting to a float array and playing that
• what happens when we play a float array?

function play(arr::AudioBuf, args...)
    player = ArrayPlayer(arr)
    play(player, args...)
end

• here we start to see some of the meat of AudioIO
• we create AudioNodes and play them
• how to audio nodes get played?

function play(node::AudioNode)
    global _stream
    if _stream == nothing
        _stream = PortAudioStream()
    end
    play(node, _stream)
end

• a stream is just a render tree and background task

function play(node::AudioNode, stream::AudioStream)
    push!(stream.root, node)
    return node
end

• here we get to the root of the play function
• we just add it to the render tree and return
• at every render block, N frames will be pulled from the AudioNode
• What other AudioNodes are there?

play(WhiteNoise())

• what is an AudioNode?

type AudioNode{T<:AudioRenderer}
    active::Bool
    end_cond::Condition
    renderer::T
    AudioNode(renderer::AudioRenderer) =
            new(true, Condition(), renderer)
    AudioNode(args...) = AudioNode{T}(T(args...))
end

• this is not the first iteration of this type
• I started with AudioNode being an abstract type
• but then I needed fields!

type AudioNode{T<:AudioRenderer}

    active::Bool
    end_cond::Condition

    renderer::T
    AudioNode(renderer::AudioRenderer) =
            new(true, Condition(), renderer)
    AudioNode(args...) = AudioNode{T}(T(args...))
end

type AudioNode{T<:AudioRenderer}
    active::Bool
    end_cond::Condition

    renderer::T

    AudioNode(renderer::AudioRenderer) =
            new(true, Condition(), renderer)
    AudioNode(args...) = AudioNode{T}(T(args...))
end

type AudioNode{T<:AudioRenderer}
    active::Bool
    end_cond::Condition
    renderer::T

    AudioNode(renderer::AudioRenderer) =
            new(true, Condition(), renderer)

    AudioNode(args...) = AudioNode{T}(T(args...))
end

type AudioNode{T<:AudioRenderer}
    active::Bool
    end_cond::Condition
    renderer::T
    AudioNode(renderer::AudioRenderer) =
            new(true, Condition(), renderer)

    AudioNode(args...) = AudioNode{T}(T(args...))

end

type MixRenderer <: AudioRenderer
    inputs::Vector{AudioNode}
    buf::AudioBuf

    MixRenderer(inputs) = new(inputs, AudioSample[])
    MixRenderer() = MixRenderer(AudioNode[])
end

typealias AudioMixer AudioNode{MixRenderer}
export AudioMixer

function render(node::MixRenderer,
                device_input::AudioBuf,
                info::DeviceInfo)
...

type MixRenderer <: AudioRenderer

    inputs::Vector{AudioNode}
    buf::AudioBuf

    _
    MixRenderer(inputs) = new(inputs, AudioSample[])
    MixRenderer() = MixRenderer(AudioNode[])
end

typealias AudioMixer AudioNode{MixRenderer}
export AudioMixer

function render(node::MixRenderer,
                device_input::AudioBuf,
                info::DeviceInfo)
...

• I'm considering having the render function take the output buffer as an arg
• then it would need to return the number of samples copied

type MixRenderer <: AudioRenderer
    inputs::Vector{AudioNode}
    buf::AudioBuf
    _

    MixRenderer(inputs) = new(inputs, AudioSample[])
    MixRenderer() = MixRenderer(AudioNode[])

end

typealias AudioMixer AudioNode{MixRenderer}
export AudioMixer

function render(node::MixRenderer,
                device_input::AudioBuf,
                info::DeviceInfo)
...

type MixRenderer <: AudioRenderer
    inputs::Vector{AudioNode}
    buf::AudioBuf

    MixRenderer(inputs) = new(inputs, AudioSample[])
    MixRenderer() = MixRenderer(AudioNode[])
end
_

typealias AudioMixer AudioNode{MixRenderer}
export AudioMixer

_
function render(node::MixRenderer,
                device_input::AudioBuf,
                info::DeviceInfo)
...

type MixRenderer <: AudioRenderer
    inputs::Vector{AudioNode}
    buf::AudioBuf

    MixRenderer(inputs) = new(inputs, AudioSample[])
    MixRenderer() = MixRenderer(AudioNode[])
end

typealias AudioMixer AudioNode{MixRenderer}
export AudioMixer
_

function render(node::MixRenderer,
                device_input::AudioBuf,
                info::DeviceInfo)
...

type SinOscRenderer{
        T<:Union(Float32, AudioNode)} <: AudioRenderer
    freq::T
    phase::Float32
    buf::AudioBuf
end

function render(node::SinOscRenderer{Float32},
        device_input::AudioBuf,
        info::DeviceInfo)

function render(node::SinOscRenderer{AudioNode},
        device_input::AudioBuf,
        info::DeviceInfo)

Challenges

• garbage Collection
• incgc branch from Oscar Blumberg is awesome! Can't wait for 0.4
• Global variables from the REPL are problematic

Demo

Future Work