Source: music21/audioSearch.js

/**
 * audioSearch module. See {@link music21.audioSearch} namespace
 *
 * @exports music21/audioSearch
 * @namespace music21.audioSearch
 * @memberof music21
 * @requires music21/pitch
 * @requires music21/common
 */
import * as MIDI from 'midicube';
import * as common from './common.js';

// functions based on the prototype created by Chris Wilson's MIT License version
// and on Jordi Bartolome Guillen's audioSearch module for music21

// TODO(msc): Rewrite as a class -- config is just a class in disguise

// noinspection JSUnresolvedVariable
/**
 *
 * @type {
 *     {
 *     _audioContext: null,
 *     lastCentsDeviationsDetected: Array<number>,
 *     sampleBuffer: (Float32Array|null),
 *     maxFrequency: number,
 *     lastPitchesDetected: number[],
 *     fftSize: number,
 *     animationFrameCallbackId: number,
 *     AudioContextCaller: *,
 *     currentAnalyser: null,
 *     pitchSmoothingSize: number,
 *     minFrequency: number,
 *     lastPitchClassesDetected: number[]
 *     }
 * }
 */
export const config = {
    fftSize: 2048,
    AudioContextCaller: window.AudioContext || window.webkitAudioContext,
    _audioContext: null,
    animationFrameCallbackId: -1,
    sampleBuffer: null,
    currentAnalyser: null,
    minFrequency: 55,
    maxFrequency: 1050,
    pitchSmoothingSize: 40,
    lastPitchClassesDetected: [],
    lastPitchesDetected: [],
    lastCentsDeviationsDetected: [],

};

Object.defineProperties(config, {
    audioContext: {
        get: () => {
            if (config._audioContext !== null) {
                return config._audioContext;
            } else {
                // AudioContext should be a singleton, but MIDI reports loaded before it is!
                if (
                    MIDI !== undefined
                    && MIDI.WebAudio !== undefined
                    && MIDI.WebAudio.getContext() !== undefined
                ) {
                    window.globalAudioContext = MIDI.WebAudio.getContext();
                } else if (typeof window.globalAudioContext === 'undefined') {
                    window.globalAudioContext = new config.AudioContextCaller();
                }
                config._audioContext = window.globalAudioContext;
                return config._audioContext;
            }
        },
        set: ac => {
            config._audioContext = ac;
        },
    },
});

/**
 *
 * @function music21.audioSearch.getUserMedia
 * @memberof music21.audioSearch
 * @param {Object} dictionary - optional dictionary to fill
 * @param {function} callback - callback on success
 * @param {function} error - callback on error
 */
export function getUserMedia(dictionary, callback, error) {
    if (error === undefined) {
        /* eslint no-alert: "off"*/
        error = () => {
            alert(
                'navigator.getUserMedia either not defined (Safari, IE) or denied.'
            );
        };
    }
    if (callback === undefined) {
        callback = mediaStream => {
            userMediaStarted(mediaStream);
        };
    }
    const n = navigator;
    // need to polyfill navigator, or binding problems are hard...
    // noinspection JSUnresolvedVariable
    n.getUserMedia
        = n.getUserMedia
        || n.webkitGetUserMedia
        || n.mozGetUserMedia
        || n.msGetUserMedia;

    if (n.getUserMedia === undefined) {
        error();
    }
    if (dictionary === undefined) {
        dictionary = {
            audio: {
                mandatory: {},
                optional: [],
            },
        };
    }
    n.getUserMedia(dictionary, callback, error);
}

export function userMediaStarted(audioStream) {
    /**
     * This function which patches Safari requires some time to get started
     * so we call it on object creation.
     */
    config.sampleBuffer = new Float32Array(config.fftSize / 2);
    const mediaStreamSource = config.audioContext.createMediaStreamSource(
        audioStream
    );
    const analyser = config.audioContext.createAnalyser();
    analyser.fftSize = config.fftSize;
    mediaStreamSource.connect(analyser);
    config.currentAnalyser = analyser;
    animateLoop();
}

export const animateLoop = () => {
    config.currentAnalyser.getFloatTimeDomainData(
        config.sampleBuffer
    );
    // returns best frequency or -1
    const frequencyDetected = autoCorrelate(
        config.sampleBuffer,
        config.audioContext.sampleRate,
        config.minFrequency,
        config.maxFrequency
    );
    const retValue = sampleCallback(frequencyDetected);
    // callback can be anything.
    // noinspection JSIncompatibleTypesComparison
    if (retValue !== -1) {
        config.animationFrameCallbackId = window.requestAnimationFrame(
            animateLoop
        );
    }
};

export function smoothPitchExtraction(frequency) {
    if (frequency === -1) {
        config.lastPitchClassesDetected.shift();
        config.lastPitchesDetected.shift();
        config.lastCentsDeviationsDetected.shift();
    } else {
        const [midiNum, centsOff] = midiNumDiffFromFrequency(
            frequency
        );
        if (
            config.lastPitchClassesDetected.length
            > config.pitchSmoothingSize
        ) {
            config.lastPitchClassesDetected.shift();
            config.lastPitchesDetected.shift();
            config.lastCentsDeviationsDetected.shift();
        }
        config.lastPitchClassesDetected.push(midiNum % 12);
        config.lastPitchesDetected.push(midiNum);
        config.lastCentsDeviationsDetected.push(centsOff);
    }
    const mostCommonPitchClass = common.statisticalMode(
        config.lastPitchClassesDetected
    );
    if (mostCommonPitchClass === null) {
        return [-1, 0];
    }
    const pitchesMatchingClass = [];
    const centsMatchingClass = [];
    for (let i = 0; i < config.lastPitchClassesDetected.length; i++) {
        if (config.lastPitchClassesDetected[i] === mostCommonPitchClass) {
            pitchesMatchingClass.push(config.lastPitchesDetected[i]);
            centsMatchingClass.push(config.lastCentsDeviationsDetected[i]);
        }
    }
    const mostCommonPitch = common.statisticalMode(pitchesMatchingClass);

    // find cents difference; weighing more recent samples more...
    let totalSamplePoints = 0;
    let totalSample = 0;
    for (let j = 0; j < centsMatchingClass.length; j++) {
        const weight = (j ** 2) + 1;
        totalSample += weight * centsMatchingClass[j];
        totalSamplePoints += weight;
    }
    const centsOff = Math.floor(totalSample / totalSamplePoints);
    return [mostCommonPitch, centsOff];
}

export function sampleCallback(frequency) {
    // noinspection JSUnusedLocalSymbols
    const [unused_midiNum, unused_centsOff] = smoothPitchExtraction(
        frequency
    );
}

// from Chris Wilson. Replace with Jordi's
export function autoCorrelate(
    buf,
    sampleRate,
    minFrequency,
    maxFrequency
) {
    const SIZE = buf.length;
    const MAX_SAMPLES = Math.floor(SIZE / 2);
    if (minFrequency === undefined) {
        minFrequency = 0;
    }
    if (maxFrequency === undefined) {
        maxFrequency = sampleRate;
    }

    let best_offset = -1;
    let best_correlation = 0;
    let rms = 0;
    let foundGoodCorrelation = false;
    const correlations = new Array(MAX_SAMPLES);

    for (let i = 0; i < SIZE; i++) {
        const val = buf[i];
        rms += val * val;
    }
    rms = Math.sqrt(rms / SIZE);
    if (rms < 0.01) {
        return -1;
    } // not enough signal

    let lastCorrelation = 1;
    for (let offset = 0; offset < MAX_SAMPLES; offset++) {
        let correlation = 0;
        const offsetFrequency = sampleRate / offset;
        if (offsetFrequency < minFrequency) {
            break;
        }
        if (offsetFrequency > maxFrequency) {
            continue;
        }

        for (let i = 0; i < MAX_SAMPLES; i++) {
            correlation += Math.abs(buf[i] - buf[i + offset]);
        }
        correlation = 1 - correlation / MAX_SAMPLES;
        correlations[offset] = correlation; // store it, for the tweaking we need to do below.
        if (correlation > 0.9 && correlation > lastCorrelation) {
            foundGoodCorrelation = true;
            if (correlation > best_correlation) {
                best_correlation = correlation;
                best_offset = offset;
            }
        } else if (foundGoodCorrelation) {
            // short-circuit - we found a good correlation, then a bad one, so we'd just be seeing copies from here.
            // Now we need to tweak the offset - by interpolating between the values to the left and right of the
            // best offset, and shifting it a bit.  This is complex, and HACKY in this code (happy to take PRs!) -
            // we need to do a curve fit on correlations[] around best_offset in order to better determine precise
            // (anti-aliased) offset.

            // we know best_offset >=1,
            // since foundGoodCorrelation cannot go to true until the second pass (offset=1), and
            // we can't drop into this clause until the following pass (else if).
            const shift
                = (correlations[best_offset + 1]
                    - correlations[best_offset - 1])
                / correlations[best_offset];
            return sampleRate / (best_offset + 8 * shift);
        }
        lastCorrelation = correlation;
    }
    if (best_correlation > 0.01) {
        // console.log("f = " + sampleRate/best_offset + "Hz (rms: " + rms + " confidence: " + best_correlation + ")")
        return sampleRate / best_offset;
    }
    return -1;
    //  var best_frequency = sampleRate/best_offset;
}

/**
 *
 * @function midiNumDiffFromFrequency
 * @param {Number} frequency
 * @returns {Array<int>} [miniNumber, centsOff]
 */
export function midiNumDiffFromFrequency(
    frequency
) {
    const midiNumFloat = 12 * (Math.log(frequency / 440) / Math.log(2)) + 69;
    const midiNum = Math.round(midiNumFloat);
    const centsOff = Math.round(100 * (midiNumFloat - midiNum));
    return [midiNum, centsOff];
}
Music21j, Copyright © 2013-2021 Michael Scott Asato Cuthbert.
Documentation generated by JSDoc 3.6.3 on Wed Jul 31st 2019 using the DocStrap template.