// What the music is doing right now, in a form visuals can move to. // // Loudness alone is useless for bass music: in a dubstep or riddim drop the sub is // loud and steady, so a loudness meter sits at its maximum and nothing moves. The // movement is in the changes, so besides slow band levels this tracks onsets (kick, // snare), the wobble of the mids, and drops (the low end jumping well above where // it has been). // // All constants that shape the feel are here, in one place, for tuning by ear. export interface Features { /// Band levels, 0..1, against the recent range of each band. For slow breathing. sub: number; low: number; mid: number; high: number; /// Onset envelopes: 1 on a hit, then decaying. kick: number; snare: number; /// Seconds since the last kick; large when there was none. kickAge: number; /// Set for the one frame a hit lands: how prominent it is against recent hits, /// 0..1 (the loudest hits of the last seconds are 1). 0 on every other frame. kickHit: number; snareHit: number; /// How much the mids move (wobble bass, growls), 0..1. wobble: number; /// The bass line's own envelope, 0..1 against the last moment: follows every /// "womb" of a riddim wobble up and down. bass: number; /// How open the wobble's filter is, 0..1: low for a dark "oo", high for a bright /// "wah" (spectral centroid of the bass and mids against its recent range). growl: number; /// 0..1, high while the low end sits well above its recent average. drop: number; /// 0..1, high while the low end has dropped out after sounding: the stop before a /// drop, the gaps a riddim leaves. gap: number; /// How often the bass line goes "womb", Hz; 0 until there were a few. wobbleRate: number; /// Where the music is in its bar, from the track's tempo locked onto the kicks: /// `beat` 0..1 within the beat, `beatNo` 0..3, `bar` 0..1 within the bar. beat: number; beatNo: number; bar: number; /// Seconds per beat. period: number; /// How sure the bar is: the strength of recent kicks on the downbeat. Above /// about 2 the downbeat has been hit several times in the last seconds. lock: number; } const BANDS = { sub: [30, 90], low: [90, 250], mid: [250, 2000], high: [2000, 10000] } as const; type Band = keyof typeof BANDS; const SILENCE_DB = -110; const CEIL_FALL = 4; // dB/s; a band's ceiling follows its peaks down this slowly const FLOOR_RISE = 3; // dB/s; and its floor follows the quiet parts up const MIN_RANGE = 14; // dB; below this a band counts as flat, not as full scale const ONSET_WINDOW = 0.75; // s of flux history the threshold adapts to const ONSET_SIGMA = 1.5; // a hit is this many deviations above the mean flux const ONSET_MIN_FLUX = 1.2; // dB of mean rise per bin; quieter changes are noise const ONSET_REFRACTORY = 0.11; // s const KICK_DECAY = 0.16; // s const SNARE_DECAY = 0.12; // s const PROMINENCE_MEMORY = 6; // s; a hit is weighed against the strongest ones this recent const WOBBLE_SMOOTH = 0.12; // s const BASS_BAND = [40, 400]; // Hz; body of a wobble bass const BASS_FOLLOW = 14; // dB/s; its range forgets this fast, so each wub spans it const BASS_MIN_RANGE = 6; // dB const BASS_ATTACK = 0.015; // s const BASS_RELEASE = 0.06; // s const GROWL_BAND = [100, 3000]; // Hz const GROWL_FOLLOW = 0.6; // octaves/s const GROWL_MIN_RANGE = 1.2; // octaves const GROWL_SMOOTH = 0.05; // s const DROP_SHORT = 1; // s of low-end loudness … const DROP_LONG = 12; // … against this much const DROP_HEARD_DB = -70; // the low end counts as sounding above this const DROP_RISE_DB = 2.5; // dB above the long average where a drop starts const DROP_FULL_DB = 5; // dB above the start for a full drop const DROP_ATTACK = 0.25; // s const DROP_RELEASE = 2; // s const GAP_DB = 12; // dB under the long average where the low end counts as gone … const GAP_FULL_DB = 12; // … and this much more for a full gap const GAP_ATTACK = 0.06; // s const GAP_RELEASE = 0.12; // s const GAP_HEARD = 2; // s of sound before a gap can be one const WOMB_UP = 0.62; // the bass envelope crossing up through this is one "womb" … const WOMB_DOWN = 0.38; // … once it fell below this since the last const WOMB_MIN = 0.06; // s; closer wombs are one const WOMB_MAX = 1.5; // s; further apart, the wobble stopped const WOMB_SMOOTH = 0.25; // weight of each new interval const BEATS_PER_BAR = 4; const BEAT_PULL = 0.25; // how far a kick pulls the beat towards itself const BEAT_MIN_HIT = 0.3; // kicks weaker than this leave the beat alone const DOWNBEAT_MEMORY = 8; // s; the downbeat is the slot with the strongest kicks this recent // Without an analyser the waveform stands in. Its steps are coarse, so a rise from // one step to the next is the only "onset" there is. const LOUD_FLOOR = 0.02; // loudness treated as silence, so a rise from nothing is finite const ease = (dt: number, tau: number) => 1 - Math.exp(-dt / tau); const clamp01 = (x: number) => Math.min(1, Math.max(0, x)); /// Maps a value (a band's dB level) into its recent range. class Range { constructor( private floor = -60, private ceil = -30, private ceilFall = CEIL_FALL, private floorRise = FLOOR_RISE, private minRange = MIN_RANGE, ) {} update(x: number, dt: number): number { this.ceil = x > this.ceil ? x : this.ceil - this.ceilFall * dt; this.floor = x < this.floor ? x : this.floor + this.floorRise * dt; const span = Math.max(this.minRange, this.ceil - this.floor); return clamp01((x - (this.ceil - span)) / span); } } /// Spectral flux against its own running mean and spread. class Onset { private mean = 0; private variance = 1; private last = -1; private peak = ONSET_MIN_FLUX; env = 0; age = 99; hit = 0; constructor(private decay: number) {} update(flux: number, now: number, dt: number): void { const threshold = this.mean + ONSET_SIGMA * Math.sqrt(this.variance); const hit = flux > threshold && flux > ONSET_MIN_FLUX && now - this.last > ONSET_REFRACTORY; const a = ease(dt, ONSET_WINDOW); const d = flux - this.mean; this.mean += a * d; this.variance += a * (d * d - this.variance); this.age += dt; this.peak = Math.max(ONSET_MIN_FLUX, this.peak * Math.exp(-dt / PROMINENCE_MEMORY)); this.hit = 0; if (hit) { this.peak = Math.max(this.peak, flux); this.hit = clamp01(flux / this.peak); this.last = now; this.env = 1; this.age = 0; } else { this.env *= Math.exp(-dt / this.decay); } } } export class FeatureTracker { readonly f: Features = { sub: 0, low: 0, mid: 0, high: 0, kick: 0, snare: 0, kickAge: 99, kickHit: 0, snareHit: 0, wobble: 0, bass: 0, growl: 0, drop: 0, gap: 0, wobbleRate: 0, beat: 0, beatNo: 0, bar: 0, period: 60 / 140, lock: 0, }; /// Seconds per beat, from the track's BPM. private period = 60 / 140; /// Beats since the track started, kept in step with the kicks. private phase = 0; /// Kick strength per beat of the bar, fading; the strongest slot is the downbeat. private slots = [0, 0, 0, 0]; private wombAt = -1; private wombArmed = true; private bins: Float32Array | null = null; private prev: Float32Array | null = null; private ranges = { sub: new Range(), low: new Range(), mid: new Range(), high: new Range() }; private kick = new Onset(KICK_DECAY); /// Kicks from the waveform's steps, where there is no analyser. private loudKick = new Onset(KICK_DECAY * 1.5); private lastLevel = 0; private snare = new Onset(SNARE_DECAY); private bassRange = new Range(-60, -30, BASS_FOLLOW, BASS_FOLLOW, BASS_MIN_RANGE); private growlRange = new Range(7, 9, GROWL_FOLLOW, GROWL_FOLLOW, GROWL_MIN_RANGE); private lastMid = 0; private wobblePeak = 1; private short = SILENCE_DB; private long = SILENCE_DB; /// Seconds of sound heard so far: the long average covers no more than that, so /// music that starts in the middle of a drop is the baseline, not a drop. private heard = 0; /// A new track: its tempo, and nothing carried over about where its bar is. /// Half-time notation (70 for a 140 tune) counts double. setTempo(bpm: number | null): void { let b = bpm && bpm >= 50 && bpm <= 220 ? bpm : 140; if (b < 100) b *= 2; this.period = this.f.period = 60 / b; this.phase = 0; this.slots = [0, 0, 0, 0]; this.wombAt = -1; this.f.wobbleRate = 0; this.f.lock = 0; } /// One frame from the analyser. fromAnalyser(an: AnalyserNode, now: number, dt: number): Features { const n = an.frequencyBinCount; if (this.bins?.length !== n) { this.bins = new Float32Array(n); this.prev = new Float32Array(n).fill(SILENCE_DB); } const bins = this.bins, prev = this.prev!; an.getFloatFrequencyData(bins); const hz = an.context.sampleRate / an.fftSize; const bin = (f: number) => Math.min(n - 1, Math.max(1, Math.round(f / hz))); const db: Record = { sub: 0, low: 0, mid: 0, high: 0 }; const flux: Record = { sub: 0, low: 0, mid: 0, high: 0 }; for (const band of Object.keys(BANDS) as Band[]) { const [lo, hi] = BANDS[band]; const a = bin(lo), b = Math.max(a, bin(hi)); let power = 0, rise = 0; for (let i = a; i <= b; i++) { const v = Math.max(SILENCE_DB, bins[i]); power += 10 ** (v / 10); rise += Math.max(0, v - prev[i]); prev[i] = v; } const count = b - a + 1; db[band] = Math.max(SILENCE_DB, 10 * Math.log10(power / count)); flux[band] = rise / count; } const f = this.f; for (const band of Object.keys(BANDS) as Band[]) f[band] = this.ranges[band].update(db[band], dt); // The wobble: loudness of the bass body, and where its energy sits. let bassPower = 0; for (let i = bin(BASS_BAND[0]); i <= bin(BASS_BAND[1]); i++) bassPower += 10 ** (Math.max(SILENCE_DB, bins[i]) / 10); const bassDb = Math.max(SILENCE_DB, 10 * Math.log10(bassPower / (bin(BASS_BAND[1]) - bin(BASS_BAND[0]) + 1))); const bass = this.bassRange.update(bassDb, dt); f.bass += (bass - f.bass) * ease(dt, bass > f.bass ? BASS_ATTACK : BASS_RELEASE); let weight = 0, sum = 0; for (let i = bin(GROWL_BAND[0]); i <= bin(GROWL_BAND[1]); i++) { const p = 10 ** (Math.max(SILENCE_DB, bins[i]) / 10); weight += p; sum += p * Math.log2(i * hz); } if (weight > 0) f.growl += (this.growlRange.update(sum / weight, dt) - f.growl) * ease(dt, GROWL_SMOOTH); this.kick.update((flux.sub + flux.low) / 2, now, dt); this.snare.update((flux.mid + flux.high) / 2, now, dt); f.kick = this.kick.env; f.kickAge = this.kick.age; f.kickHit = this.kick.hit; f.snare = this.snare.env; f.snareHit = this.snare.hit; this.track(f.mid, Math.max(db.sub, db.low), dt); this.rhythm(now, dt); return f; } /// One frame where there is no analyser: only the loudness from the waveform. /// Jumps in it count as kicks, so the drop still lands on the drops. fromLoudness(level: number, now: number, dt: number): Features { const f = this.f; f.sub = f.low = f.mid = f.high = f.bass = level; f.growl = 0.3; const rise = 20 * Math.log10(Math.max(LOUD_FLOOR, level) / Math.max(LOUD_FLOOR, this.lastLevel)); this.lastLevel = level; this.loudKick.update(Math.max(0, rise), now, dt); f.kick = this.loudKick.env; f.kickAge = this.loudKick.age; f.kickHit = this.loudKick.hit; f.snare = f.snareHit = 0; this.track(level, level > 0 ? 20 * Math.log10(level) : SILENCE_DB, dt); this.rhythm(now, dt); return f; } /// Nothing playing: let everything settle. idle(dt: number): Features { const f = this.f; const k = Math.exp(-dt / 0.35); f.sub *= k; f.low *= k; f.mid *= k; f.high *= k; f.kick *= k; f.snare *= k; f.wobble *= k; f.drop *= k; f.bass *= k; f.growl *= k; f.gap *= k; f.kickAge += dt; f.kickHit = f.snareHit = 0; return f; } /// Wobble from how fast the mids move, drop from the low end against its past. private track(mid: number, lowDb: number, dt: number): void { const f = this.f; const speed = Math.abs(mid - this.lastMid) / Math.max(dt, 1 / 240); this.lastMid = mid; this.wobblePeak = Math.max(1, speed, this.wobblePeak * Math.exp(-dt / 4)); f.wobble += (clamp01(speed / this.wobblePeak) - f.wobble) * ease(dt, WOBBLE_SMOOTH); if (this.heard === 0) { if (lowDb < DROP_HEARD_DB) return; this.short = this.long = lowDb; } this.heard += dt; this.short += (lowDb - this.short) * ease(dt, DROP_SHORT); this.long += (lowDb - this.long) * ease(dt, Math.min(DROP_LONG, DROP_SHORT + this.heard)); const target = clamp01((this.short - this.long - DROP_RISE_DB) / DROP_FULL_DB); f.drop += (target - f.drop) * ease(dt, target > f.drop ? DROP_ATTACK : DROP_RELEASE); // The gap is against this frame's level: a stop of half a bar is over before // a smoothed level would notice it. const gap = this.heard < GAP_HEARD ? 0 : clamp01((this.long - lowDb - GAP_DB) / GAP_FULL_DB); f.gap += (gap - f.gap) * ease(dt, gap > f.gap ? GAP_ATTACK : GAP_RELEASE); } /// The beat and the bar, and the rate of the wobble. private rhythm(now: number, dt: number): void { const f = this.f; this.phase += dt / this.period; const fade = Math.exp(-dt / DOWNBEAT_MEMORY); for (let i = 0; i < BEATS_PER_BAR; i++) this.slots[i] *= fade; if (f.kickHit >= BEAT_MIN_HIT) { // A kick lands on the nearest beat: pull the clock towards it. const nearest = Math.round(this.phase); this.phase -= (this.phase - nearest) * BEAT_PULL * f.kickHit; const slot = ((nearest % BEATS_PER_BAR) + BEATS_PER_BAR) % BEATS_PER_BAR; this.slots[slot] += f.kickHit; } f.lock = Math.max(...this.slots); const down = this.slots.indexOf(f.lock); const inBar = (((this.phase - down) % BEATS_PER_BAR) + BEATS_PER_BAR) % BEATS_PER_BAR; f.beatNo = Math.floor(inBar); f.beat = inBar - f.beatNo; f.bar = inBar / BEATS_PER_BAR; // Each "womb" of the bass is its envelope rising through WOMB_UP. if (f.bass < WOMB_DOWN) this.wombArmed = true; if (this.wombArmed && f.bass > WOMB_UP) { this.wombArmed = false; const since = this.wombAt < 0 ? Infinity : now - this.wombAt; if (since >= WOMB_MIN) { if (since <= WOMB_MAX) { const rate = 1 / since; f.wobbleRate = f.wobbleRate ? f.wobbleRate + (rate - f.wobbleRate) * WOMB_SMOOTH : rate; } this.wombAt = now; } } if (this.wombAt >= 0 && now - this.wombAt > WOMB_MAX * 2) f.wobbleRate *= Math.exp(-dt / 0.5); } }