diff --git a/web/index.html b/web/index.html index 3381dc7..4c9f159 100644 --- a/web/index.html +++ b/web/index.html @@ -88,6 +88,11 @@ +
+ visuals + + +
@@ -148,6 +153,7 @@
r
artist radio
x
not for me
s
share
+
v
liquid / simple visuals
m
mute
?
this help
diff --git a/web/src/blob/blob.ts b/web/src/blob/blob.ts new file mode 100644 index 0000000..5a45886 --- /dev/null +++ b/web/src/blob/blob.ts @@ -0,0 +1,259 @@ +// The liquid artwork drop: one WebGL2 canvas, moved into the playing card's art slot. +// Browsers allow only a handful of WebGL contexts, so there is one for the whole +// page; cards around it show their artwork as a plain image. + +import type { Features } from "../features"; +import { icosphere } from "./mesh"; +import { FRAGMENT, VERTEX } from "./shaders"; +import type { Look } from "./style"; + +const FOV = (30 * Math.PI) / 180; +/// The canvas is this much larger than the art slot, so the liquid can spill out. +export const SPILL = 1.5; +const DIST = SPILL / Math.tan(FOV / 2); +const MORPH_IN = 0.7; // s from flat artwork to liquid +const TEXTURE_SIZE = 512; +const SLOW_FRAME = 0.02; // s; frames slower than this for SLOW_FOR … +const SLOW_FOR = 2; // … seconds step the quality down +/// Quality steps: icosphere subdivisions and resolution scale. +const QUALITY: [number, number][] = [[5, 1], [4, 1], [4, 0.6], [3, 0.5]]; + +const UNIFORMS = [ + "uProj", "uDist", "uTime", "uMorph", "uLevel", "uWobble", "uKick", "uKickAge", "uSnare", "uDrop", + "uGain", "uSpike", "uStep", "uTwist", "uRipple", "uShock", "uPointer", + "uArt", "uAccent", "uHue", "uIri", "uMetal", "uHeat", +] as const; + +export class LiquidBlob { + readonly canvas = document.createElement("canvas"); + /// Called once if the GPU drops the context; the page then falls back to plain art. + onLost: (() => void) | null = null; + private gl: WebGL2RenderingContext; + private u = {} as Record<(typeof UNIFORMS)[number], WebGLUniformLocation | null>; + private vao: WebGLVertexArrayObject; + private buffers: WebGLBuffer[] = []; + private count = 0; + private tex: WebGLTexture; + private quality: number; + private look: Look | null = null; + private accent: [number, number, number] = [1, 0.3, 0.43]; + private ready = false; + private gen = 0; + private morph = 0; + private time = 0; + private pointer = [0, 0, 1, 0]; + private pointerGoal = [0, 0, 1, 0]; + private slow = 0; + + /// Null where WebGL2 is not available. + static create(touch: boolean): LiquidBlob | null { + try { + return new LiquidBlob(touch); + } catch (err) { + console.warn("liquid visuals off:", (err as Error).message); + return null; + } + } + + private constructor(private touch: boolean) { + const gl = this.canvas.getContext("webgl2", { antialias: true, premultipliedAlpha: true }); + if (!gl) throw new Error("no WebGL2"); + this.gl = gl; + this.canvas.className = "liquid-canvas"; + this.canvas.addEventListener("webglcontextlost", () => { + this.ready = false; + this.onLost?.(); + }); + const prog = gl.createProgram(); + gl.attachShader(prog, this.shader(gl.VERTEX_SHADER, VERTEX)); + gl.attachShader(prog, this.shader(gl.FRAGMENT_SHADER, FRAGMENT)); + gl.bindAttribLocation(prog, 0, "position"); + gl.linkProgram(prog); + if (!gl.getProgramParameter(prog, gl.LINK_STATUS)) throw new Error(gl.getProgramInfoLog(prog) || "link failed"); + gl.useProgram(prog); + for (const name of UNIFORMS) this.u[name] = gl.getUniformLocation(prog, name); + this.vao = gl.createVertexArray(); + this.quality = touch ? 1 : 0; + this.buildMesh(); + this.tex = gl.createTexture(); + gl.bindTexture(gl.TEXTURE_2D, this.tex); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR_MIPMAP_LINEAR); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); + gl.uniform1i(this.u.uArt, 0); + gl.uniformMatrix4fv(this.u.uProj, false, perspective(FOV, 1, 0.1, 20)); + gl.uniform1f(this.u.uDist, DIST); + gl.enable(gl.DEPTH_TEST); + gl.clearColor(0, 0, 0, 0); + } + + private shader(type: number, src: string): WebGLShader { + const gl = this.gl; + const s = gl.createShader(type)!; + gl.shaderSource(s, src); + gl.compileShader(s); + if (!gl.getShaderParameter(s, gl.COMPILE_STATUS)) throw new Error(gl.getShaderInfoLog(s) || "compile failed"); + return s; + } + + private buildMesh(): void { + const gl = this.gl; + const mesh = icosphere(QUALITY[this.quality][0]); + for (const b of this.buffers) gl.deleteBuffer(b); + gl.bindVertexArray(this.vao); + const pos = gl.createBuffer(); + gl.bindBuffer(gl.ARRAY_BUFFER, pos); + gl.bufferData(gl.ARRAY_BUFFER, mesh.positions, gl.STATIC_DRAW); + gl.enableVertexAttribArray(0); + gl.vertexAttribPointer(0, 3, gl.FLOAT, false, 0, 0); + const idx = gl.createBuffer(); + gl.bindBuffer(gl.ELEMENT_ARRAY_BUFFER, idx); + gl.bufferData(gl.ELEMENT_ARRAY_BUFFER, mesh.indices, gl.STATIC_DRAW); + this.buffers = [pos, idx]; + this.count = mesh.indices.length; + } + + /// Show a new track in `slot`. Resolves once its artwork is on the GPU, which is + /// when the slot's own image can be hidden. + async mount(slot: HTMLElement, look: Look, art: string | null, initial: string): Promise { + if (this.canvas.parentElement !== slot) slot.append(this.canvas); + const gen = ++this.gen; + this.look = look; + this.ready = false; + this.morph = 0; + this.slow = 0; + const source = await artwork(art, initial).catch(() => artwork(null, initial)); + if (gen !== this.gen || this.gl.isContextLost()) return false; + const gl = this.gl; + gl.bindTexture(gl.TEXTURE_2D, this.tex); + gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, source); + gl.generateMipmap(gl.TEXTURE_2D); + this.ready = true; + return true; + } + + /// Rim colour, as CSS `hsl()` from color.ts or any `#rrggbb`. + setAccent(css: string | null): void { + this.accent = parseColor(css) ?? [1, 0.3, 0.43]; + } + + /// Pointer over the slot: x, y from the centre in units of the art's radius + /// (y up); `on` false when it left or was lifted. + setPointer(x: number, y: number, on: boolean): void { + const r = Math.hypot(x, y); + const [dx, dy, dz] = r < 1 ? [x, y, Math.sqrt(1 - r * r)] : [x / r, y / r, 0]; + this.pointerGoal = [dx, dy, dz, on && r < 1.4 ? 1 : 0]; + } + + /// Draw one frame. Returns true while it still moves without music (morphing in, + /// settling, following the pointer). + frame(f: Features, dt: number, playing: boolean): boolean { + const look = this.look; + if (!look || !this.ready) return false; + const gl = this.gl; + this.watch(dt); + const scale = Math.min(devicePixelRatio || 1, this.touch ? 1 : 1.5) * QUALITY[this.quality][1]; + const w = Math.round(this.canvas.clientWidth * scale), h = Math.round(this.canvas.clientHeight * scale); + if (!w || !h) return false; + if (this.canvas.width !== w || this.canvas.height !== h) { + this.canvas.width = w; + this.canvas.height = h; + } + gl.viewport(0, 0, w, h); + + this.morph = Math.min(1, this.morph + dt / MORPH_IN); + const morph = 1 - (1 - this.morph) ** 3; + // The liquid flows with the tempo, faster when the music moves more. + this.time += dt * look.flow * (playing ? 0.6 + 0.8 * f.wobble + 0.6 * f.drop : 0.25); + const k = 1 - Math.exp(-dt / 0.15); + for (let i = 0; i < 4; i++) this.pointer[i] += (this.pointerGoal[i] - this.pointer[i]) * k; + + const u = this.u; + gl.uniform1f(u.uTime, this.time); + gl.uniform1f(u.uMorph, morph); + gl.uniform1f(u.uLevel, (f.sub + f.low) / 2); + gl.uniform1f(u.uWobble, f.wobble); + gl.uniform1f(u.uKick, f.kick); + gl.uniform1f(u.uKickAge, f.kickAge); + gl.uniform1f(u.uSnare, f.snare); + gl.uniform1f(u.uDrop, f.drop); + gl.uniform1f(u.uGain, look.gain); + gl.uniform1f(u.uSpike, look.spike); + gl.uniform1f(u.uStep, look.step); + gl.uniform1f(u.uTwist, look.twist); + gl.uniform1f(u.uRipple, look.ripple); + gl.uniform1f(u.uShock, look.shock); + gl.uniform1f(u.uIri, look.iri); + gl.uniform1f(u.uMetal, look.metal); + gl.uniform1f(u.uHeat, look.heat); + gl.uniform1f(u.uHue, look.hue); + gl.uniform3fv(u.uAccent, this.accent); + gl.uniform4fv(u.uPointer, this.pointer); + + gl.clear(gl.COLOR_BUFFER_BIT | gl.DEPTH_BUFFER_BIT); + gl.bindVertexArray(this.vao); + gl.drawElements(gl.TRIANGLES, this.count, gl.UNSIGNED_SHORT, 0); + return this.morph < 1 || this.pointer[3] > 0.01 || this.pointerGoal[3] > 0; + } + + /// Step the quality down when frames keep coming in slow. + private watch(dt: number): void { + this.slow = dt > SLOW_FRAME ? this.slow + dt : Math.max(0, this.slow - dt); + if (this.slow < SLOW_FOR || this.quality >= QUALITY.length - 1) return; + this.slow = 0; + const before = QUALITY[this.quality][0]; + this.quality++; + if (QUALITY[this.quality][0] !== before) this.buildMesh(); + } +} + +/// The artwork cropped square, or a gradient with the artist's initial. +async function artwork(src: string | null, initial: string): Promise { + const c = document.createElement("canvas"); + c.width = c.height = TEXTURE_SIZE; + const ctx = c.getContext("2d")!; + if (src) { + const img = new Image(); + img.src = src; + await img.decode(); + const side = Math.min(img.naturalWidth, img.naturalHeight); + ctx.drawImage(img, (img.naturalWidth - side) / 2, (img.naturalHeight - side) / 2, side, side, 0, 0, TEXTURE_SIZE, TEXTURE_SIZE); + } else { + const g = ctx.createLinearGradient(0, 0, TEXTURE_SIZE, TEXTURE_SIZE); + g.addColorStop(0, "#2a2a2e"); + g.addColorStop(1, "#111"); + ctx.fillStyle = g; + ctx.fillRect(0, 0, TEXTURE_SIZE, TEXTURE_SIZE); + ctx.fillStyle = "#ffffff66"; + ctx.font = `700 ${TEXTURE_SIZE * 0.4}px system-ui, sans-serif`; + ctx.textAlign = "center"; + ctx.textBaseline = "middle"; + ctx.fillText(initial, TEXTURE_SIZE / 2, TEXTURE_SIZE / 2); + } + return c; +} + +function perspective(fov: number, aspect: number, near: number, far: number): Float32Array { + const f = 1 / Math.tan(fov / 2); + const nf = 1 / (near - far); + return new Float32Array([f / aspect, 0, 0, 0, 0, f, 0, 0, 0, 0, (far + near) * nf, -1, 0, 0, 2 * far * near * nf, 0]); +} + +/// `hsl(h s% l%)` (as color.ts makes them) or `#rrggbb`, as 0..1 RGB. +function parseColor(css: string | null): [number, number, number] | null { + if (!css) return null; + const hex = css.match(/^#([0-9a-f]{6})$/i); + if (hex) { + const n = parseInt(hex[1], 16); + return [(n >> 16) / 255, ((n >> 8) & 255) / 255, (n & 255) / 255]; + } + const m = css.match(/hsl\(\s*([\d.]+)\s+([\d.]+)%\s+([\d.]+)%/); + if (!m) return null; + const h = +m[1] / 360, s = +m[2] / 100, l = +m[3] / 100; + const a = s * Math.min(l, 1 - l); + const ch = (n: number) => { + const k = (n + h * 12) % 12; + return l - a * Math.max(-1, Math.min(k - 3, 9 - k, 1)); + }; + return [ch(0), ch(8), ch(4)]; +} diff --git a/web/src/blob/mesh.ts b/web/src/blob/mesh.ts new file mode 100644 index 0000000..256a61e --- /dev/null +++ b/web/src/blob/mesh.ts @@ -0,0 +1,46 @@ +// Unit icosphere: an icosahedron with each triangle split into four, `levels` times. +// Level 5 has 10 242 vertices, which still fits 16-bit indices. + +export interface Mesh { + positions: Float32Array; + indices: Uint16Array; +} + +export function icosphere(levels: number): Mesh { + const t = (1 + Math.sqrt(5)) / 2; + const verts: number[][] = [ + [-1, t, 0], [1, t, 0], [-1, -t, 0], [1, -t, 0], + [0, -1, t], [0, 1, t], [0, -1, -t], [0, 1, -t], + [t, 0, -1], [t, 0, 1], [-t, 0, -1], [-t, 0, 1], + ].map(unit); + let faces = [ + [0, 11, 5], [0, 5, 1], [0, 1, 7], [0, 7, 10], [0, 10, 11], + [1, 5, 9], [5, 11, 4], [11, 10, 2], [10, 7, 6], [7, 1, 8], + [3, 9, 4], [3, 4, 2], [3, 2, 6], [3, 6, 8], [3, 8, 9], + [4, 9, 5], [2, 4, 11], [6, 2, 10], [8, 6, 7], [9, 8, 1], + ]; + for (let l = 0; l < levels; l++) { + // Each edge is split once, shared by the two faces on it. + const mids = new Map(); + const mid = (a: number, b: number) => { + const key = a < b ? `${a},${b}` : `${b},${a}`; + let i = mids.get(key); + if (i === undefined) { + const [p, q] = [verts[a], verts[b]]; + i = verts.push(unit([p[0] + q[0], p[1] + q[1], p[2] + q[2]])) - 1; + mids.set(key, i); + } + return i; + }; + faces = faces.flatMap(([a, b, c]) => { + const ab = mid(a, b), bc = mid(b, c), ca = mid(c, a); + return [[a, ab, ca], [b, bc, ab], [c, ca, bc], [ab, bc, ca]]; + }); + } + return { positions: new Float32Array(verts.flat()), indices: new Uint16Array(faces.flat()) }; +} + +function unit([x, y, z]: number[]): number[] { + const l = Math.hypot(x, y, z); + return [x / l, y / l, z / l]; +} diff --git a/web/src/blob/shaders.ts b/web/src/blob/shaders.ts new file mode 100644 index 0000000..bf89b8f --- /dev/null +++ b/web/src/blob/shaders.ts @@ -0,0 +1,164 @@ +// GLSL for the liquid artwork drop. The vertex shader moves a unit sphere with +// noise driven by the music; the fragment shader shows the artwork through the +// surface, with a rim in the card's accent colour. + +// 3D simplex noise by Ian McEwan and Stefan Gustavson (Ashima Arts), MIT licence: +// https://github.com/ashima/webgl-noise +const NOISE = /* glsl */ ` +vec3 mod289(vec3 x) { return x - floor(x * (1.0 / 289.0)) * 289.0; } +vec4 mod289(vec4 x) { return x - floor(x * (1.0 / 289.0)) * 289.0; } +vec4 permute(vec4 x) { return mod289(((x * 34.0) + 10.0) * x); } +vec4 taylorInvSqrt(vec4 r) { return 1.79284291400159 - 0.85373472095314 * r; } + +float snoise(vec3 v) { + const vec2 C = vec2(1.0 / 6.0, 1.0 / 3.0); + const vec4 D = vec4(0.0, 0.5, 1.0, 2.0); + vec3 i = floor(v + dot(v, C.yyy)); + vec3 x0 = v - i + dot(i, C.xxx); + vec3 g = step(x0.yzx, x0.xyz); + vec3 l = 1.0 - g; + vec3 i1 = min(g.xyz, l.zxy); + vec3 i2 = max(g.xyz, l.zxy); + vec3 x1 = x0 - i1 + C.xxx; + vec3 x2 = x0 - i2 + C.yyy; + vec3 x3 = x0 - D.yyy; + i = mod289(i); + vec4 p = permute(permute(permute( + i.z + vec4(0.0, i1.z, i2.z, 1.0)) + + i.y + vec4(0.0, i1.y, i2.y, 1.0)) + + i.x + vec4(0.0, i1.x, i2.x, 1.0)); + float n_ = 0.142857142857; + vec3 ns = n_ * D.wyz - D.xzx; + vec4 j = p - 49.0 * floor(p * ns.z * ns.z); + vec4 x_ = floor(j * ns.z); + vec4 y_ = floor(j - 7.0 * x_); + vec4 x = x_ * ns.x + ns.yyyy; + vec4 y = y_ * ns.x + ns.yyyy; + vec4 h = 1.0 - abs(x) - abs(y); + vec4 b0 = vec4(x.xy, y.xy); + vec4 b1 = vec4(x.zw, y.zw); + vec4 s0 = floor(b0) * 2.0 + 1.0; + vec4 s1 = floor(b1) * 2.0 + 1.0; + vec4 sh = -step(h, vec4(0.0)); + vec4 a0 = b0.xzyw + s0.xzyw * sh.xxyy; + vec4 a1 = b1.xzyw + s1.xzyw * sh.zzww; + vec3 p0 = vec3(a0.xy, h.x); + vec3 p1 = vec3(a0.zw, h.y); + vec3 p2 = vec3(a1.xy, h.z); + vec3 p3 = vec3(a1.zw, h.w); + vec4 norm = taylorInvSqrt(vec4(dot(p0, p0), dot(p1, p1), dot(p2, p2), dot(p3, p3))); + p0 *= norm.x; p1 *= norm.y; p2 *= norm.z; p3 *= norm.w; + vec4 m = max(0.6 - vec4(dot(x0, x0), dot(x1, x1), dot(x2, x2), dot(x3, x3)), 0.0); + m = m * m; + return 42.0 * dot(m * m, vec4(dot(p0, x0), dot(p1, x1), dot(p2, x2), dot(p3, x3))); +} +`; + +export const VERTEX = /* glsl */ `#version 300 es +precision highp float; + +in vec3 position; + +uniform mat4 uProj; +uniform float uDist; // camera distance +uniform float uTime; // flow time, runs with the tempo +uniform float uMorph; // 0 flat disc (the static artwork), 1 liquid +// Music, see features.ts. +uniform float uLevel, uWobble, uKick, uKickAge, uSnare, uDrop; +// Character of the track, see style.ts. +uniform float uGain, uSpike, uStep, uTwist, uRipple, uShock; +uniform vec4 uPointer; // direction on the sphere, strength + +out vec3 vPos; // view space +out vec3 vNormal; // view space +out vec2 vFlat; // front projection of the resting sphere, for the artwork + +${NOISE} + +float disp(vec3 p) { + vec3 q = p; + float a = uTwist * (p.y + 0.3 * sin(uTime)) * (0.5 + uWobble); + q.xz = mat2(cos(a), -sin(a), sin(a), cos(a)) * q.xz; + float t = uTime; + float lobes = snoise(q * 1.1 + vec3(0.0, t * 0.35, t * 0.2)) * (0.05 + 0.12 * uLevel + 0.12 * uDrop); + // Terraces with soft edges: stiff, but no torn shards. + float steps = lobes * 10.0; + lobes = mix(lobes, (floor(steps) + smoothstep(0.3, 0.7, fract(steps))) / 10.0, uStep); + float ripples = snoise(q * 3.2 + vec3(t * 1.3, -t, 0.0)) * (0.012 + 0.07 * uWobble) * uRipple; + float n = snoise(q * 2.4 - vec3(0.0, t * 0.6, t * 0.9)); + float spikes = pow(max(n, 0.0), 3.0) * (0.06 + 0.35 * uSnare + 0.15 * uDrop) * uSpike; + // After each kick a ring runs from the front of the drop round to the back. + float angle = acos(clamp(p.z, -1.0, 1.0)); + float ring = exp(-pow((angle - uKickAge * 6.0) * 2.5, 2.0)) * exp(-uKickAge * 3.0) * 0.14 * uShock; + float swell = uKick * 0.05; + float dent = -0.18 * uPointer.w * exp(-pow(distance(p, uPointer.xyz) * 2.2, 2.0)); + return clamp((lobes + ripples + spikes + ring + swell) * uGain + dent, -0.3, 0.42); +} + +vec3 shape(vec3 p) { + vec3 s = p * (1.0 + disp(p) * uMorph); + s.z *= mix(0.04, 1.0, uMorph); + return s; +} + +void main() { + vec3 p = position; + // Normal from two displaced neighbours along the surface. + vec3 t1 = normalize(cross(p, abs(p.y) < 0.99 ? vec3(0.0, 1.0, 0.0) : vec3(1.0, 0.0, 0.0))); + vec3 t2 = cross(p, t1); + const float e = 0.012; + vec3 s = shape(p); + vec3 n = normalize(cross(shape(normalize(p + t1 * e)) - s, shape(normalize(p + t2 * e)) - s)); + if (dot(n, p) < 0.0) n = -n; + vPos = s - vec3(0.0, 0.0, uDist); + vNormal = n; + vFlat = p.xy; + gl_Position = uProj * vec4(vPos, 1.0); +} +`; + +export const FRAGMENT = /* glsl */ `#version 300 es +precision highp float; + +in vec3 vPos; +in vec3 vNormal; +in vec2 vFlat; + +uniform sampler2D uArt; +uniform vec3 uAccent; +uniform float uTime, uMorph, uKick, uDrop; +uniform float uHue; // rim hue offset, radians +uniform float uIri; // how much the rim hue drifts +uniform float uMetal; // sharper, brighter highlights +uniform float uHeat; // pulls the rim towards orange + +out vec4 outColor; + +vec3 hueRotate(vec3 c, float a) { + const vec3 k = vec3(0.57735); + float ca = cos(a); + return c * ca + cross(k, c) * sin(a) + k * dot(k, c) * (1.0 - ca); +} + +void main() { + vec3 n = normalize(vNormal); + vec3 v = normalize(-vPos); + float fres = pow(1.0 - max(dot(n, v), 0.0), 3.0); + // Looking through the surface bends the artwork like water does. + vec2 bend = n.xy * (0.22 + 0.1 * uDrop) * uMorph; + vec2 uv = 0.5 + vec2(vFlat.x, -vFlat.y) * 0.48 - vec2(bend.x, -bend.y) * 0.5; + vec2 split = n.xy * (0.004 + 0.02 * uKick) * uMorph; + vec3 art = vec3(texture(uArt, uv + split).r, texture(uArt, uv).g, texture(uArt, uv - split).b); + art = mix(art, smoothstep(0.0, 1.0, art), 0.6 * uDrop); + + vec3 light = normalize(vec3(-0.5, 0.7, 0.6)); + float spec = pow(max(dot(n, normalize(light + v)), 0.0), mix(40.0, 140.0, uMetal)) * mix(0.35, 0.8, uMetal); + vec3 rim = hueRotate(uAccent, uHue + uIri * (fres * 3.0 + uTime * 0.3)); + rim = mix(rim, vec3(1.0, 0.35, 0.1), uHeat * 0.6); + + vec3 col = art * mix(1.0, 0.45, fres * uMorph); + col += rim * fres * (0.6 + 0.7 * uKick + 0.4 * uDrop) * uMorph; + col += spec * uMorph; + outColor = vec4(col, 1.0); +} +`; diff --git a/web/src/blob/style.ts b/web/src/blob/style.ts new file mode 100644 index 0000000..b49563c --- /dev/null +++ b/web/src/blob/style.ts @@ -0,0 +1,61 @@ +// The character of the drop for one track, from its tempo, tags and key. Each +// field is a shader uniform; tune here. + +import type { Track } from "../api"; + +export interface Look { + /// Flow speed of the noise. + flow: number; + /// Overall size of the movement. + gain: number; + /// Spikes on the snare. + spike: number; + /// 0 smooth, 1 stepped (stiff, quantised) lobes. + step: number; + twist: number; + /// Fast ripples from the wobble. + ripple: number; + /// Ring that runs over the surface on each kick. + shock: number; + /// Rim hue drift. + iri: number; + metal: number; + heat: number; + /// Rim hue offset from the key, radians. + hue: number; +} + +const BASE: Look = { flow: 1, gain: 1, spike: 0.5, step: 0, twist: 0.15, ripple: 1, shock: 1, iri: 0.1, metal: 0.3, heat: 0, hue: 0 }; + +const TAGS: Record void> = { + riddim: (l) => Object.assign(l, { step: 0.8, metal: 0.9, shock: 1.4, spike: 0.6, ripple: 0.7 }), + tearout: (l) => Object.assign(l, { spike: 1.4, gain: l.gain * 1.25, heat: 0.7 }), + deathstep: (l) => Object.assign(l, { spike: 1.5, gain: l.gain * 1.25, heat: 0.9, metal: 0.7 }), + melodic: (l) => Object.assign(l, { iri: 0.8, spike: 0.15, gain: l.gain * 0.9, flow: l.flow * 0.8, metal: 0.1 }), + colourbass: (l) => Object.assign(l, { iri: 1, spike: 0.25, metal: 0.15 }), + dnb: (l) => Object.assign(l, { ripple: 1.6 }), + "170": (l) => Object.assign(l, { ripple: 1.5 }), + halftime: (l) => Object.assign(l, { flow: l.flow * 0.7, gain: l.gain * 1.15 }), + wonk: (l) => Object.assign(l, { twist: 0.9 }), +}; + +export function lookFor(t: Track): Look { + // About one noise cycle per bar. + const l: Look = { ...BASE, flow: ((t.bpm || 140) / 240) * 0.9 }; + for (const tag of t.tags ?? []) TAGS[tag]?.(l); + l.hue = keyHue(t.key); + return l; +} + +const PITCH: Record = { c: 0, d: 2, e: 4, f: 5, g: 7, a: 9, b: 11 }; + +/// Keys a fifth apart get neighbouring hues: up to ±26° around the accent colour. +function keyHue(key: string | null): number { + const m = key?.trim().match(/^([a-g])\s*([#♯b♭]?)\s*(.*)$/i); + if (!m) return 0; + let pc = PITCH[m[1].toLowerCase()] + (/[#♯]/.test(m[2]) ? 1 : /[b♭]/.test(m[2]) ? -1 : 0); + // A minor key sits with its relative major. + if (/^(m(?!aj)|min)/i.test(m[3])) pc += 3; + const fifths = (((pc * 7) % 12) + 12) % 12; + return (fifths / 12 - 0.5) * 0.9; +} diff --git a/web/src/debug.ts b/web/src/debug.ts new file mode 100644 index 0000000..7d975c4 --- /dev/null +++ b/web/src/debug.ts @@ -0,0 +1,26 @@ +// `?debug=audio`: live bars of what features.ts hears, for tuning its constants by +// ear. Loaded only when asked for. + +import type { Features } from "./features"; + +const KEYS: (keyof Features)[] = ["sub", "low", "mid", "high", "kick", "snare", "wobble", "drop"]; + +export function audioDebug(): (f: Features) => void { + const canvas = document.createElement("canvas"); + canvas.className = "audio-debug"; + const w = (canvas.width = 240), h = (canvas.height = 110); + document.body.append(canvas); + const ctx = canvas.getContext("2d")!; + ctx.font = "10px system-ui, sans-serif"; + const bar = w / KEYS.length; + return (f) => { + ctx.clearRect(0, 0, w, h); + KEYS.forEach((k, i) => { + const v = Math.min(1, Math.max(0, f[k])); + ctx.fillStyle = k === "kick" || k === "snare" ? "#ff4d6d" : k === "drop" ? "#ffb02e" : "#7fd4ff"; + ctx.fillRect(i * bar + 3, h - 14 - v * (h - 20), bar - 6, v * (h - 20)); + ctx.fillStyle = "#fff9"; + ctx.fillText(k, i * bar + 3, h - 3); + }); + }; +} diff --git a/web/src/features.ts b/web/src/features.ts new file mode 100644 index 0000000..04678a3 --- /dev/null +++ b/web/src/features.ts @@ -0,0 +1,182 @@ +// What the music is doing right now, in a form visuals can move to. +// +// Loudness alone is useless for bass music: in a dubstep or riddim drop the sub is +// loud and steady, so a loudness meter sits at its maximum and nothing moves. The +// movement is in the changes, so besides slow band levels this tracks onsets (kick, +// snare), the wobble of the mids, and drops (the low end jumping well above where +// it has been). +// +// All constants that shape the feel are here, in one place, for tuning by ear. + +export interface Features { + /// Band levels, 0..1, against the recent range of each band. For slow breathing. + sub: number; + low: number; + mid: number; + high: number; + /// Onset envelopes: 1 on a hit, then decaying. + kick: number; + snare: number; + /// Seconds since the last kick; large when there was none. + kickAge: number; + /// How much the mids move (wobble bass, growls), 0..1. + wobble: number; + /// 0..1, high while the low end sits well above its recent average. + drop: number; +} + +const BANDS = { sub: [30, 90], low: [90, 250], mid: [250, 2000], high: [2000, 10000] } as const; +type Band = keyof typeof BANDS; + +const SILENCE_DB = -110; +const CEIL_FALL = 4; // dB/s; a band's ceiling follows its peaks down this slowly +const FLOOR_RISE = 3; // dB/s; and its floor follows the quiet parts up +const MIN_RANGE = 14; // dB; below this a band counts as flat, not as full scale + +const ONSET_WINDOW = 0.75; // s of flux history the threshold adapts to +const ONSET_SIGMA = 1.5; // a hit is this many deviations above the mean flux +const ONSET_MIN_FLUX = 1.2; // dB of mean rise per bin; quieter changes are noise +const ONSET_REFRACTORY = 0.11; // s +const KICK_DECAY = 0.16; // s +const SNARE_DECAY = 0.12; // s + +const WOBBLE_SMOOTH = 0.12; // s +const DROP_SHORT = 1; // s of low-end loudness … +const DROP_LONG = 12; // … against this much +const DROP_RISE_DB = 2.5; // dB above the long average where a drop starts +const DROP_FULL_DB = 5; // dB above the start for a full drop +const DROP_ATTACK = 0.25; // s +const DROP_RELEASE = 2; // s + +const ease = (dt: number, tau: number) => 1 - Math.exp(-dt / tau); +const clamp01 = (x: number) => Math.min(1, Math.max(0, x)); + +/// Maps a band's dB level into its recent range. +class Range { + private floor = -60; + private ceil = -30; + + update(db: number, dt: number): number { + this.ceil = db > this.ceil ? db : this.ceil - CEIL_FALL * dt; + this.floor = db < this.floor ? db : this.floor + FLOOR_RISE * dt; + const span = Math.max(MIN_RANGE, this.ceil - this.floor); + return clamp01((db - (this.ceil - span)) / span); + } +} + +/// Spectral flux against its own running mean and spread. +class Onset { + private mean = 0; + private variance = 1; + private last = -1; + env = 0; + age = 99; + + constructor(private decay: number) {} + + update(flux: number, now: number, dt: number): void { + const threshold = this.mean + ONSET_SIGMA * Math.sqrt(this.variance); + const hit = flux > threshold && flux > ONSET_MIN_FLUX && now - this.last > ONSET_REFRACTORY; + const a = ease(dt, ONSET_WINDOW); + const d = flux - this.mean; + this.mean += a * d; + this.variance += a * (d * d - this.variance); + this.age += dt; + if (hit) { + this.last = now; + this.env = 1; + this.age = 0; + } else { + this.env *= Math.exp(-dt / this.decay); + } + } +} + +export class FeatureTracker { + readonly f: Features = { sub: 0, low: 0, mid: 0, high: 0, kick: 0, snare: 0, kickAge: 99, wobble: 0, drop: 0 }; + private bins: Float32Array | null = null; + private prev: Float32Array | null = null; + private ranges = { sub: new Range(), low: new Range(), mid: new Range(), high: new Range() }; + private kick = new Onset(KICK_DECAY); + private snare = new Onset(SNARE_DECAY); + private lastMid = 0; + private wobblePeak = 1; + private short = SILENCE_DB; + private long = SILENCE_DB; + + /// One frame from the analyser. + fromAnalyser(an: AnalyserNode, now: number, dt: number): Features { + const n = an.frequencyBinCount; + if (this.bins?.length !== n) { + this.bins = new Float32Array(n); + this.prev = new Float32Array(n).fill(SILENCE_DB); + } + const bins = this.bins, prev = this.prev!; + an.getFloatFrequencyData(bins); + const hz = an.context.sampleRate / an.fftSize; + const bin = (f: number) => Math.min(n - 1, Math.max(1, Math.round(f / hz))); + + const db: Record = { sub: 0, low: 0, mid: 0, high: 0 }; + const flux: Record = { sub: 0, low: 0, mid: 0, high: 0 }; + for (const band of Object.keys(BANDS) as Band[]) { + const [lo, hi] = BANDS[band]; + const a = bin(lo), b = Math.max(a, bin(hi)); + let power = 0, rise = 0; + for (let i = a; i <= b; i++) { + const v = Math.max(SILENCE_DB, bins[i]); + power += 10 ** (v / 10); + rise += Math.max(0, v - prev[i]); + prev[i] = v; + } + const count = b - a + 1; + db[band] = Math.max(SILENCE_DB, 10 * Math.log10(power / count)); + flux[band] = rise / count; + } + + const f = this.f; + for (const band of Object.keys(BANDS) as Band[]) f[band] = this.ranges[band].update(db[band], dt); + this.kick.update((flux.sub + flux.low) / 2, now, dt); + this.snare.update((flux.mid + flux.high) / 2, now, dt); + f.kick = this.kick.env; + f.kickAge = this.kick.age; + f.snare = this.snare.env; + this.track(f.mid, Math.max(db.sub, db.low), dt); + return f; + } + + /// One frame where there is no analyser: only the loudness from the waveform. + fromLoudness(level: number, dt: number): Features { + const f = this.f; + f.sub = f.low = f.mid = f.high = level; + f.kick = f.snare = 0; + f.kickAge = 99; + this.track(level, level > 0 ? 20 * Math.log10(level) : SILENCE_DB, dt); + return f; + } + + /// Nothing playing: let everything settle. + idle(dt: number): Features { + const f = this.f; + const k = Math.exp(-dt / 0.35); + f.sub *= k; f.low *= k; f.mid *= k; f.high *= k; + f.kick *= k; f.snare *= k; f.wobble *= k; f.drop *= k; + f.kickAge += dt; + return f; + } + + /// Wobble from how fast the mids move, drop from the low end against its past. + private track(mid: number, lowDb: number, dt: number): void { + const f = this.f; + const speed = Math.abs(mid - this.lastMid) / Math.max(dt, 1 / 240); + this.lastMid = mid; + this.wobblePeak = Math.max(1, speed, this.wobblePeak * Math.exp(-dt / 4)); + f.wobble += (clamp01(speed / this.wobblePeak) - f.wobble) * ease(dt, WOBBLE_SMOOTH); + + // Start from the first sound heard, or every track would open with a drop. + if (this.long <= SILENCE_DB) this.short = this.long = lowDb; + this.short += (lowDb - this.short) * ease(dt, DROP_SHORT); + this.long += (lowDb - this.long) * ease(dt, DROP_LONG); + const target = clamp01((this.short - this.long - DROP_RISE_DB) / DROP_FULL_DB); + f.drop += (target - f.drop) * ease(dt, target > f.drop ? DROP_ATTACK : DROP_RELEASE); + } +} diff --git a/web/src/main.ts b/web/src/main.ts index 53eb321..ff0e9d2 100644 --- a/web/src/main.ts +++ b/web/src/main.ts @@ -5,7 +5,7 @@ import type { Filters, Mode, Pl8Status, Stats, Track, User } from "./api"; import { Card, type CardActions, cleanTitle } from "./cards"; import { Pager } from "./pager"; import { Player } from "./player"; -import { Visualizer } from "./visual"; +import { type VisualsSetting, Visualizer } from "./visual"; const PREFETCH_AHEAD = 4; // fetch more when this close to the end const SKIP_SECONDS = 8; // leaving earlier than this counts as a skip @@ -36,7 +36,7 @@ const loginEl = $("login"); const isTouch = !matchMedia("(hover: hover) and (pointer: fine)").matches; const player = new Player(); -const visual = new Visualizer(player); +const visual = new Visualizer(player, isTouch); const cards: Card[] = []; let active = -1; let loading = false; @@ -524,12 +524,25 @@ modeChip.addEventListener("click", () => setMode({ mode: "classic" })); filtersForm.addEventListener("change", (e) => { const input = e.target as HTMLInputElement; if (input.name === "mode") setMode({ mode: input.value as "classic" | "riddim" | "fresh" | "deep" }); + if (input.name === "visuals") setVisuals(input.value as VisualsSetting); }); + +function setVisuals(v: VisualsSetting): void { + visual.visuals = v; + for (const input of filtersForm.querySelectorAll("input[name=visuals]")) input.checked = input.value === v; +} +setVisuals(visual.visuals); showMode(); filtersBtn.addEventListener("click", () => togglePanel(filtersForm)); filtersForm.addEventListener("submit", (e) => { e.preventDefault(); applyFilters(); }); -$("filters-clear").addEventListener("click", () => { filtersForm.reset(); applyFilters(); }); +$("filters-clear").addEventListener("click", () => { + filtersForm.reset(); + // reset() also clears the mode and visuals radios, which "clear" leaves alone. + showMode(); + setVisuals(visual.visuals); + applyFilters(); +}); showFilters(); // Keyboard -------------------------------------------------------------- @@ -559,6 +572,7 @@ document.addEventListener("keydown", (e) => { else if (k === "r" && card) actions.radio(card); else if (k === "x" && card) dislike(card); else if (k === "s" && card) share(card); + else if (k === "v") { setVisuals(visual.visuals === "liquid" ? "simple" : "liquid"); toast(`${visual.visuals} visuals`); } else if (k === "m") { player.muted = !player.muted; toast(player.muted ? "muted" : "sound on"); } else if (k === "?") { helpEl.hidden ? showHelp() : hideHelp(); } }); diff --git a/web/src/player.ts b/web/src/player.ts index 082542e..db38b3e 100644 --- a/web/src/player.ts +++ b/web/src/player.ts @@ -148,8 +148,8 @@ export class Player { return; } const analyser = ctx.createAnalyser(); - analyser.fftSize = 1024; - analyser.smoothingTimeConstant = 0.7; + analyser.fftSize = 2048; + analyser.smoothingTimeConstant = 0.2; // features.ts smooths per feature; this only takes the edge off the FFT analyser.connect(ctx.destination); for (const d of this.decks) { const gain = ctx.createGain(); diff --git a/web/src/style.css b/web/src/style.css index ba03e4d..cbdee5f 100644 --- a/web/src/style.css +++ b/web/src/style.css @@ -226,7 +226,8 @@ button.danger:not(.link) { background: var(--accent); border: none; color: #fff; position: relative; width: min(16rem, 42vh, 70vw); aspect-ratio: 1; - border-radius: 14px; + /* Round, so the static artwork matches the liquid drop (blob/) it turns into. */ + border-radius: 50%; background: #222 center / cover no-repeat; box-shadow: 0 20px 60px #000a, @@ -240,6 +241,28 @@ button.danger:not(.link) { background: var(--accent); border: none; color: #fff; } .initial { font-size: 4rem; font-weight: 700; color: #fff6; } +/* The liquid drop spills past the art slot; taps go through it to the card. */ +.liquid-canvas { + position: absolute; + inset: -25%; + width: 150%; + height: 150%; + pointer-events: none; +} +.card.liquid .art { background-image: none !important; background-color: transparent; box-shadow: none; scale: 1; } +.card.liquid .initial { display: none; } +.paused-badge, .spinner { z-index: 1; } + +.audio-debug { + position: fixed; + left: 0.7rem; + bottom: 0.7rem; + z-index: 30; + border-radius: var(--radius-s); + background: #000b; + pointer-events: none; +} + .card.paused .art { transform: scale(0.96); filter: brightness(0.7); } .paused-badge { position: absolute; diff --git a/web/src/visual.ts b/web/src/visual.ts index 1fc2d38..ce136a4 100644 --- a/web/src/visual.ts +++ b/web/src/visual.ts @@ -1,32 +1,103 @@ -// Per-frame visuals of the playing card: waveform progress, and a bass level that -// the stylesheet turns into a pulse of the artwork and a glow (`--energy`, 0..1). -// The level comes from the analyser where audio runs through Web Audio, otherwise -// from the track's waveform at the current position. +// Per-frame visuals of the playing card: waveform progress, the liquid artwork drop +// (blob/), and `--energy` (0..1) for the stylesheet's glow and pulse, which is all +// there is where the drop is off or WebGL2 is missing. +// +// What drives them comes from features.ts: from the analyser where audio runs through +// Web Audio, otherwise from the track's waveform at the current position. +import { artistImg } from "./api"; +import { LiquidBlob, SPILL } from "./blob/blob"; +import { lookFor } from "./blob/style"; import type { Card } from "./cards"; +import { accentFor } from "./color"; +import { type Features, FeatureTracker } from "./features"; import type { Player } from "./player"; +export type VisualsSetting = "liquid" | "simple"; +const VISUALS_KEY = "pl8inf.visuals"; + export class Visualizer { private raf = 0; - private bins: Uint8Array | null = null; - private level = 0; - private peak = 0.3; + private last = 0; private card: Card | null = null; + private features = new FeatureTracker(); + /// Undefined until first needed; null where WebGL2 failed. + private blob: LiquidBlob | null | undefined; private reduced = matchMedia("(prefers-reduced-motion: reduce)"); + private setting: VisualsSetting = localStorage.getItem(VISUALS_KEY) === "simple" ? "simple" : "liquid"; + private debug: ((f: Features) => void) | null = null; - constructor(private player: Player) { + constructor( + private player: Player, + private touch: boolean, + ) { document.addEventListener("visibilitychange", () => (document.hidden ? this.stop() : this.start())); + this.reduced.addEventListener("change", () => this.attach(this.card, true)); + for (const ev of ["pointermove", "pointerdown"]) document.addEventListener(ev, (e) => this.pointer(e as PointerEvent, true), { passive: true }); + for (const ev of ["pointerup", "pointercancel", "pointerleave"]) document.addEventListener(ev, (e) => this.pointer(e as PointerEvent, false), { passive: true }); + if (new URLSearchParams(location.search).get("debug") === "audio") { + import("./debug").then((m) => { this.debug = m.audioDebug(); }); + } + } + + get visuals(): VisualsSetting { + return this.setting; + } + + set visuals(v: VisualsSetting) { + this.setting = v; + localStorage.setItem(VISUALS_KEY, v); + this.attach(this.card, true); + } + + /// Whether the liquid drop runs: wanted, motion allowed, and WebGL2 works. + private liquid(): LiquidBlob | null { + if (this.setting !== "liquid" || this.reduced.matches || this.blob === null) return null; + if (this.blob === undefined) { + this.blob = LiquidBlob.create(this.touch); + if (this.blob) { + this.blob.onLost = () => { + this.blob?.canvas.remove(); + this.blob = null; + this.card?.el?.classList.remove("liquid"); + }; + } + } + return this.blob; } /// Follow `card` from now on. - attach(card: Card | null): void { - if (this.card && this.card !== card) this.card.el?.style.setProperty("--energy", "0"); + attach(card: Card | null, again = false): void { + if (this.card === card && !again) return; + if (this.card && this.card !== card) { + this.card.el?.style.setProperty("--energy", "0"); + this.card.el?.classList.remove("liquid"); + } this.card = card; + const blob = this.liquid(); + const slot = card?.el?.querySelector(".art"); + if (!card || !slot || !blob) { + this.blob?.canvas.remove(); + card?.el?.classList.remove("liquid"); + } else { + const img = artistImg(card.track); + blob.setAccent(null); + if (img) accentFor(img).then((c) => { if (this.card === card) blob.setAccent(c); }); + blob.mount(slot, lookFor(card.track), img, (card.track.artist_name || "?")[0].toUpperCase()).then((ok) => { + if (ok && this.card === card) { + card.el?.classList.add("liquid"); + this.start(); + } + }); + } this.start(); } start(): void { - if (!this.raf && !document.hidden) this.raf = requestAnimationFrame(this.frame); + if (!this.raf && !document.hidden) { + this.last = 0; + this.raf = requestAnimationFrame(this.frame); + } } stop(): void { @@ -34,36 +105,42 @@ export class Visualizer { this.raf = 0; } - private frame = () => { + /// Pointer over the drop pushes a dent into it. + private pointer(e: PointerEvent, on: boolean): void { + const blob = this.blob; + if (!blob?.canvas.isConnected) return; + const r = blob.canvas.getBoundingClientRect(); + const radius = r.width / SPILL / 2; + blob.setPointer((e.clientX - r.left - r.width / 2) / radius, -(e.clientY - r.top - r.height / 2) / radius, on); + if (on) this.start(); + } + + private frame = (now: number) => { this.raf = 0; + const dt = this.last ? Math.min(0.1, (now - this.last) / 1000) : 1 / 60; + this.last = now; const card = this.card; - const a = this.player.audio; if (!card) return; + const a = this.player.audio; const progress = a.duration ? a.currentTime / a.duration : 0; card.drawWave(progress); - if (!this.reduced.matches) { - const raw = a.paused ? 0 : this.bass(card, progress); - // Fast attack, slow release, against a slowly falling peak: kicks stand out - // the same whether a track is mastered loud or quiet. - this.level += (raw - this.level) * (raw > this.level ? 0.55 : 0.12); - this.peak = Math.max(this.peak * 0.997, this.level, 0.2); - const energy = Math.max(0, (this.level / this.peak - 0.35) / 0.65); - card.el?.style.setProperty("--energy", energy.toFixed(3)); - } - // Keep running while playing; one last frame after a pause lets the glow fade. - if (!a.paused || this.level > 0.01) this.raf = requestAnimationFrame(this.frame); - }; - - private bass(card: Card, progress: number): number { + const playing = !a.paused; const an = this.player.analyser; - if (!an || !this.player.mixing) return card.loudnessAt(progress) * 0.8; - if (this.bins?.length !== an.frequencyBinCount) this.bins = new Uint8Array(an.frequencyBinCount); - an.getByteFrequencyData(this.bins); - // Bins up to ~150 Hz: kick and sub. - const hz = an.context.sampleRate / an.fftSize; - const top = Math.max(2, Math.round(150 / hz)); - let sum = 0; - for (let i = 1; i <= top; i++) sum += this.bins[i]; - return sum / top / 255; - } + const mixed = !!an && this.player.mixing; + const f = !playing + ? this.features.idle(dt) + : mixed + ? this.features.fromAnalyser(an, now / 1000, dt) + : this.features.fromLoudness(card.loudnessAt(progress), dt); + this.debug?.(f); + let busy = playing; + if (!this.reduced.matches) { + // The kick drives the pulse; without an analyser there are no kicks, only loudness. + const energy = Math.min(1, mixed ? 0.65 * f.kick + 0.35 * f.low : 0.7 * f.low); + card.el?.style.setProperty("--energy", energy.toFixed(3)); + busy ||= energy > 0.01; + if (this.blob && card.el?.classList.contains("liquid")) busy = this.blob.frame(f, dt, playing) || busy; + } + if (busy) this.raf = requestAnimationFrame(this.frame); + }; }