From 46afd5d31801dedda2af53ca726f81930d43107b Mon Sep 17 00:00:00 2001 From: luciano Date: Fri, 17 Jul 2026 15:32:18 +0200 Subject: [PATCH] Auto VJ: a director that adapts the visuals to the playing track in real time The audio engine now tracks musical features: fast/slow energy EMAs, BPM (median of beat intervals), a quiet/groove/peak mood with hysteresis, and drop detection (fast energy diverging from slow right after a lull). An output-side director, toggled from the control UI, switches presets every 16 beats (or ~14s without beats) and immediately on a drop, drawing from mood-matched family pools (scenic for quiet, fluid/geometry for groove, fractals/tunnels/fire for peak) while avoiding recent repeats. A manual effect pick takes over and turns the director off. Co-Authored-By: Claude Fable 5 --- src/audio.js | 42 ++++++++++++++++++++++++++++++++++++++++++ src/control.html | 1 + src/control.js | 12 ++++++++++++ src/output.js | 44 +++++++++++++++++++++++++++++++++++++++++++- src/style.css | 3 +++ 5 files changed, 101 insertions(+), 1 deletion(-) diff --git a/src/audio.js b/src/audio.js index b18317e..1fa30c1 100644 --- a/src/audio.js +++ b/src/audio.js @@ -51,6 +51,17 @@ class AudioEngine { this._bassAvg = 0; // running average for beat detection this._beatCooldown = 0; + // Musical features for the Auto VJ director. + this.energyFast = 0; // ~0.2s EMA of level + this.energySlow = 0; // ~4s EMA of level (the track's section energy) + this.bpm = 0; // median of recent beat intervals + this.beatCount = 0; // total beats seen (for phrase-synced switching) + this.mood = 'groove'; // 'quiet' | 'groove' | 'peak' (with hysteresis) + this.drop = 0; // 1 for one frame when a drop is detected + this._beatIvs = []; + this._lastBeatT = 0; + this._dropCool = 0; + // Log-spaced spectrum for VU/bar visualisers. this.NB = 32; this.spectrum = new Float32Array(this.NB); @@ -270,10 +281,41 @@ class AudioEngine { if (b > this._bassAvg * 1.35 && b > 0.12 && this._beatCooldown <= 0) { this.beat = 1.0; this._beatCooldown = 8; // ~min frames between beats + // BPM: median of the recent beat-to-beat intervals (0.24-2s plausible). + const now = performance.now(); + if (this._lastBeatT) { + const iv = (now - this._lastBeatT) / 1000; + if (iv > 0.24 && iv < 2.0) { + this._beatIvs = this._beatIvs.slice(-15); + this._beatIvs.push(iv); + const s = this._beatIvs.slice().sort((a, b2) => a - b2); + this.bpm = Math.round(60 / s[s.length >> 1]); + } + } + this._lastBeatT = now; + this.beatCount++; } else { this.beat *= 0.86; // decay } + // Musical mood for the Auto VJ: slow energy places the section, a sudden + // fast/slow divergence right after a lull reads as a drop. + this.energyFast = this.energyFast * 0.92 + this.level * 0.08; + this.energySlow = this.energySlow * 0.995 + this.level * 0.005; + this.drop = 0; + this._dropCool--; + if (this.energyFast > this.energySlow * 1.6 && this.energyFast > 0.35 && + this.energySlow > 0.04 && this._dropCool <= 0) { + this.drop = 1; + this._dropCool = 360; // at most one drop flag every ~6s + } + let mood = this.mood; + if (this.energySlow < 0.12) mood = 'quiet'; + else if (this.energySlow > 0.38 || this.drop) mood = 'peak'; + else if (mood === 'quiet' && this.energySlow > 0.16) mood = 'groove'; + else if (mood === 'peak' && this.energySlow < 0.32) mood = 'groove'; + this.mood = mood; + // Log-spaced spectrum bands (for VU-meter style effects). const NB = this.NB; const minBin = 1; diff --git a/src/control.html b/src/control.html index 10e7cf8..846c2ed 100644 --- a/src/control.html +++ b/src/control.html @@ -108,6 +108,7 @@
+
diff --git a/src/control.js b/src/control.js index 379c35a..472ca23 100644 --- a/src/control.js +++ b/src/control.js @@ -16,6 +16,10 @@ function applyEffect(effect) { renderLibrary(); } +// Auto VJ: the output-side director picks presets in time with the music. +let autoVjOn = false; +$('#auto-vj').addEventListener('click', () => send({ type: 'autoVj', on: !autoVjOn })); + // --- Library browser (hundreds of presets, filterable) --- const familySel = $('#effect-family'); EFFECTS.families.forEach((name, i) => { @@ -1322,6 +1326,14 @@ window.addEventListener('keydown', (e) => { // ---------------------------------------------------------------- reports in djv.onReport((m) => { switch (m.type) { + case 'autoVj': + autoVjOn = !!m.on; + $('#auto-vj').classList.toggle('vj-on', autoVjOn); + $('#auto-vj').textContent = autoVjOn ? '🤖 Auto VJ ●' : '🤖 Auto VJ'; + if (autoVjOn && m.name) { + $('#scene-name').textContent = m.name + (m.bpm ? ' · ' + m.bpm + ' BPM' : ''); + } + break; case 'devices': { const sel = $('#device-select'); const cur = sel.value; diff --git a/src/output.js b/src/output.js index 9e3bae6..0d614ab 100644 --- a/src/output.js +++ b/src/output.js @@ -299,7 +299,16 @@ async function reportDevices() { // ---------------------------------------------------------------- command bus djv.onControl(async (m) => { switch (m.type) { - case 'effect': viz.setEffect(m.effect); break; + case 'effect': + // A manual pick takes over from the Auto VJ. + if (autoVj) { autoVj = false; djv.report({ type: 'autoVj', on: false }); } + viz.setEffect(m.effect); + break; + case 'autoVj': + autoVj = !!m.on; + if (autoVj) { avLastSwitch = 0; avBeatMark = audio.beatCount || 0; } + djv.report({ type: 'autoVj', on: autoVj }); + break; case 'svg': loadCustomTexture(m.dataUrl); break; case 'recStart': startRecording(); break; case 'recStop': stopRecording(m); break; @@ -459,10 +468,43 @@ djv.onControl(async (m) => { const startTime = performance.now(); let frames = 0, fpsT = performance.now(), fps = 0, lastReport = 0, prevBeat = 0, lastProgress = 0; +// ---- Auto VJ: a director that adapts the visuals to the playing track ------ +// Uses the engine's musical features (mood/bpm/drop/beatCount): switches preset +// every 16 beats (or ~14s without beats), immediately on a detected drop, and +// draws from pools of families that fit the current mood. +let autoVj = false, avLastSwitch = 0, avBeatMark = 0, avRecent = []; +const AV_POOLS = { + quiet: ['Cielo', 'Aurora', 'Mare', 'Montagne', 'Galassia', 'Nuvole', 'Fluido', 'Onde'], + groove: ['Plasma', 'Vortice', 'Truchet', 'Esagoni', 'Spirale', 'Cellule', 'Gyroide', + 'Fluido', 'Fluido Onda', 'Fluido Vortici', 'Griglia Neon', 'Solidi 3D'], + peak: ['Julia', 'Iperspazio', 'Tunnel', 'Moiré', 'Fluido Anello', 'Fluido Fuoco', + 'Vortice', 'Griglia Neon', 'Cristalli'] +}; +function avPick(mood) { + const fams = AV_POOLS[mood] || AV_POOLS.groove; + const cands = window.EFFECTS.list.filter(e => fams.indexOf(e.familyName) >= 0 && + (mood === 'peak' ? e.speed >= 1.0 : e.speed <= 1.4) && avRecent.indexOf(e.name) < 0); + const e = cands.length ? cands[Math.floor(Math.random() * cands.length)] : window.EFFECTS.list[0]; + avRecent = [e.name].concat(avRecent).slice(0, 6); // don't repeat the last few + return e; +} +function avTick(now) { + if (!autoVj) return; + const dropNow = audio.drop === 1; + const due = (audio.beatCount - avBeatMark) >= 16 || (now - avLastSwitch) > 14000; + if (!due && !dropNow) return; + avBeatMark = audio.beatCount; avLastSwitch = now; + const mood = dropNow ? 'peak' : audio.mood; + const e = avPick(mood); + viz.setEffect(e); + djv.report({ type: 'autoVj', on: true, name: e.name, mood, bpm: audio.bpm || 0 }); +} + function frame() { const t = (performance.now() - startTime) / 1000; audio.update(); const a = audio.values; + avTick(performance.now()); viz.render(t * speed, a); // Report beat rising-edges so the control window can auto-cycle effects. diff --git a/src/style.css b/src/style.css index 7586ff1..69c1e13 100644 --- a/src/style.css +++ b/src/style.css @@ -796,3 +796,6 @@ kbd { padding: 1px 5px; margin: 0 1px; } + +/* Auto VJ toggle */ +#auto-vj.vj-on { background: #2e7d32; border-color: #4caf50; color: #fff; }