Auto VJ: a director that adapts the visuals to the playing track in real time

The audio engine now tracks musical features: fast/slow energy EMAs, BPM
(median of beat intervals), a quiet/groove/peak mood with hysteresis, and
drop detection (fast energy diverging from slow right after a lull). An
output-side director, toggled from the control UI, switches presets every
16 beats (or ~14s without beats) and immediately on a drop, drawing from
mood-matched family pools (scenic for quiet, fluid/geometry for groove,
fractals/tunnels/fire for peak) while avoiding recent repeats. A manual
effect pick takes over and turns the director off.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

src/audio.js

@@ -51,6 +51,17 @@ class AudioEngine {
     this._bassAvg = 0;        // running average for beat detection
     this._beatCooldown = 0;
 
+    // Musical features for the Auto VJ director.
+    this.energyFast = 0;      // ~0.2s EMA of level
+    this.energySlow = 0;      // ~4s EMA of level (the track's section energy)
+    this.bpm = 0;             // median of recent beat intervals
+    this.beatCount = 0;       // total beats seen (for phrase-synced switching)
+    this.mood = 'groove';     // 'quiet' | 'groove' | 'peak' (with hysteresis)
+    this.drop = 0;            // 1 for one frame when a drop is detected
+    this._beatIvs = [];
+    this._lastBeatT = 0;
+    this._dropCool = 0;
+
     // Log-spaced spectrum for VU/bar visualisers.
     this.NB = 32;
     this.spectrum = new Float32Array(this.NB);
@@ -270,10 +281,41 @@ class AudioEngine {
     if (b > this._bassAvg * 1.35 && b > 0.12 && this._beatCooldown <= 0) {
       this.beat = 1.0;
       this._beatCooldown = 8; // ~min frames between beats
+      // BPM: median of the recent beat-to-beat intervals (0.24-2s plausible).
+      const now = performance.now();
+      if (this._lastBeatT) {
+        const iv = (now - this._lastBeatT) / 1000;
+        if (iv > 0.24 && iv < 2.0) {
+          this._beatIvs = this._beatIvs.slice(-15);
+          this._beatIvs.push(iv);
+          const s = this._beatIvs.slice().sort((a, b2) => a - b2);
+          this.bpm = Math.round(60 / s[s.length >> 1]);
+        }
+      }
+      this._lastBeatT = now;
+      this.beatCount++;
     } else {
       this.beat *= 0.86; // decay
     }
 
+    // Musical mood for the Auto VJ: slow energy places the section, a sudden
+    // fast/slow divergence right after a lull reads as a drop.
+    this.energyFast = this.energyFast * 0.92 + this.level * 0.08;
+    this.energySlow = this.energySlow * 0.995 + this.level * 0.005;
+    this.drop = 0;
+    this._dropCool--;
+    if (this.energyFast > this.energySlow * 1.6 && this.energyFast > 0.35 &&
+        this.energySlow > 0.04 && this._dropCool <= 0) {
+      this.drop = 1;
+      this._dropCool = 360; // at most one drop flag every ~6s
+    }
+    let mood = this.mood;
+    if (this.energySlow < 0.12) mood = 'quiet';
+    else if (this.energySlow > 0.38 || this.drop) mood = 'peak';
+    else if (mood === 'quiet' && this.energySlow > 0.16) mood = 'groove';
+    else if (mood === 'peak' && this.energySlow < 0.32) mood = 'groove';
+    this.mood = mood;
+
     // Log-spaced spectrum bands (for VU-meter style effects).
     const NB = this.NB;
     const minBin = 1;

src/control.html

@@ -108,6 +108,7 @@
       <section class="span2" id="effects-section">
         <label>Libreria effetti <span id="effects-count"></span></label>
         <div class="row">
+          <button id="auto-vj" title="L'Auto VJ ascolta il brano (energia, BPM, drop) e cambia effetto da solo a tempo di musica">🤖 Auto VJ</button>
           <select id="effect-family"><option value="-1">Tutte le famiglie</option></select>
           <input type="text" id="effect-search" class="textfield" placeholder="Cerca effetto…" />
         </div>

src/control.js

@@ -16,6 +16,10 @@ function applyEffect(effect) {
   renderLibrary();
 }
 
+// Auto VJ: the output-side director picks presets in time with the music.
+let autoVjOn = false;
+$('#auto-vj').addEventListener('click', () => send({ type: 'autoVj', on: !autoVjOn }));
+
 // --- Library browser (hundreds of presets, filterable) ---
 const familySel = $('#effect-family');
 EFFECTS.families.forEach((name, i) => {
@@ -1322,6 +1326,14 @@ window.addEventListener('keydown', (e) => {
 // ---------------------------------------------------------------- reports in
 djv.onReport((m) => {
   switch (m.type) {
+    case 'autoVj':
+      autoVjOn = !!m.on;
+      $('#auto-vj').classList.toggle('vj-on', autoVjOn);
+      $('#auto-vj').textContent = autoVjOn ? '🤖 Auto VJ ●' : '🤖 Auto VJ';
+      if (autoVjOn && m.name) {
+        $('#scene-name').textContent = m.name + (m.bpm ? ' · ' + m.bpm + ' BPM' : '');
+      }
+      break;
     case 'devices': {
       const sel = $('#device-select');
       const cur = sel.value;

src/output.js

@@ -299,7 +299,16 @@ async function reportDevices() {
 // ---------------------------------------------------------------- command bus
 djv.onControl(async (m) => {
   switch (m.type) {
-    case 'effect': viz.setEffect(m.effect); break;
+    case 'effect':
+      // A manual pick takes over from the Auto VJ.
+      if (autoVj) { autoVj = false; djv.report({ type: 'autoVj', on: false }); }
+      viz.setEffect(m.effect);
+      break;
+    case 'autoVj':
+      autoVj = !!m.on;
+      if (autoVj) { avLastSwitch = 0; avBeatMark = audio.beatCount || 0; }
+      djv.report({ type: 'autoVj', on: autoVj });
+      break;
     case 'svg': loadCustomTexture(m.dataUrl); break;
     case 'recStart': startRecording(); break;
     case 'recStop': stopRecording(m); break;
@@ -459,10 +468,43 @@ djv.onControl(async (m) => {
 const startTime = performance.now();
 let frames = 0, fpsT = performance.now(), fps = 0, lastReport = 0, prevBeat = 0, lastProgress = 0;
 
+// ---- Auto VJ: a director that adapts the visuals to the playing track ------
+// Uses the engine's musical features (mood/bpm/drop/beatCount): switches preset
+// every 16 beats (or ~14s without beats), immediately on a detected drop, and
+// draws from pools of families that fit the current mood.
+let autoVj = false, avLastSwitch = 0, avBeatMark = 0, avRecent = [];
+const AV_POOLS = {
+  quiet: ['Cielo', 'Aurora', 'Mare', 'Montagne', 'Galassia', 'Nuvole', 'Fluido', 'Onde'],
+  groove: ['Plasma', 'Vortice', 'Truchet', 'Esagoni', 'Spirale', 'Cellule', 'Gyroide',
+    'Fluido', 'Fluido Onda', 'Fluido Vortici', 'Griglia Neon', 'Solidi 3D'],
+  peak: ['Julia', 'Iperspazio', 'Tunnel', 'Moiré', 'Fluido Anello', 'Fluido Fuoco',
+    'Vortice', 'Griglia Neon', 'Cristalli']
+};
+function avPick(mood) {
+  const fams = AV_POOLS[mood] || AV_POOLS.groove;
+  const cands = window.EFFECTS.list.filter(e => fams.indexOf(e.familyName) >= 0 &&
+    (mood === 'peak' ? e.speed >= 1.0 : e.speed <= 1.4) && avRecent.indexOf(e.name) < 0);
+  const e = cands.length ? cands[Math.floor(Math.random() * cands.length)] : window.EFFECTS.list[0];
+  avRecent = [e.name].concat(avRecent).slice(0, 6); // don't repeat the last few
+  return e;
+}
+function avTick(now) {
+  if (!autoVj) return;
+  const dropNow = audio.drop === 1;
+  const due = (audio.beatCount - avBeatMark) >= 16 || (now - avLastSwitch) > 14000;
+  if (!due && !dropNow) return;
+  avBeatMark = audio.beatCount; avLastSwitch = now;
+  const mood = dropNow ? 'peak' : audio.mood;
+  const e = avPick(mood);
+  viz.setEffect(e);
+  djv.report({ type: 'autoVj', on: true, name: e.name, mood, bpm: audio.bpm || 0 });
+}
+
 function frame() {
   const t = (performance.now() - startTime) / 1000;
   audio.update();
   const a = audio.values;
+  avTick(performance.now());
   viz.render(t * speed, a);
 
   // Report beat rising-edges so the control window can auto-cycle effects.

src/style.css

@@ -796,3 +796,6 @@ kbd {
   padding: 1px 5px;
   margin: 0 1px;
 }
+
+/* Auto VJ toggle */
+#auto-vj.vj-on { background: #2e7d32; border-color: #4caf50; color: #fff; }