Interattivo Modello: camera-driven 3D model puppet

New interactive mode 'model': the webcam motion drives the GLB model as
a physical puppet — a hand swipe spins it with inertia, it leans toward
the motion centroid, strong agitation makes it hop with squash & stretch
and a touch-glow rim. ModelSim.render accepts an optional pose (yaw,
leanX, hopY, squash, rim) and slows its auto-spin when driven live.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

src/effects.js

@@ -76,7 +76,10 @@ const FAMILIES = [
   { name: 'Metallo Liquido', scale: 1.0 },
   // GLB model renderer (model3d.js), not the uber-shader: spins a loaded 3D
   // model with palette lighting; a torus knot shows before any GLB is loaded.
-  { name: 'Modello 3D', scale: 1.0, model3d: true }
+  { name: 'Modello 3D', scale: 1.0, model3d: true },
+  // Camera-interactive puppet: swipe to spin the model, it leans toward your
+  // movement and hops when you get agitated. Excluded from Auto VJ.
+  { name: 'Interattivo Modello', scale: 1.0, interactive: true, interMode: 'model' }
 ];
 
 // Palettes: low colour (a) -> high colour (b), plus optional hue-cycle/sat.

src/interactive.js

@@ -278,6 +278,7 @@ class InteractiveSim {
   }
 
   _resetMode() {
+    this.mpose = null;
     this.balls = null;
     this.parts = [];       // transient particles (bubble pops)
     this.mirror = null;
@@ -944,6 +945,37 @@ class InteractiveSim {
 
     if (mode === 'fluid') { this._stepFluid(timeSec, audio, e, canvas); return; }
 
+    // ---- 3D model puppet: swipe to spin (with inertia), the model leans
+    // toward where you move, big motion makes it hop with squash & stretch.
+    if (mode === 'model') {
+      if (!this.modelSim) { this._drawField(7, e, audio, canvas, 0); return; }
+      if (!this.mpose) this.mpose = { yaw: 0, yawVel: 0, hopY: 0, hopVel: 0,
+        leanX: 0, squash: 1, rim: 0, hopCool: 0 };
+      const p = this.mpose;
+      // horizontal flow (a hand swiping across the camera) spins the model
+      p.yawVel += this.flowX * dt * 14;
+      p.yawVel *= Math.exp(-dt*1.1);
+      p.yaw += p.yawVel * dt * (e.speed || 1);
+      // lean toward the motion centroid
+      const leanTarget = this.motTotal > 2 ? (this.motCx - 0.5) * 0.55 : 0;
+      p.leanX += (leanTarget - p.leanX) * (1 - Math.exp(-dt*3.5));
+      // strong agitation makes it jump
+      p.hopCool -= dt;
+      if (this.motTotal > 16 && p.hopY <= 0.001 && p.hopCool <= 0) {
+        p.hopVel = 1.5; p.hopCool = 0.9;
+      }
+      p.hopVel -= 5.5 * dt;
+      p.hopY = Math.max(0, p.hopY + p.hopVel * dt);
+      if (p.hopY === 0 && p.hopVel < 0) p.hopVel = 0;
+      // stretch going up, squash on the ground after landing
+      const sqTarget = p.hopY > 0.001 ? 1 + Math.max(-0.2, Math.min(0.18, p.hopVel*0.12)) : 1;
+      p.squash += (sqTarget - p.squash) * (1 - Math.exp(-dt*10));
+      // touch glow from overall motion
+      p.rim += (Math.min(1, this.motTotal/22) - p.rim) * (1 - Math.exp(-dt*5));
+      this.modelSim.render(timeSec, audio, e, canvas, p);
+      return;
+    }
+
     const hueA = (e.hueBase || 0)*6.2832 + timeSec*(e.hueCycle || 0)*6.2832;
     const ca = e.colorA || [0.05, 0, 0.2], cb = e.colorB || [0.2, 1, 1];
     const palCol = (f, off) => {

src/model3d.js

@@ -30,8 +30,11 @@ function m4rotY(a) {
   const c = Math.cos(a), s = Math.sin(a);
   return new Float32Array([c,0,-s,0, 0,1,0,0, s,0,c,0, 0,0,0,1]);
 }
-function m4scaleT(s, t) { // uniform scale then translate
+function m4scale3(sx, sy, sz) {
-  return new Float32Array([s,0,0,0, 0,s,0,0, 0,0,s,0, t[0],t[1],t[2],1]);
+  return new Float32Array([sx,0,0,0, 0,sy,0,0, 0,0,sz,0, 0,0,0,1]);
+}
+function m4trans(t) {
+  return new Float32Array([1,0,0,0, 0,1,0,0, 0,0,1,0, t[0],t[1],t[2],1]);
 }
 // quaternion + TRS -> mat4 (column major)
 function trsToMat(t, r, s) {
@@ -65,7 +68,7 @@ out vec4 frag;
 uniform sampler2D uTex;
 uniform int uHasTex;
 uniform vec3 uBase, uColA, uColB, uCam;
-uniform float uBeat, uLevel, uTreble;
+uniform float uBeat, uLevel, uTreble, uRim;
 void main(){
   vec3 N = normalize(vN);
   vec3 V = normalize(uCam - vW);
@@ -76,7 +79,7 @@ void main(){
   vec3 col = base * (0.26 + 0.85*d);
   col += uColA * 1.6 * max(dot(N, -L), 0.0) * 0.4;          // palette fill light
   float fr = pow(1.0 - max(dot(N, V), 0.0), 3.0);
-  col += uColB * fr * (0.45 + 0.95*uBeat + 0.4*uLevel);     // rim pulses on beat
+  col += uColB * fr * (0.45 + 0.95*uBeat + 0.4*uLevel + 1.3*uRim); // beat + touch
   vec3 H = normalize(L + V);
   col += vec3(1.0) * pow(max(dot(N, H), 0.0), 42.0) * (0.3 + 0.5*uTreble);
   frag = vec4(col, 1.0);
@@ -274,7 +277,7 @@ class ModelSim {
       uBase: U(this.progMesh,'uBase'), uColA: U(this.progMesh,'uColA'),
       uColB: U(this.progMesh,'uColB'), uCam: U(this.progMesh,'uCam'),
       uBeat: U(this.progMesh,'uBeat'), uLevel: U(this.progMesh,'uLevel'),
-      uTreble: U(this.progMesh,'uTreble') };
+      uTreble: U(this.progMesh,'uTreble'), uRim: U(this.progMesh,'uRim') };
     this.ub = { uColA: U(this.progBg,'uColA'), uColB: U(this.progBg,'uColB'),
       uRes: U(this.progBg,'uRes'), uT: U(this.progBg,'uT'),
       uBass: U(this.progBg,'uBass'), uBeat: U(this.progBg,'uBeat') };
@@ -355,7 +358,10 @@ class ModelSim {
     }
   }
 
-  render(timeSec, audio, e, canvas) {
+  // pose (optional, from the camera-interactive mode): { yaw, leanX, hopY,
+  // squash, rim } — extra rotation, sideways lean, jump height (in radii),
+  // vertical squash & stretch, touch-glow 0..1.
+  render(timeSec, audio, e, canvas, pose) {
     const gl = this.gl;
     const speed = Math.min(2.5, e.speed || 1);
     const mix = e.audioMix !== undefined ? e.audioMix : 1;
@@ -383,14 +389,20 @@ class ModelSim {
     // model
     const asp = canvas.width/Math.max(1, canvas.height);
     const dist = this.radius*2.6;
-    const yaw = timeSec*0.45*speed;
+    // With a live pose the auto-spin slows right down: the person drives it.
+    const yaw = timeSec*0.45*speed*(pose ? 0.12 : 1) + (pose ? pose.yaw : 0);
     const eye = [Math.sin(timeSec*0.13)*this.radius*0.35,
                  this.radius*(0.25 + 0.15*Math.sin(timeSec*0.09)), dist];
     const proj = m4persp(0.72, asp, dist*0.05, dist*4.0);
     const view = m4lookAt(eye, [0, 0, 0]);
     const scale = 1 + 0.05*bass + 0.07*beat;
-    const model = m4mul(m4mul(m4rotY(yaw), m4scaleT(scale, [0,0,0])),
+    const sq = pose ? Math.max(0.7, Math.min(1.3, pose.squash || 1)) : 1;
-      m4scaleT(1, [-this.center[0], -this.center[1], -this.center[2]]));
+    const world = pose
+      ? [pose.leanX*this.radius*1.6, (pose.hopY || 0)*this.radius, 0]
+      : [0, 0, 0];
+    const model = m4mul(m4mul(m4mul(m4trans(world), m4rotY(yaw)),
+      m4scale3(scale/Math.sqrt(sq), scale*sq, scale/Math.sqrt(sq))),
+      m4trans([-this.center[0], -this.center[1], -this.center[2]]));
 
     gl.enable(gl.DEPTH_TEST);
     gl.depthFunc(gl.LEQUAL);
@@ -405,6 +417,7 @@ class ModelSim {
     gl.uniform1f(this.um.uBeat, beat);
     gl.uniform1f(this.um.uLevel, (audio.level || 0)*mix);
     gl.uniform1f(this.um.uTreble, (audio.treble || 0)*mix);
+    gl.uniform1f(this.um.uRim, pose ? (pose.rim || 0) : 0);
     for (const m of this.meshes) {
       const bind = (buf, loc, n) => {
         gl.bindBuffer(gl.ARRAY_BUFFER, buf);

src/visualizer.js

@@ -120,6 +120,12 @@ class Visualizer {
     if (e.isInteractive && window.InteractiveSim) {
       if (!this.inter) this.inter = new window.InteractiveSim(gl);
       this.inter.customSource = this.customSource || null;
+      // The model-puppet mode drives the shared 3D renderer.
+      if (e.interactiveMode === 'model' && window.ModelSim) {
+        if (!this.model3d) this.model3d = new window.ModelSim(gl);
+        if (this._pendingGlb) { this.model3d.setModel(this._pendingGlb); this._pendingGlb = null; }
+        this.inter.modelSim = this.model3d;
+      }
       this.inter.render(timeSec, audio, e, this.canvas);
       return;
     }