Interattivo Avatar: a robot whose limbs mirror your tracked body 1:1

Raise your arm and the robot raises its arm. ModelSim gains primitive
meshes (cube/sphere) and renderAvatar(): parts placed by limb matrices
(basis aligned to each bone segment) with the same lighting as the GLB
renderer. interactive.js keeps the mirrored raw landmarks (with depth,
smoothed) and builds the robot every frame: torso sized from the
shoulders, head with accent visor and antenna, articulated arms/legs
with joint spheres, glowing hands, feet. Falls back to the motion mist
when no body is tracked.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

src/effects.js

@@ -79,7 +79,10 @@ const FAMILIES = [
   { name: 'Modello 3D', scale: 1.0, model3d: true },
   // Camera-interactive puppet: swipe to spin the model, it leans toward your
   // movement and hops when you get agitated. Excluded from Auto VJ.
-  { name: 'Interattivo Modello', scale: 1.0, interactive: true, interMode: 'model' }
+  { name: 'Interattivo Modello', scale: 1.0, interactive: true, interMode: 'model' },
+  // Robot avatar: a robot made of 3D parts whose limbs mirror the tracked
+  // body 1:1 (needs the pose tracker; shows motion mist without it).
+  { name: 'Interattivo Avatar', scale: 1.0, interactive: true, interMode: 'avatar' }
 ];
 
 // Palettes: low colour (a) -> high colour (b), plus optional hue-cycle/sat.

src/interactive.js

@@ -343,7 +343,7 @@ class InteractiveSim {
     let raw = null;
     try { raw = this.pose.detect(v, nowMs); } catch (e) { return; }
     if (!raw) {
-      if (++this._bodyMiss > 15) { this.body = null; this.lms = null; }
+      if (++this._bodyMiss > 15) { this.body = null; this.lms = null; this.lmsRaw = null; this._lmsS = null; }
       return;
     }
     this._bodyMiss = 0;
@@ -352,6 +352,14 @@ class InteractiveSim {
       vis: l.visibility !== undefined ? l.visibility : 1 });
     const lms = raw.map(map);
     this.lms = lms;
+    // Mirrored raw landmarks with depth, smoothed — the avatar's skeleton.
+    const rawM = raw.map(l => ({ x: 1 - l.x, y: l.y, z: l.z || 0,
+      vis: l.visibility !== undefined ? l.visibility : 1 }));
+    if (!this._lmsS) this._lmsS = rawM;
+    else this._lmsS = this._lmsS.map((p, i) => ({
+      x: p.x + (rawM[i].x - p.x)*0.5, y: p.y + (rawM[i].y - p.y)*0.5,
+      z: p.z + (rawM[i].z - p.z)*0.4, vis: rawM[i].vis }));
+    this.lmsRaw = this._lmsS;
     const KEYS = { head: 0, handL: 16, handR: 15, footL: 28, footR: 27 }; // mirrored L/R
     if (!this.body) this.body = {};
     for (const k in KEYS) {
@@ -1015,6 +1023,61 @@ class InteractiveSim {
 
     if (mode === 'fluid') { this._stepFluid(timeSec, audio, e, canvas); return; }
 
+    // ---- Robot avatar: a robot built from 3D parts whose limbs follow your
+    // tracked skeleton 1:1 — raise your arm and the robot raises its arm.
+    if (mode === 'avatar') {
+      const L = this.lmsRaw;
+      const ok = this.modelSim && L && L[11] && L[11].vis > 0.5 && L[12].vis > 0.5
+        && L[23].vis > 0.4 && L[24].vis > 0.4;
+      if (!ok) { this._drawField(7, e, audio, canvas, 0); return; }
+      const S = 2.6;
+      const W = (i) => [(L[i].x - 0.5)*S, (0.5 - L[i].y)*S, -(L[i].z || 0)*1.2];
+      const mid = (p, q) => [(p[0]+q[0])/2, (p[1]+q[1])/2, (p[2]+q[2])/2];
+      const sh1 = W(11), sh2 = W(12), hp1 = W(23), hp2 = W(24);
+      const neck = mid(sh1, sh2), pelvis = mid(hp1, hp2);
+      const shw = Math.max(0.15, Math.hypot(sh1[0]-sh2[0], sh1[1]-sh2[1], sh1[2]-sh2[2]));
+      const t = shw;
+      const metal = [0.60, 0.64, 0.72], dark = [0.28, 0.30, 0.36];
+      const acc = e.colorB || [0.2, 1, 1];
+      const parts = [];
+      const limb = (a, b, r, col) => parts.push({ a, b, rx: r, rz: r, kind: 'cube', col });
+      const ball = (a, r, col) => parts.push({ a, b: null, rx: r, kind: 'sphere', col });
+      // torso + chest light
+      parts.push({ a: neck, b: pelvis, rx: shw*0.52, rz: shw*0.28, kind: 'cube', col: metal });
+      ball(mid(neck, mid(neck, pelvis)), t*0.10, acc);
+      // head with visor + antenna
+      const nose = L[0].vis > 0.4 ? W(0) : [neck[0], neck[1]+t*0.55, neck[2]];
+      const headC = [nose[0], nose[1] + t*0.05, nose[2]];
+      parts.push({ a: headC, b: null, rx: t*0.34, ry: t*0.38, rz: t*0.30, kind: 'cube', col: metal });
+      parts.push({ a: [headC[0], headC[1]+t*0.02, headC[2]+t*0.30], b: null,
+        rx: t*0.24, ry: t*0.08, rz: t*0.06, kind: 'cube', col: acc });
+      ball([headC[0], headC[1]+t*0.52, headC[2]], t*0.07, acc);
+      // arms (shoulder->elbow->wrist), glowing hands
+      const arm = (s, elI, wrI) => {
+        if (L[elI].vis < 0.4 || L[wrI].vis < 0.4) return;
+        const el = W(elI), wr = W(wrI);
+        limb(s, el, t*0.14, metal);
+        limb(el, wr, t*0.12, metal);
+        ball(el, t*0.15, dark);
+        ball(wr, t*0.17, acc);
+      };
+      arm(sh1, 13, 15); arm(sh2, 14, 16);
+      ball(sh1, t*0.16, dark); ball(sh2, t*0.16, dark);
+      // legs with feet
+      const leg = (h, knI, anI) => {
+        if (L[knI].vis < 0.4 || L[anI].vis < 0.4) return;
+        const kn = W(knI), an = W(anI);
+        limb(h, kn, t*0.16, metal);
+        limb(kn, an, t*0.13, metal);
+        ball(kn, t*0.16, dark);
+        parts.push({ a: [an[0], an[1]-t*0.06, an[2]+t*0.12], b: null,
+          rx: t*0.14, ry: t*0.08, rz: t*0.22, kind: 'cube', col: dark });
+      };
+      leg(hp1, 25, 27); leg(hp2, 26, 28);
+      this.modelSim.renderAvatar(timeSec, audio, e, canvas, parts);
+      return;
+    }
+
     // ---- 3D model puppet: swipe to spin (with inertia), the model leans
     // toward where you move, big motion makes it hop with squash & stretch.
     if (mode === 'model') {

src/model3d.js

@@ -358,6 +358,137 @@ class ModelSim {
     }
   }
 
+  // ---------------------------------------------------- avatar primitives
+  _primMesh(pos, nrm, idx) {
+    const gl = this.gl;
+    const mk = (d, target) => {
+      const b = gl.createBuffer();
+      gl.bindBuffer(target, b);
+      gl.bufferData(target, d, gl.STATIC_DRAW);
+      return b;
+    };
+    return { vboP: mk(new Float32Array(pos), gl.ARRAY_BUFFER),
+      vboN: mk(new Float32Array(nrm), gl.ARRAY_BUFFER),
+      vboU: mk(new Float32Array(pos.length/3*2), gl.ARRAY_BUFFER),
+      ibo: mk(new Uint16Array(idx), gl.ELEMENT_ARRAY_BUFFER),
+      count: idx.length, idxType: 5123 };
+  }
+
+  _ensurePrims() {
+    if (this.prims) return;
+    // unit cube (±1) with face normals
+    const P = [], N = [], I = [];
+    const faces = [[[1,0,0],[0,1,0],[0,0,1]], [[-1,0,0],[0,0,1],[0,1,0]],
+      [[0,1,0],[0,0,1],[1,0,0]], [[0,-1,0],[1,0,0],[0,0,1]],
+      [[0,0,1],[1,0,0],[0,1,0]], [[0,0,-1],[0,1,0],[1,0,0]]];
+    faces.forEach(([n, u, v]) => {
+      const b = P.length/3;
+      for (const [su, sv] of [[-1,-1],[1,-1],[1,1],[-1,1]]) {
+        P.push(n[0]+u[0]*su+v[0]*sv, n[1]+u[1]*su+v[1]*sv, n[2]+u[2]*su+v[2]*sv);
+        N.push(n[0], n[1], n[2]);
+      }
+      I.push(b, b+1, b+2, b, b+2, b+3);
+    });
+    const cube = this._primMesh(P, N, I);
+    // unit sphere
+    const SP = [], SN = [], SI = [], ST = 12, SE = 18;
+    for (let i = 0; i <= ST; i++) {
+      const ph = i/ST*Math.PI, y = Math.cos(ph), r = Math.sin(ph);
+      for (let j = 0; j <= SE; j++) {
+        const th = j/SE*Math.PI*2;
+        const x = r*Math.cos(th), z = r*Math.sin(th);
+        SP.push(x, y, z); SN.push(x, y, z);
+        if (i < ST && j < SE) {
+          const a = i*(SE+1)+j;
+          SI.push(a, a+SE+1, a+1, a+1, a+SE+1, a+SE+2);
+        }
+      }
+    }
+    this.prims = { cube, sphere: this._primMesh(SP, SN, SI) };
+  }
+
+  // Matrix placing a unit primitive as a limb from a to b (thickness rx/rz)
+  // or as a sphere/box at a (b = null).
+  _partMatrix(spec) {
+    if (!spec.b) return m4mul(m4trans(spec.a), m4scale3(spec.rx, spec.ry || spec.rx, spec.rz || spec.rx));
+    const dx = spec.b[0]-spec.a[0], dy = spec.b[1]-spec.a[1], dz = spec.b[2]-spec.a[2];
+    const len = Math.max(1e-4, Math.hypot(dx, dy, dz));
+    const yx = dx/len, yy = dy/len, yz = dz/len;
+    // basis around the limb axis
+    let ax = 0, ay = 0, az = 1;
+    if (Math.abs(yz) > 0.9) { ax = 1; az = 0; }
+    let xx = yy*az - yz*ay, xy = yz*ax - yx*az, xz = yx*ay - yy*ax;
+    const xl = Math.hypot(xx, xy, xz) || 1; xx/=xl; xy/=xl; xz/=xl;
+    const zx = xy*yz - xz*yy, zy = xz*yx - xx*yz, zz = xx*yy - xy*yx;
+    const R = new Float32Array([xx,xy,xz,0, yx,yy,yz,0, zx,zy,zz,0,
+      (spec.a[0]+spec.b[0])/2, (spec.a[1]+spec.b[1])/2, (spec.a[2]+spec.b[2])/2, 1]);
+    return m4mul(R, m4scale3(spec.rx, len/2, spec.rz || spec.rx));
+  }
+
+  // Render a robot avatar made of primitive parts driven by the body pose.
+  // parts: [{ a:[x,y,z], b:[x,y,z]|null, rx, ry?, rz?, kind:'cube'|'sphere', col:[r,g,b] }]
+  renderAvatar(timeSec, audio, e, canvas, parts) {
+    const gl = this.gl;
+    this._ensurePrims();
+    const mix = e.audioMix !== undefined ? e.audioMix : 1;
+    const bass = (audio.bass || 0)*mix, beat = (audio.beat || 0)*mix;
+    const ca = e.colorA || [0.05, 0, 0.2], cb = e.colorB || [0.2, 1, 1];
+
+    gl.viewport(0, 0, canvas.width, canvas.height);
+    gl.disable(gl.BLEND);
+    gl.clearColor(0, 0, 0, 1);
+    gl.clear(gl.COLOR_BUFFER_BIT | gl.DEPTH_BUFFER_BIT);
+
+    gl.useProgram(this.progBg);
+    gl.bindBuffer(gl.ARRAY_BUFFER, this.quad);
+    gl.enableVertexAttribArray(this.aBg);
+    gl.vertexAttribPointer(this.aBg, 2, gl.FLOAT, false, 0, 0);
+    gl.uniform3fv(this.ub.uColA, ca);
+    gl.uniform3fv(this.ub.uColB, cb);
+    gl.uniform2f(this.ub.uRes, canvas.width, canvas.height);
+    gl.uniform1f(this.ub.uT, timeSec);
+    gl.uniform1f(this.ub.uBass, bass);
+    gl.uniform1f(this.ub.uBeat, beat);
+    gl.drawArrays(gl.TRIANGLES, 0, 3);
+
+    const asp = canvas.width/Math.max(1, canvas.height);
+    const eye = [0, 0.1, 3.4];
+    const proj = m4persp(0.72, asp, 0.1, 20);
+    const view = m4lookAt(eye, [0, 0, 0]);
+    gl.enable(gl.DEPTH_TEST);
+    gl.depthFunc(gl.LEQUAL);
+    gl.useProgram(this.progMesh);
+    gl.uniformMatrix4fv(this.um.uProj, false, proj);
+    gl.uniformMatrix4fv(this.um.uView, false, view);
+    gl.uniform1f(this.um.uPulse, 0);
+    gl.uniform3fv(this.um.uColA, ca);
+    gl.uniform3fv(this.um.uColB, cb);
+    gl.uniform3fv(this.um.uCam, eye);
+    gl.uniform1f(this.um.uBeat, beat);
+    gl.uniform1f(this.um.uLevel, (audio.level || 0)*mix);
+    gl.uniform1f(this.um.uTreble, (audio.treble || 0)*mix);
+    gl.uniform1f(this.um.uRim, 0.15 + 0.3*beat);
+    gl.uniform1i(this.um.uHasTex, 0);
+    for (const spec of parts) {
+      const m = this.prims[spec.kind === 'sphere' ? 'sphere' : 'cube'];
+      gl.uniformMatrix4fv(this.um.uModel, false, this._partMatrix(spec));
+      gl.uniform3fv(this.um.uBase, spec.col);
+      const bind = (buf, loc, n) => {
+        gl.bindBuffer(gl.ARRAY_BUFFER, buf);
+        gl.enableVertexAttribArray(loc);
+        gl.vertexAttribPointer(loc, n, gl.FLOAT, false, 0, 0);
+      };
+      bind(m.vboP, this.aMesh.pos, 3);
+      bind(m.vboN, this.aMesh.nrm, 3);
+      bind(m.vboU, this.aMesh.uv, 2);
+      gl.bindBuffer(gl.ELEMENT_ARRAY_BUFFER, m.ibo);
+      gl.drawElements(gl.TRIANGLES, m.count, gl.UNSIGNED_SHORT, 0);
+    }
+    gl.disable(gl.DEPTH_TEST);
+    gl.disableVertexAttribArray(this.aMesh.nrm);
+    gl.disableVertexAttribArray(this.aMesh.uv);
+  }
+
   // pose (optional, from the camera-interactive mode): { yaw, leanX, hopY,
   // squash, rim } — extra rotation, sideways lean, jump height (in radii),
   // vertical squash & stretch, touch-glow 0..1.

src/visualizer.js

@@ -120,8 +120,8 @@ class Visualizer {
     if (e.isInteractive && window.InteractiveSim) {
       if (!this.inter) this.inter = new window.InteractiveSim(gl);
       this.inter.customSource = this.customSource || null;
-      // The model-puppet mode drives the shared 3D renderer.
+      // The model-puppet and avatar modes drive the shared 3D renderer.
-      if (e.interactiveMode === 'model' && window.ModelSim) {
+      if ((e.interactiveMode === 'model' || e.interactiveMode === 'avatar') && window.ModelSim) {
         if (!this.model3d) this.model3d = new window.ModelSim(gl);
         if (this._pendingGlb) { this.model3d.setModel(this._pendingGlb); this._pendingGlb = null; }
         this.inter.modelSim = this.model3d;