From 08414070a9aedad0e54bcddc64cc67cbe1c1bfe3 Mon Sep 17 00:00:00 2001 From: luciano Date: Thu, 30 Jul 2026 18:39:24 +0200 Subject: [PATCH] Interattivo Avatar: a robot whose limbs mirror your tracked body 1:1 Raise your arm and the robot raises its arm. ModelSim gains primitive meshes (cube/sphere) and renderAvatar(): parts placed by limb matrices (basis aligned to each bone segment) with the same lighting as the GLB renderer. interactive.js keeps the mirrored raw landmarks (with depth, smoothed) and builds the robot every frame: torso sized from the shoulders, head with accent visor and antenna, articulated arms/legs with joint spheres, glowing hands, feet. Falls back to the motion mist when no body is tracked. Co-Authored-By: Claude Fable 5 --- src/effects.js | 5 +- src/interactive.js | 65 +++++++++++++++++++++- src/model3d.js | 131 +++++++++++++++++++++++++++++++++++++++++++++ src/visualizer.js | 4 +- 4 files changed, 201 insertions(+), 4 deletions(-) diff --git a/src/effects.js b/src/effects.js index e62d507..f98696b 100644 --- a/src/effects.js +++ b/src/effects.js @@ -79,7 +79,10 @@ const FAMILIES = [ { name: 'Modello 3D', scale: 1.0, model3d: true }, // Camera-interactive puppet: swipe to spin the model, it leans toward your // movement and hops when you get agitated. Excluded from Auto VJ. - { name: 'Interattivo Modello', scale: 1.0, interactive: true, interMode: 'model' } + { name: 'Interattivo Modello', scale: 1.0, interactive: true, interMode: 'model' }, + // Robot avatar: a robot made of 3D parts whose limbs mirror the tracked + // body 1:1 (needs the pose tracker; shows motion mist without it). + { name: 'Interattivo Avatar', scale: 1.0, interactive: true, interMode: 'avatar' } ]; // Palettes: low colour (a) -> high colour (b), plus optional hue-cycle/sat. diff --git a/src/interactive.js b/src/interactive.js index 4075739..bf14078 100644 --- a/src/interactive.js +++ b/src/interactive.js @@ -343,7 +343,7 @@ class InteractiveSim { let raw = null; try { raw = this.pose.detect(v, nowMs); } catch (e) { return; } if (!raw) { - if (++this._bodyMiss > 15) { this.body = null; this.lms = null; } + if (++this._bodyMiss > 15) { this.body = null; this.lms = null; this.lmsRaw = null; this._lmsS = null; } return; } this._bodyMiss = 0; @@ -352,6 +352,14 @@ class InteractiveSim { vis: l.visibility !== undefined ? l.visibility : 1 }); const lms = raw.map(map); this.lms = lms; + // Mirrored raw landmarks with depth, smoothed — the avatar's skeleton. + const rawM = raw.map(l => ({ x: 1 - l.x, y: l.y, z: l.z || 0, + vis: l.visibility !== undefined ? l.visibility : 1 })); + if (!this._lmsS) this._lmsS = rawM; + else this._lmsS = this._lmsS.map((p, i) => ({ + x: p.x + (rawM[i].x - p.x)*0.5, y: p.y + (rawM[i].y - p.y)*0.5, + z: p.z + (rawM[i].z - p.z)*0.4, vis: rawM[i].vis })); + this.lmsRaw = this._lmsS; const KEYS = { head: 0, handL: 16, handR: 15, footL: 28, footR: 27 }; // mirrored L/R if (!this.body) this.body = {}; for (const k in KEYS) { @@ -1015,6 +1023,61 @@ class InteractiveSim { if (mode === 'fluid') { this._stepFluid(timeSec, audio, e, canvas); return; } + // ---- Robot avatar: a robot built from 3D parts whose limbs follow your + // tracked skeleton 1:1 — raise your arm and the robot raises its arm. + if (mode === 'avatar') { + const L = this.lmsRaw; + const ok = this.modelSim && L && L[11] && L[11].vis > 0.5 && L[12].vis > 0.5 + && L[23].vis > 0.4 && L[24].vis > 0.4; + if (!ok) { this._drawField(7, e, audio, canvas, 0); return; } + const S = 2.6; + const W = (i) => [(L[i].x - 0.5)*S, (0.5 - L[i].y)*S, -(L[i].z || 0)*1.2]; + const mid = (p, q) => [(p[0]+q[0])/2, (p[1]+q[1])/2, (p[2]+q[2])/2]; + const sh1 = W(11), sh2 = W(12), hp1 = W(23), hp2 = W(24); + const neck = mid(sh1, sh2), pelvis = mid(hp1, hp2); + const shw = Math.max(0.15, Math.hypot(sh1[0]-sh2[0], sh1[1]-sh2[1], sh1[2]-sh2[2])); + const t = shw; + const metal = [0.60, 0.64, 0.72], dark = [0.28, 0.30, 0.36]; + const acc = e.colorB || [0.2, 1, 1]; + const parts = []; + const limb = (a, b, r, col) => parts.push({ a, b, rx: r, rz: r, kind: 'cube', col }); + const ball = (a, r, col) => parts.push({ a, b: null, rx: r, kind: 'sphere', col }); + // torso + chest light + parts.push({ a: neck, b: pelvis, rx: shw*0.52, rz: shw*0.28, kind: 'cube', col: metal }); + ball(mid(neck, mid(neck, pelvis)), t*0.10, acc); + // head with visor + antenna + const nose = L[0].vis > 0.4 ? W(0) : [neck[0], neck[1]+t*0.55, neck[2]]; + const headC = [nose[0], nose[1] + t*0.05, nose[2]]; + parts.push({ a: headC, b: null, rx: t*0.34, ry: t*0.38, rz: t*0.30, kind: 'cube', col: metal }); + parts.push({ a: [headC[0], headC[1]+t*0.02, headC[2]+t*0.30], b: null, + rx: t*0.24, ry: t*0.08, rz: t*0.06, kind: 'cube', col: acc }); + ball([headC[0], headC[1]+t*0.52, headC[2]], t*0.07, acc); + // arms (shoulder->elbow->wrist), glowing hands + const arm = (s, elI, wrI) => { + if (L[elI].vis < 0.4 || L[wrI].vis < 0.4) return; + const el = W(elI), wr = W(wrI); + limb(s, el, t*0.14, metal); + limb(el, wr, t*0.12, metal); + ball(el, t*0.15, dark); + ball(wr, t*0.17, acc); + }; + arm(sh1, 13, 15); arm(sh2, 14, 16); + ball(sh1, t*0.16, dark); ball(sh2, t*0.16, dark); + // legs with feet + const leg = (h, knI, anI) => { + if (L[knI].vis < 0.4 || L[anI].vis < 0.4) return; + const kn = W(knI), an = W(anI); + limb(h, kn, t*0.16, metal); + limb(kn, an, t*0.13, metal); + ball(kn, t*0.16, dark); + parts.push({ a: [an[0], an[1]-t*0.06, an[2]+t*0.12], b: null, + rx: t*0.14, ry: t*0.08, rz: t*0.22, kind: 'cube', col: dark }); + }; + leg(hp1, 25, 27); leg(hp2, 26, 28); + this.modelSim.renderAvatar(timeSec, audio, e, canvas, parts); + return; + } + // ---- 3D model puppet: swipe to spin (with inertia), the model leans // toward where you move, big motion makes it hop with squash & stretch. if (mode === 'model') { diff --git a/src/model3d.js b/src/model3d.js index 9c4404b..fe8711f 100644 --- a/src/model3d.js +++ b/src/model3d.js @@ -358,6 +358,137 @@ class ModelSim { } } + // ---------------------------------------------------- avatar primitives + _primMesh(pos, nrm, idx) { + const gl = this.gl; + const mk = (d, target) => { + const b = gl.createBuffer(); + gl.bindBuffer(target, b); + gl.bufferData(target, d, gl.STATIC_DRAW); + return b; + }; + return { vboP: mk(new Float32Array(pos), gl.ARRAY_BUFFER), + vboN: mk(new Float32Array(nrm), gl.ARRAY_BUFFER), + vboU: mk(new Float32Array(pos.length/3*2), gl.ARRAY_BUFFER), + ibo: mk(new Uint16Array(idx), gl.ELEMENT_ARRAY_BUFFER), + count: idx.length, idxType: 5123 }; + } + + _ensurePrims() { + if (this.prims) return; + // unit cube (±1) with face normals + const P = [], N = [], I = []; + const faces = [[[1,0,0],[0,1,0],[0,0,1]], [[-1,0,0],[0,0,1],[0,1,0]], + [[0,1,0],[0,0,1],[1,0,0]], [[0,-1,0],[1,0,0],[0,0,1]], + [[0,0,1],[1,0,0],[0,1,0]], [[0,0,-1],[0,1,0],[1,0,0]]]; + faces.forEach(([n, u, v]) => { + const b = P.length/3; + for (const [su, sv] of [[-1,-1],[1,-1],[1,1],[-1,1]]) { + P.push(n[0]+u[0]*su+v[0]*sv, n[1]+u[1]*su+v[1]*sv, n[2]+u[2]*su+v[2]*sv); + N.push(n[0], n[1], n[2]); + } + I.push(b, b+1, b+2, b, b+2, b+3); + }); + const cube = this._primMesh(P, N, I); + // unit sphere + const SP = [], SN = [], SI = [], ST = 12, SE = 18; + for (let i = 0; i <= ST; i++) { + const ph = i/ST*Math.PI, y = Math.cos(ph), r = Math.sin(ph); + for (let j = 0; j <= SE; j++) { + const th = j/SE*Math.PI*2; + const x = r*Math.cos(th), z = r*Math.sin(th); + SP.push(x, y, z); SN.push(x, y, z); + if (i < ST && j < SE) { + const a = i*(SE+1)+j; + SI.push(a, a+SE+1, a+1, a+1, a+SE+1, a+SE+2); + } + } + } + this.prims = { cube, sphere: this._primMesh(SP, SN, SI) }; + } + + // Matrix placing a unit primitive as a limb from a to b (thickness rx/rz) + // or as a sphere/box at a (b = null). + _partMatrix(spec) { + if (!spec.b) return m4mul(m4trans(spec.a), m4scale3(spec.rx, spec.ry || spec.rx, spec.rz || spec.rx)); + const dx = spec.b[0]-spec.a[0], dy = spec.b[1]-spec.a[1], dz = spec.b[2]-spec.a[2]; + const len = Math.max(1e-4, Math.hypot(dx, dy, dz)); + const yx = dx/len, yy = dy/len, yz = dz/len; + // basis around the limb axis + let ax = 0, ay = 0, az = 1; + if (Math.abs(yz) > 0.9) { ax = 1; az = 0; } + let xx = yy*az - yz*ay, xy = yz*ax - yx*az, xz = yx*ay - yy*ax; + const xl = Math.hypot(xx, xy, xz) || 1; xx/=xl; xy/=xl; xz/=xl; + const zx = xy*yz - xz*yy, zy = xz*yx - xx*yz, zz = xx*yy - xy*yx; + const R = new Float32Array([xx,xy,xz,0, yx,yy,yz,0, zx,zy,zz,0, + (spec.a[0]+spec.b[0])/2, (spec.a[1]+spec.b[1])/2, (spec.a[2]+spec.b[2])/2, 1]); + return m4mul(R, m4scale3(spec.rx, len/2, spec.rz || spec.rx)); + } + + // Render a robot avatar made of primitive parts driven by the body pose. + // parts: [{ a:[x,y,z], b:[x,y,z]|null, rx, ry?, rz?, kind:'cube'|'sphere', col:[r,g,b] }] + renderAvatar(timeSec, audio, e, canvas, parts) { + const gl = this.gl; + this._ensurePrims(); + const mix = e.audioMix !== undefined ? e.audioMix : 1; + const bass = (audio.bass || 0)*mix, beat = (audio.beat || 0)*mix; + const ca = e.colorA || [0.05, 0, 0.2], cb = e.colorB || [0.2, 1, 1]; + + gl.viewport(0, 0, canvas.width, canvas.height); + gl.disable(gl.BLEND); + gl.clearColor(0, 0, 0, 1); + gl.clear(gl.COLOR_BUFFER_BIT | gl.DEPTH_BUFFER_BIT); + + gl.useProgram(this.progBg); + gl.bindBuffer(gl.ARRAY_BUFFER, this.quad); + gl.enableVertexAttribArray(this.aBg); + gl.vertexAttribPointer(this.aBg, 2, gl.FLOAT, false, 0, 0); + gl.uniform3fv(this.ub.uColA, ca); + gl.uniform3fv(this.ub.uColB, cb); + gl.uniform2f(this.ub.uRes, canvas.width, canvas.height); + gl.uniform1f(this.ub.uT, timeSec); + gl.uniform1f(this.ub.uBass, bass); + gl.uniform1f(this.ub.uBeat, beat); + gl.drawArrays(gl.TRIANGLES, 0, 3); + + const asp = canvas.width/Math.max(1, canvas.height); + const eye = [0, 0.1, 3.4]; + const proj = m4persp(0.72, asp, 0.1, 20); + const view = m4lookAt(eye, [0, 0, 0]); + gl.enable(gl.DEPTH_TEST); + gl.depthFunc(gl.LEQUAL); + gl.useProgram(this.progMesh); + gl.uniformMatrix4fv(this.um.uProj, false, proj); + gl.uniformMatrix4fv(this.um.uView, false, view); + gl.uniform1f(this.um.uPulse, 0); + gl.uniform3fv(this.um.uColA, ca); + gl.uniform3fv(this.um.uColB, cb); + gl.uniform3fv(this.um.uCam, eye); + gl.uniform1f(this.um.uBeat, beat); + gl.uniform1f(this.um.uLevel, (audio.level || 0)*mix); + gl.uniform1f(this.um.uTreble, (audio.treble || 0)*mix); + gl.uniform1f(this.um.uRim, 0.15 + 0.3*beat); + gl.uniform1i(this.um.uHasTex, 0); + for (const spec of parts) { + const m = this.prims[spec.kind === 'sphere' ? 'sphere' : 'cube']; + gl.uniformMatrix4fv(this.um.uModel, false, this._partMatrix(spec)); + gl.uniform3fv(this.um.uBase, spec.col); + const bind = (buf, loc, n) => { + gl.bindBuffer(gl.ARRAY_BUFFER, buf); + gl.enableVertexAttribArray(loc); + gl.vertexAttribPointer(loc, n, gl.FLOAT, false, 0, 0); + }; + bind(m.vboP, this.aMesh.pos, 3); + bind(m.vboN, this.aMesh.nrm, 3); + bind(m.vboU, this.aMesh.uv, 2); + gl.bindBuffer(gl.ELEMENT_ARRAY_BUFFER, m.ibo); + gl.drawElements(gl.TRIANGLES, m.count, gl.UNSIGNED_SHORT, 0); + } + gl.disable(gl.DEPTH_TEST); + gl.disableVertexAttribArray(this.aMesh.nrm); + gl.disableVertexAttribArray(this.aMesh.uv); + } + // pose (optional, from the camera-interactive mode): { yaw, leanX, hopY, // squash, rim } — extra rotation, sideways lean, jump height (in radii), // vertical squash & stretch, touch-glow 0..1. diff --git a/src/visualizer.js b/src/visualizer.js index 9aeb3f3..940cdfc 100644 --- a/src/visualizer.js +++ b/src/visualizer.js @@ -120,8 +120,8 @@ class Visualizer { if (e.isInteractive && window.InteractiveSim) { if (!this.inter) this.inter = new window.InteractiveSim(gl); this.inter.customSource = this.customSource || null; - // The model-puppet mode drives the shared 3D renderer. - if (e.interactiveMode === 'model' && window.ModelSim) { + // The model-puppet and avatar modes drive the shared 3D renderer. + if ((e.interactiveMode === 'model' || e.interactiveMode === 'avatar') && window.ModelSim) { if (!this.model3d) this.model3d = new window.ModelSim(gl); if (this._pendingGlb) { this.model3d.setModel(this._pendingGlb); this._pendingGlb = null; } this.inter.modelSim = this.model3d;