diff --git a/src/effects.js b/src/effects.js index e62d507..f98696b 100644 --- a/src/effects.js +++ b/src/effects.js @@ -79,7 +79,10 @@ const FAMILIES = [ { name: 'Modello 3D', scale: 1.0, model3d: true }, // Camera-interactive puppet: swipe to spin the model, it leans toward your // movement and hops when you get agitated. Excluded from Auto VJ. - { name: 'Interattivo Modello', scale: 1.0, interactive: true, interMode: 'model' } + { name: 'Interattivo Modello', scale: 1.0, interactive: true, interMode: 'model' }, + // Robot avatar: a robot made of 3D parts whose limbs mirror the tracked + // body 1:1 (needs the pose tracker; shows motion mist without it). + { name: 'Interattivo Avatar', scale: 1.0, interactive: true, interMode: 'avatar' } ]; // Palettes: low colour (a) -> high colour (b), plus optional hue-cycle/sat. diff --git a/src/interactive.js b/src/interactive.js index 4075739..bf14078 100644 --- a/src/interactive.js +++ b/src/interactive.js @@ -343,7 +343,7 @@ class InteractiveSim { let raw = null; try { raw = this.pose.detect(v, nowMs); } catch (e) { return; } if (!raw) { - if (++this._bodyMiss > 15) { this.body = null; this.lms = null; } + if (++this._bodyMiss > 15) { this.body = null; this.lms = null; this.lmsRaw = null; this._lmsS = null; } return; } this._bodyMiss = 0; @@ -352,6 +352,14 @@ class InteractiveSim { vis: l.visibility !== undefined ? l.visibility : 1 }); const lms = raw.map(map); this.lms = lms; + // Mirrored raw landmarks with depth, smoothed — the avatar's skeleton. + const rawM = raw.map(l => ({ x: 1 - l.x, y: l.y, z: l.z || 0, + vis: l.visibility !== undefined ? l.visibility : 1 })); + if (!this._lmsS) this._lmsS = rawM; + else this._lmsS = this._lmsS.map((p, i) => ({ + x: p.x + (rawM[i].x - p.x)*0.5, y: p.y + (rawM[i].y - p.y)*0.5, + z: p.z + (rawM[i].z - p.z)*0.4, vis: rawM[i].vis })); + this.lmsRaw = this._lmsS; const KEYS = { head: 0, handL: 16, handR: 15, footL: 28, footR: 27 }; // mirrored L/R if (!this.body) this.body = {}; for (const k in KEYS) { @@ -1015,6 +1023,61 @@ class InteractiveSim { if (mode === 'fluid') { this._stepFluid(timeSec, audio, e, canvas); return; } + // ---- Robot avatar: a robot built from 3D parts whose limbs follow your + // tracked skeleton 1:1 — raise your arm and the robot raises its arm. + if (mode === 'avatar') { + const L = this.lmsRaw; + const ok = this.modelSim && L && L[11] && L[11].vis > 0.5 && L[12].vis > 0.5 + && L[23].vis > 0.4 && L[24].vis > 0.4; + if (!ok) { this._drawField(7, e, audio, canvas, 0); return; } + const S = 2.6; + const W = (i) => [(L[i].x - 0.5)*S, (0.5 - L[i].y)*S, -(L[i].z || 0)*1.2]; + const mid = (p, q) => [(p[0]+q[0])/2, (p[1]+q[1])/2, (p[2]+q[2])/2]; + const sh1 = W(11), sh2 = W(12), hp1 = W(23), hp2 = W(24); + const neck = mid(sh1, sh2), pelvis = mid(hp1, hp2); + const shw = Math.max(0.15, Math.hypot(sh1[0]-sh2[0], sh1[1]-sh2[1], sh1[2]-sh2[2])); + const t = shw; + const metal = [0.60, 0.64, 0.72], dark = [0.28, 0.30, 0.36]; + const acc = e.colorB || [0.2, 1, 1]; + const parts = []; + const limb = (a, b, r, col) => parts.push({ a, b, rx: r, rz: r, kind: 'cube', col }); + const ball = (a, r, col) => parts.push({ a, b: null, rx: r, kind: 'sphere', col }); + // torso + chest light + parts.push({ a: neck, b: pelvis, rx: shw*0.52, rz: shw*0.28, kind: 'cube', col: metal }); + ball(mid(neck, mid(neck, pelvis)), t*0.10, acc); + // head with visor + antenna + const nose = L[0].vis > 0.4 ? W(0) : [neck[0], neck[1]+t*0.55, neck[2]]; + const headC = [nose[0], nose[1] + t*0.05, nose[2]]; + parts.push({ a: headC, b: null, rx: t*0.34, ry: t*0.38, rz: t*0.30, kind: 'cube', col: metal }); + parts.push({ a: [headC[0], headC[1]+t*0.02, headC[2]+t*0.30], b: null, + rx: t*0.24, ry: t*0.08, rz: t*0.06, kind: 'cube', col: acc }); + ball([headC[0], headC[1]+t*0.52, headC[2]], t*0.07, acc); + // arms (shoulder->elbow->wrist), glowing hands + const arm = (s, elI, wrI) => { + if (L[elI].vis < 0.4 || L[wrI].vis < 0.4) return; + const el = W(elI), wr = W(wrI); + limb(s, el, t*0.14, metal); + limb(el, wr, t*0.12, metal); + ball(el, t*0.15, dark); + ball(wr, t*0.17, acc); + }; + arm(sh1, 13, 15); arm(sh2, 14, 16); + ball(sh1, t*0.16, dark); ball(sh2, t*0.16, dark); + // legs with feet + const leg = (h, knI, anI) => { + if (L[knI].vis < 0.4 || L[anI].vis < 0.4) return; + const kn = W(knI), an = W(anI); + limb(h, kn, t*0.16, metal); + limb(kn, an, t*0.13, metal); + ball(kn, t*0.16, dark); + parts.push({ a: [an[0], an[1]-t*0.06, an[2]+t*0.12], b: null, + rx: t*0.14, ry: t*0.08, rz: t*0.22, kind: 'cube', col: dark }); + }; + leg(hp1, 25, 27); leg(hp2, 26, 28); + this.modelSim.renderAvatar(timeSec, audio, e, canvas, parts); + return; + } + // ---- 3D model puppet: swipe to spin (with inertia), the model leans // toward where you move, big motion makes it hop with squash & stretch. if (mode === 'model') { diff --git a/src/model3d.js b/src/model3d.js index 9c4404b..fe8711f 100644 --- a/src/model3d.js +++ b/src/model3d.js @@ -358,6 +358,137 @@ class ModelSim { } } + // ---------------------------------------------------- avatar primitives + _primMesh(pos, nrm, idx) { + const gl = this.gl; + const mk = (d, target) => { + const b = gl.createBuffer(); + gl.bindBuffer(target, b); + gl.bufferData(target, d, gl.STATIC_DRAW); + return b; + }; + return { vboP: mk(new Float32Array(pos), gl.ARRAY_BUFFER), + vboN: mk(new Float32Array(nrm), gl.ARRAY_BUFFER), + vboU: mk(new Float32Array(pos.length/3*2), gl.ARRAY_BUFFER), + ibo: mk(new Uint16Array(idx), gl.ELEMENT_ARRAY_BUFFER), + count: idx.length, idxType: 5123 }; + } + + _ensurePrims() { + if (this.prims) return; + // unit cube (±1) with face normals + const P = [], N = [], I = []; + const faces = [[[1,0,0],[0,1,0],[0,0,1]], [[-1,0,0],[0,0,1],[0,1,0]], + [[0,1,0],[0,0,1],[1,0,0]], [[0,-1,0],[1,0,0],[0,0,1]], + [[0,0,1],[1,0,0],[0,1,0]], [[0,0,-1],[0,1,0],[1,0,0]]]; + faces.forEach(([n, u, v]) => { + const b = P.length/3; + for (const [su, sv] of [[-1,-1],[1,-1],[1,1],[-1,1]]) { + P.push(n[0]+u[0]*su+v[0]*sv, n[1]+u[1]*su+v[1]*sv, n[2]+u[2]*su+v[2]*sv); + N.push(n[0], n[1], n[2]); + } + I.push(b, b+1, b+2, b, b+2, b+3); + }); + const cube = this._primMesh(P, N, I); + // unit sphere + const SP = [], SN = [], SI = [], ST = 12, SE = 18; + for (let i = 0; i <= ST; i++) { + const ph = i/ST*Math.PI, y = Math.cos(ph), r = Math.sin(ph); + for (let j = 0; j <= SE; j++) { + const th = j/SE*Math.PI*2; + const x = r*Math.cos(th), z = r*Math.sin(th); + SP.push(x, y, z); SN.push(x, y, z); + if (i < ST && j < SE) { + const a = i*(SE+1)+j; + SI.push(a, a+SE+1, a+1, a+1, a+SE+1, a+SE+2); + } + } + } + this.prims = { cube, sphere: this._primMesh(SP, SN, SI) }; + } + + // Matrix placing a unit primitive as a limb from a to b (thickness rx/rz) + // or as a sphere/box at a (b = null). + _partMatrix(spec) { + if (!spec.b) return m4mul(m4trans(spec.a), m4scale3(spec.rx, spec.ry || spec.rx, spec.rz || spec.rx)); + const dx = spec.b[0]-spec.a[0], dy = spec.b[1]-spec.a[1], dz = spec.b[2]-spec.a[2]; + const len = Math.max(1e-4, Math.hypot(dx, dy, dz)); + const yx = dx/len, yy = dy/len, yz = dz/len; + // basis around the limb axis + let ax = 0, ay = 0, az = 1; + if (Math.abs(yz) > 0.9) { ax = 1; az = 0; } + let xx = yy*az - yz*ay, xy = yz*ax - yx*az, xz = yx*ay - yy*ax; + const xl = Math.hypot(xx, xy, xz) || 1; xx/=xl; xy/=xl; xz/=xl; + const zx = xy*yz - xz*yy, zy = xz*yx - xx*yz, zz = xx*yy - xy*yx; + const R = new Float32Array([xx,xy,xz,0, yx,yy,yz,0, zx,zy,zz,0, + (spec.a[0]+spec.b[0])/2, (spec.a[1]+spec.b[1])/2, (spec.a[2]+spec.b[2])/2, 1]); + return m4mul(R, m4scale3(spec.rx, len/2, spec.rz || spec.rx)); + } + + // Render a robot avatar made of primitive parts driven by the body pose. + // parts: [{ a:[x,y,z], b:[x,y,z]|null, rx, ry?, rz?, kind:'cube'|'sphere', col:[r,g,b] }] + renderAvatar(timeSec, audio, e, canvas, parts) { + const gl = this.gl; + this._ensurePrims(); + const mix = e.audioMix !== undefined ? e.audioMix : 1; + const bass = (audio.bass || 0)*mix, beat = (audio.beat || 0)*mix; + const ca = e.colorA || [0.05, 0, 0.2], cb = e.colorB || [0.2, 1, 1]; + + gl.viewport(0, 0, canvas.width, canvas.height); + gl.disable(gl.BLEND); + gl.clearColor(0, 0, 0, 1); + gl.clear(gl.COLOR_BUFFER_BIT | gl.DEPTH_BUFFER_BIT); + + gl.useProgram(this.progBg); + gl.bindBuffer(gl.ARRAY_BUFFER, this.quad); + gl.enableVertexAttribArray(this.aBg); + gl.vertexAttribPointer(this.aBg, 2, gl.FLOAT, false, 0, 0); + gl.uniform3fv(this.ub.uColA, ca); + gl.uniform3fv(this.ub.uColB, cb); + gl.uniform2f(this.ub.uRes, canvas.width, canvas.height); + gl.uniform1f(this.ub.uT, timeSec); + gl.uniform1f(this.ub.uBass, bass); + gl.uniform1f(this.ub.uBeat, beat); + gl.drawArrays(gl.TRIANGLES, 0, 3); + + const asp = canvas.width/Math.max(1, canvas.height); + const eye = [0, 0.1, 3.4]; + const proj = m4persp(0.72, asp, 0.1, 20); + const view = m4lookAt(eye, [0, 0, 0]); + gl.enable(gl.DEPTH_TEST); + gl.depthFunc(gl.LEQUAL); + gl.useProgram(this.progMesh); + gl.uniformMatrix4fv(this.um.uProj, false, proj); + gl.uniformMatrix4fv(this.um.uView, false, view); + gl.uniform1f(this.um.uPulse, 0); + gl.uniform3fv(this.um.uColA, ca); + gl.uniform3fv(this.um.uColB, cb); + gl.uniform3fv(this.um.uCam, eye); + gl.uniform1f(this.um.uBeat, beat); + gl.uniform1f(this.um.uLevel, (audio.level || 0)*mix); + gl.uniform1f(this.um.uTreble, (audio.treble || 0)*mix); + gl.uniform1f(this.um.uRim, 0.15 + 0.3*beat); + gl.uniform1i(this.um.uHasTex, 0); + for (const spec of parts) { + const m = this.prims[spec.kind === 'sphere' ? 'sphere' : 'cube']; + gl.uniformMatrix4fv(this.um.uModel, false, this._partMatrix(spec)); + gl.uniform3fv(this.um.uBase, spec.col); + const bind = (buf, loc, n) => { + gl.bindBuffer(gl.ARRAY_BUFFER, buf); + gl.enableVertexAttribArray(loc); + gl.vertexAttribPointer(loc, n, gl.FLOAT, false, 0, 0); + }; + bind(m.vboP, this.aMesh.pos, 3); + bind(m.vboN, this.aMesh.nrm, 3); + bind(m.vboU, this.aMesh.uv, 2); + gl.bindBuffer(gl.ELEMENT_ARRAY_BUFFER, m.ibo); + gl.drawElements(gl.TRIANGLES, m.count, gl.UNSIGNED_SHORT, 0); + } + gl.disable(gl.DEPTH_TEST); + gl.disableVertexAttribArray(this.aMesh.nrm); + gl.disableVertexAttribArray(this.aMesh.uv); + } + // pose (optional, from the camera-interactive mode): { yaw, leanX, hopY, // squash, rim } — extra rotation, sideways lean, jump height (in radii), // vertical squash & stretch, touch-glow 0..1. diff --git a/src/visualizer.js b/src/visualizer.js index 9aeb3f3..940cdfc 100644 --- a/src/visualizer.js +++ b/src/visualizer.js @@ -120,8 +120,8 @@ class Visualizer { if (e.isInteractive && window.InteractiveSim) { if (!this.inter) this.inter = new window.InteractiveSim(gl); this.inter.customSource = this.customSource || null; - // The model-puppet mode drives the shared 3D renderer. - if (e.interactiveMode === 'model' && window.ModelSim) { + // The model-puppet and avatar modes drive the shared 3D renderer. + if ((e.interactiveMode === 'model' || e.interactiveMode === 'avatar') && window.ModelSim) { if (!this.model3d) this.model3d = new window.ModelSim(gl); if (this._pendingGlb) { this.model3d.setModel(this._pendingGlb); this._pendingGlb = null; } this.inter.modelSim = this.model3d;