diff --git a/main.js b/main.js index 5b86af4..a59accc 100644 --- a/main.js +++ b/main.js @@ -108,6 +108,34 @@ async function ensureMicAccess() { } catch (e) { /* getUserMedia will prompt again if needed */ } } +// Convert a Mixamo FBX animation into a skeleton-only GLB clip via Blender +// (when installed): lets the user add moves to the dance library from the +// panel without any manual conversion step. +function blenderPath() { + const cands = ['/Applications/Blender.app/Contents/MacOS/Blender', + '/opt/homebrew/bin/blender', '/usr/local/bin/blender']; + for (const c of cands) if (fs.existsSync(c)) return c; + return null; +} +ipcMain.handle('anim:convert', async (_e, fbxPath) => { + const blender = blenderPath(); + if (!blender) return { ok: false, error: 'Blender non trovato: serve per convertire gli FBX (oppure carica un .glb)' }; + const outDir = path.join(app.getPath('userData'), 'anims'); + fs.mkdirSync(outDir, { recursive: true }); + const out = path.join(outDir, path.basename(fbxPath).replace(/\.fbx$/i, '') + '.glb'); + // packaged app: the .py must live outside the asar for Blender to read it + const script = path.join(__dirname, 'scripts', 'fbx2animglb.py') + .replace('app.asar' + path.sep, 'app.asar.unpacked' + path.sep); + return new Promise((resolve) => { + execFile(blender, ['-b', '-P', script, '--', fbxPath, out], + { maxBuffer: 1 << 24, timeout: 180000 }, (err, _so, se) => { + if (err || !fs.existsSync(out)) + resolve({ ok: false, error: 'conversione fallita: ' + String(se || (err && err.message) || '').slice(-300) }); + else resolve({ ok: true, path: out }); + }); + }); +}); + // Camera permission is requested lazily (only when an interactive preset is // activated), not at startup: no camera prompt for users who never use it. ipcMain.handle('cam:ensure', async () => { diff --git a/models/dance_library.glb b/models/dance_library.glb new file mode 100644 index 0000000..569fe7c Binary files /dev/null and b/models/dance_library.glb differ diff --git a/package.json b/package.json index 60d4049..ecaed1c 100644 --- a/package.json +++ b/package.json @@ -27,7 +27,11 @@ "preload.js", "src/**/*", "svg/**/*", - "models/**/*" + "models/**/*", + "scripts/fbx2animglb.py" + ], + "asarUnpack": [ + "scripts/fbx2animglb.py" ] }, "devDependencies": { diff --git a/preload.js b/preload.js index aa7d525..488e372 100644 --- a/preload.js +++ b/preload.js @@ -38,5 +38,7 @@ contextBridge.exposeInMainWorld('djv', { peaks: (path, buckets) => ipcRenderer.invoke('audio:peaks', path, buckets), // Ask macOS for camera permission (interactive family) right before use. camAccess: () => ipcRenderer.invoke('cam:ensure'), + // Convert a Mixamo FBX animation to a library GLB clip (via Blender). + convertAnim: (p) => ipcRenderer.invoke('anim:convert', p), readFile: (path) => ipcRenderer.invoke('file:read', path) }); diff --git a/scripts/fbx2animglb.py b/scripts/fbx2animglb.py new file mode 100644 index 0000000..c1783cd --- /dev/null +++ b/scripts/fbx2animglb.py @@ -0,0 +1,31 @@ +# Convert a Mixamo animation FBX into a skeleton-only GLB clip for the +# DJ Visualizer dance library (bones only, no mesh — tiny files). +# blender -b -P fbx2animglb.py -- input.fbx output.glb +import bpy, sys, os +argv = sys.argv[sys.argv.index('--')+1:] +SRC, OUT = argv[0], argv[1] +bpy.ops.wm.read_factory_settings(use_empty=True) +bpy.ops.import_scene.fbx(filepath=SRC) +arm = None +for o in list(bpy.data.objects): + if o.type == 'ARMATURE' and arm is None: + arm = o + elif o.type == 'MESH': + bpy.data.objects.remove(o, do_unlink=True) +name = os.path.splitext(os.path.basename(SRC))[0].replace('_with_skin', '') +act = arm.animation_data.action if arm and arm.animation_data else None +if not act: + raise SystemExit('nessuna animazione nel file') +act.name = name +act.use_fake_user = True +ad = arm.animation_data +ad.action = None +tr = ad.nla_tracks.new() +tr.name = name +tr.strips.new(name, int(act.frame_range[0]), act) +try: + bpy.ops.export_scene.gltf(filepath=OUT, export_format='GLB', + export_animations=True, export_animation_mode='NLA_TRACKS') +except TypeError: + bpy.ops.export_scene.gltf(filepath=OUT, export_format='GLB', export_animations=True) +print('EXPORT_OK') diff --git a/src/control.html b/src/control.html index 8120fbc..60517d8 100644 --- a/src/control.html +++ b/src/control.html @@ -141,6 +141,19 @@ + +
+ + + +
+ + +
+
Spunta le mosse che il regista può usare (nessuna spuntata = tutte). + Le animazioni extra arrivano da Mixamo: .glb diretti, .fbx convertiti al volo (serve Blender). + BPM: 0 = segue il beat; oppure scrivilo o battilo col TAP.
diff --git a/src/control.js b/src/control.js index 4dd7f17..6483341 100644 --- a/src/control.js +++ b/src/control.js @@ -147,9 +147,40 @@ $('#svg-input').addEventListener('change', (e) => { // The path persists and is re-sent when the output window (re)starts. let glbPath = localStorage.getItem('glb3d') || ''; let glbBg = localStorage.getItem('glbbg') || 'gradient'; +let glbClipSel = []; +try { glbClipSel = JSON.parse(localStorage.getItem('glbclips') || '[]'); } catch (e) {} +let glbAnimFiles = []; +try { glbAnimFiles = JSON.parse(localStorage.getItem('glbanimfiles') || '[]'); } catch (e) {} +let glbBpm = parseInt(localStorage.getItem('glbbpm') || '0', 10) || 0; function glbSend() { if (glbPath) send({ type: 'glb', path: glbPath }); send({ type: 'modelBg', mode: glbBg }); + glbAnimFiles.forEach(p => send({ type: 'animAdd', path: p })); + send({ type: 'glbAnims', names: glbClipSel }); + send({ type: 'modelBpm', bpm: glbBpm }); +} +function renderGlbClips(names) { + const box = $('#glb-anims'); + box.innerHTML = ''; + if (!names || !names.length) { + box.innerHTML = 'Carica un modello riggato per vedere le mosse.'; + return; + } + names.forEach((n) => { + const l = document.createElement('label'); + l.className = 'av-fam' + (glbClipSel.includes(n) ? ' on' : ''); + const c = document.createElement('input'); + c.type = 'checkbox'; c.checked = glbClipSel.includes(n); + c.addEventListener('change', () => { + glbClipSel = c.checked ? glbClipSel.concat(n) : glbClipSel.filter(x => x !== n); + l.classList.toggle('on', c.checked); + localStorage.setItem('glbclips', JSON.stringify(glbClipSel)); + send({ type: 'glbAnims', names: glbClipSel }); + }); + l.appendChild(c); + l.appendChild(document.createTextNode(' ' + n)); + box.appendChild(l); + }); } function glbLabel(name) { $('#glb-name').textContent = name @@ -173,6 +204,57 @@ $('#glb-bg').addEventListener('change', (e) => { localStorage.setItem('glbbg', glbBg); send({ type: 'modelBg', mode: glbBg }); }); +renderGlbClips([]); +$('#btn-anim-load').addEventListener('click', () => $('#anim-input').click()); +$('#anim-input').addEventListener('change', async (e) => { + for (const f of [...e.target.files]) { + const p = djv.pathForFile(f); + if (!p) continue; + if (/\.glb$/i.test(p)) { + if (!glbAnimFiles.includes(p)) glbAnimFiles.push(p); + send({ type: 'animAdd', path: p }); + } else { + glbLabel('⏳ converto ' + f.name + '…'); + const res = await djv.convertAnim(p); + if (res && res.ok) { + if (!glbAnimFiles.includes(res.path)) glbAnimFiles.push(res.path); + send({ type: 'animAdd', path: res.path }); + glbLabel(glbPath ? glbPath.split('/').pop() : ''); + } else { + $('#glb-name').textContent = '⚠️ ' + ((res && res.error) || 'conversione fallita'); + } + } + } + localStorage.setItem('glbanimfiles', JSON.stringify(glbAnimFiles)); + e.target.value = ''; +}); +function glbBpmLabel() { $('#glb-bpm-val').textContent = glbBpm > 0 ? glbBpm + ' BPM' : 'auto'; } +$('#glb-bpm').value = glbBpm; +glbBpmLabel(); +$('#glb-bpm').addEventListener('change', (e) => { + glbBpm = Math.max(0, Math.min(220, parseInt(e.target.value || '0', 10) || 0)); + e.target.value = glbBpm; + localStorage.setItem('glbbpm', String(glbBpm)); + glbBpmLabel(); + send({ type: 'modelBpm', bpm: glbBpm }); +}); +let tapTimes = []; +$('#glb-tap').addEventListener('click', () => { + const now = performance.now(); + if (tapTimes.length && now - tapTimes[tapTimes.length-1] > 2000) tapTimes = []; + tapTimes.push(now); + if (tapTimes.length >= 4) { + const iv = []; + for (let i = Math.max(1, tapTimes.length-8); i < tapTimes.length; i++) + iv.push(tapTimes[i] - tapTimes[i-1]); + const avg = iv.reduce((a, b) => a + b, 0) / iv.length; + glbBpm = Math.max(40, Math.min(220, Math.round(60000/avg))); + $('#glb-bpm').value = glbBpm; + localStorage.setItem('glbbpm', String(glbBpm)); + glbBpmLabel(); + send({ type: 'modelBpm', bpm: glbBpm }); + } +}); $('#btn-glb-load').addEventListener('click', () => $('#glb-input').click()); $('#glb-input').addEventListener('change', (e) => { const f = e.target.files[0]; @@ -1535,6 +1617,7 @@ djv.onReport((m) => { $('#scene-name').textContent = m.name + (m.bpm ? ' · ' + m.bpm + ' BPM' : ''); } break; + case 'glbClips': renderGlbClips(m.names); break; case 'devices': { // The output reports its devices once at startup: good moment to push // config the (possibly recreated) output window doesn't have yet. diff --git a/src/model3d.js b/src/model3d.js index f5bb7c8..64a9c35 100644 --- a/src/model3d.js +++ b/src/model3d.js @@ -187,7 +187,7 @@ function readAccessor(json, bin, idx) { return { data: out, acc }; } -function parseGLB(buf) { +function splitGLB(buf) { const dv = new DataView(buf); if (dv.getUint32(0, true) !== 0x46546C67) throw new Error('non è un file GLB'); let off = 12, json = null, bin = null; @@ -199,6 +199,36 @@ function parseGLB(buf) { off += 8 + len + (len % 4 ? 4 - len % 4 : 0); } if (!json || !bin) throw new Error('GLB incompleto'); + return { json, bin }; +} + +const PACE_RE = /thriller|break|flair|freeze|spin|moonwalk/i; + +// Parse a skeleton-only GLB as an animation LIBRARY: clips whose channels are +// keyed by bone base-name, bindable to any mixamorig-style rig. +function parseAnimLib(buf) { + const { json, bin } = splitGLB(buf); + if (!json.animations || !json.animations.length) throw new Error('nessuna animazione nel GLB'); + const nodeBase = json.nodes.map(n => (n.name || '').split(':').pop().split('.').pop()); + return json.animations.map(a => { + const channels = a.channels + .filter(ch => ch.target.node != null && + (ch.target.path === 'rotation' || ch.target.path === 'translation')) + .map(ch => { + const s = a.samplers[ch.sampler]; + return { bone: nodeBase[ch.target.node], path: ch.target.path, + times: readAccessor(json, bin, s.input).data, + vals: readAccessor(json, bin, s.output).data }; + }); + let dur = 0; + channels.forEach(c => { const e = c.times[c.times.length-1]; if (e > dur) dur = e; }); + const name = a.name || 'clip'; + return { name, channels, dur: Math.max(0.1, dur), pace: PACE_RE.test(name) ? 0.5 : 1 }; + }); +} + +function parseGLB(buf) { + const { json, bin } = splitGLB(buf); // node table (hierarchy kept for skinning) + world transforms const nodesInfo = json.nodes.map((n, i) => ({ @@ -348,8 +378,7 @@ function parseGLB(buf) { channels.forEach(c => { const e = c.times[c.times.length-1]; if (e > dur) dur = e; }); const name = a.name || 'clip'; // expressive/half-time choreographies: one count every TWO beats - const pace = /thriller|break|flair|freeze|spin|moonwalk/i.test(name) ? 0.5 : 1; - return { name, channels, dur: Math.max(0.1, dur), pace }; + return { name, channels, dur: Math.max(0.1, dur), pace: PACE_RE.test(name) ? 0.5 : 1 }; }); } return { prims, min, max, skel: hasSkin ? skel : null, anims }; @@ -442,9 +471,51 @@ class ModelSim { gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([-1,-1, 3,-1, -1,3]), gl.STATIC_DRAW); this.meshes = null; this.loadError = ''; + this.animLib = null; // bone-name clips (bundled dance library) + this.ownAnims = []; // clips carried by the loaded GLB itself + this.libAnims = []; // library clips bound to the current skeleton + this.clipFilter = null; // Set of allowed clip names (empty/null = all) + this.manualBpm = 0; // 0 = follow the detected beat this._upload(torusKnot()); } + // Bundled animation library (skeleton-only GLB): bound by bone base-name to + // whatever rigged model is currently loaded. + setAnimLibrary(buf) { + try { this.animLib = parseAnimLib(buf); } catch (e) { this.animLib = null; return; } + this._bindLibrary(); + } + + _bindLibrary() { + this.libAnims = []; + if (this.skel && this.animLib) { + const idxByBase = {}; + this.skel.nodes.forEach((n, i) => { + idxByBase[n.name.split(':').pop().split('.').pop()] = i; + }); + for (const clip of this.animLib) { + const channels = []; + for (const ch of clip.channels) { + const ni = idxByBase[ch.bone]; + if (ni === undefined) continue; + channels.push({ node: ni, path: ch.path, times: ch.times, vals: ch.vals }); + } + if (channels.length > 4) + this.libAnims.push({ name: clip.name, channels, dur: clip.dur, pace: clip.pace }); + } + } + this._mergeAnims(); + } + + _mergeAnims() { + if (!this.skel) { this.anims = null; return; } + const own = this.ownAnims || []; + const lib = this.libAnims.filter(l => !own.some(o => o.name === l.name)); + const all = own.concat(lib); + this.anims = all.length ? all : null; + this._dir = null; + } + _freeMeshes() { const gl = this.gl; (this.meshes || []).forEach(m => { @@ -459,7 +530,7 @@ class ModelSim { const gl = this.gl; this._freeMeshes(); this.skel = model.skel || null; - this.anims = model.anims || null; + this.ownAnims = model.anims || []; this.restJoints = null; this._db = null; if (this.skel) { @@ -531,10 +602,23 @@ class ModelSim { } return m; }); + this._bindLibrary(); } get hasSkin() { return !!this.skel; } + // Append extra clips (a user-loaded GLB with animations) to the library. + addAnimLibrary(buf) { + const extra = parseAnimLib(buf); // throws on invalid input + if (!this.animLib) this.animLib = []; + for (const c of extra) { + const i = this.animLib.findIndex(x => x.name === c.name); + if (i >= 0) this.animLib[i] = c; else this.animLib.push(c); + } + this._bindLibrary(); + return extra.map(c => c.name); + } + // Joint matrices (world * inverseBind) for the whole skeleton. With // `targets` (base bone name -> world-space direction) an extra local // rotation is solved per bone so that its chain child points along the @@ -583,6 +667,15 @@ class ModelSim { // Musical beat clock: counts beats (with a free-running metronome between // and without detected beats) and exposes a smoothed phase inside the beat. _beatClock(t, beat) { + // manual BPM: a pure metronome, immune to the beat detector + if (this.manualBpm > 0) { + const per = 60/this.manualBpm; + const db = this._db || (this._db = { n: 0, last: t, period: per, prevBeat: 0 }); + db.period = per; + if (t - db.last > 10) db.last = t; + while (t - db.last >= per) { db.last += per; db.n++; } + return { n: db.n, p: Math.max(0, (t - db.last)/per), period: per }; + } const db = this._db || (this._db = { n: 0, last: -1, period: 0.5, prevBeat: 0 }); if (beat > 0.6 && db.prevBeat <= 0.6) { // refractory window: energetic tracks fire on kick AND snare/hats, @@ -615,11 +708,18 @@ class ModelSim { pn0: 0, pp0: 0, lastSwitch: t, sb: 0, psb: 0, lastT: t }); const dt = Math.min(0.1, Math.max(0.001, t - d.lastT)); d.lastT = t; - if (this.anims.length > 1 && - ((bc.n - d.n0) >= 16 || t - d.lastSwitch > 14)) { - d.prev = d.clip; d.pn0 = d.n0; d.pp0 = d.p0; d.psb = d.sb; - let next = Math.floor(Math.random()*this.anims.length); - if (next === d.clip) next = (next + 1) % this.anims.length; + // repertoire: only the clips ticked in the panel (none ticked = all) + const pool = []; + this.anims.forEach((a, i) => { + if (!this.clipFilter || !this.clipFilter.size || this.clipFilter.has(a.name)) pool.push(i); + }); + if (!pool.length) this.anims.forEach((_, i) => pool.push(i)); + const outOfPool = pool.indexOf(d.clip) < 0; + if ((pool.length > 1 || outOfPool) && + (outOfPool || (bc.n - d.n0) >= 16 || t - d.lastSwitch > 14)) { + d.prev = outOfPool ? -1 : d.clip; d.pn0 = d.n0; d.pp0 = d.p0; d.psb = d.sb; + let next = pool[Math.floor(Math.random()*pool.length)]; + if (next === d.clip && pool.length > 1) next = pool[(pool.indexOf(next) + 1) % pool.length]; d.clip = next; d.n0 = bc.n; d.p0 = bc.p; d.sb = 0; d.lastSwitch = t; } const SPB = 0.5*speed; // clip-seconds per music beat diff --git a/src/output.js b/src/output.js index 8661926..5744b30 100644 --- a/src/output.js +++ b/src/output.js @@ -321,6 +321,18 @@ djv.onControl(async (m) => { break; case 'svg': loadCustomTexture(m.dataUrl); break; case 'modelBg': viz.modelBg = m.mode || 'gradient'; break; + case 'glbAnims': viz.setClipFilter(m.names || []); break; + case 'modelBpm': viz.setManualBpm(m.bpm || 0); break; + case 'animAdd': + // extra animation GLB for the dance library + try { + const b = await djv.readFile(m.path); + if (!b) throw new Error('file non leggibile'); + const names = viz.addAnimLibrary(b.buffer.slice(b.byteOffset, b.byteOffset + b.byteLength)); + reportClips(); + if (names.length) djv.report({ type: 'animAdded', names }); + } catch (e) { djv.report({ type: 'error', message: 'Animazione: ' + e.message }); } + break; case 'glb': try { const bytes = await djv.readFile(m.path); @@ -329,6 +341,7 @@ djv.onControl(async (m) => { const ok = viz.setModelData(buf); if (ok === false) throw new Error((viz.model3d && viz.model3d.loadError) || 'GLB non valido'); djv.report({ type: 'glbLoaded', name: m.path.split('/').pop() }); + reportClips(); hideHint(); } catch (e) { djv.report({ type: 'error', message: 'GLB: ' + e.message }); } break; @@ -615,5 +628,18 @@ function frame() { } requestAnimationFrame(frame); +// Tell the panel which dance clips the current model can play. +function reportClips() { + const names = viz.model3d && viz.model3d.anims ? viz.model3d.anims.map(a => a.name) : []; + djv.report({ type: 'glbClips', names }); +} + +// Bundled dance library (skeleton-only GLB with the Mixamo clips): binds to +// any rigged model the user loads. +fetch('djvres://models/dance_library.glb') + .then(r => r.arrayBuffer()) + .then(b => { viz.setAnimLibrary(b); reportClips(); }) + .catch(() => { /* library optional: models with own clips still dance */ }); + // Populate the device list once at startup so the control dropdown fills in. reportDevices(); diff --git a/src/visualizer.js b/src/visualizer.js index 807f53c..dd1f190 100644 --- a/src/visualizer.js +++ b/src/visualizer.js @@ -86,12 +86,48 @@ class Visualizer { this.effect = Object.assign({}, this.effect, effect); } - // Load a GLB model for the 'Modello 3D' family. Parsed lazily on the next - // render so the engine is only created when the family is active. + // The shared 3D engine, fed with any pending model, the bundled animation + // library and the persisted repertoire/BPM settings. + _ensureModel3d() { + if (!this.model3d) { + this.model3d = new window.ModelSim(this.gl); + if (this._libBuf) this.model3d.setAnimLibrary(this._libBuf); + if (this._extraLibs) this._extraLibs.forEach(b => { + try { this.model3d.addAnimLibrary(b); } catch (e) { /* già segnalato */ } + }); + if (this._clipFilter) this.model3d.clipFilter = this._clipFilter; + if (this._manualBpm) this.model3d.manualBpm = this._manualBpm; + } + if (this._pendingGlb) { this.model3d.setModel(this._pendingGlb); this._pendingGlb = null; } + return this.model3d; + } + + // Load a GLB model for the 'Modello 3D' family (parsed immediately so the + // panel can list its clips). setModelData(buf) { - if (this.model3d) return this.model3d.setModel(buf); - this._pendingGlb = buf; - return true; + if (!window.ModelSim) { this._pendingGlb = buf; return true; } + return this._ensureModel3d().setModel(buf); + } + + setAnimLibrary(buf) { + this._libBuf = buf; + if (this.model3d) this.model3d.setAnimLibrary(buf); + } + + addAnimLibrary(buf) { + (this._extraLibs = this._extraLibs || []).push(buf); + if (!window.ModelSim) return []; + return this._ensureModel3d().addAnimLibrary(buf); + } + + setClipFilter(names) { + this._clipFilter = new Set(names || []); + if (this.model3d) this.model3d.clipFilter = this._clipFilter; + } + + setManualBpm(bpm) { + this._manualBpm = bpm || 0; + if (this.model3d) this.model3d.manualBpm = this._manualBpm; } // Upload a custom source (HTMLImageElement / HTMLCanvasElement) for the @@ -112,8 +148,7 @@ class Visualizer { // Release the camera as soon as a non-interactive preset takes over. if (this.inter && !e.isInteractive) this.inter.suspend(); if (e.isModel3d && window.ModelSim) { - if (!this.model3d) this.model3d = new window.ModelSim(gl); - if (this._pendingGlb) { this.model3d.setModel(this._pendingGlb); this._pendingGlb = null; } + this._ensureModel3d(); // optional backdrop behind the model: a fluid mode ('ink'/'fire'/...) // or any shader family ('fam:'), tinted with the active palette const bg = this.modelBg || 'gradient'; @@ -142,9 +177,7 @@ class Visualizer { this.inter.customSource = this.customSource || null; // The model-puppet and avatar modes drive the shared 3D renderer. if ((e.interactiveMode === 'model' || e.interactiveMode === 'avatar') && window.ModelSim) { - if (!this.model3d) this.model3d = new window.ModelSim(gl); - if (this._pendingGlb) { this.model3d.setModel(this._pendingGlb); this._pendingGlb = null; } - this.inter.modelSim = this.model3d; + this.inter.modelSim = this._ensureModel3d(); } this.inter.render(timeSec, audio, e, this.canvas); return;