diff --git a/qml/PropertiesPanel.qml b/qml/PropertiesPanel.qml index cc1e95eb..724e2dfd 100644 --- a/qml/PropertiesPanel.qml +++ b/qml/PropertiesPanel.qml @@ -8727,7 +8727,7 @@ Rectangle { // skewing every new clip is exactly the bug this fixes). var gr = AnimationControlController.generateMotion( genPromptIn.text, 0.0, useModelChk.checked, - 0.0) + 0.0, footPinChk.checked) if (gr && gr.animation) { lastGeneratedAnim = gr.animation // Point the slider at the fresh clip at a neutral 0 @@ -8770,6 +8770,29 @@ Rectangle { anchors.verticalCenter: parent.verticalCenter } } + // #856: foot-contact cleanup — ON by default (detect ground-contact + // spans + IK-pin the feet so they plant instead of skating). + Row { + spacing: 6 + Rectangle { + id: footPinChk + property bool checked: true + width: 14; height: 14; radius: 2 + anchors.verticalCenter: parent.verticalCenter + color: checked ? PropertiesPanelController.highlightColor + : PropertiesPanelController.inputColor + border.color: PropertiesPanelController.borderColor + Text { anchors.centerIn: parent; visible: parent.checked + text: "✓"; color: "white"; font.pixelSize: 10 } + MouseArea { anchors.fill: parent + onClicked: footPinChk.checked = !footPinChk.checked } + } + Text { + text: "Pin feet (contact cleanup)" + color: PropertiesPanelController.textColor; font.pixelSize: 10 + anchors.verticalCenter: parent.verticalCenter + } + } // ── Arm space (#854): Mixamo-style widen/tuck post-process ──────── // Targets `armSpaceAnim` — the last generated clip by default, or // any animation the user picks via the per-row arm button below. diff --git a/scripts/build-motion-library-v5.py b/scripts/build-motion-library-v5.py index 5171b566..6eeb0753 100644 --- a/scripts/build-motion-library-v5.py +++ b/scripts/build-motion-library-v5.py @@ -50,6 +50,9 @@ # animation-name (normalized) → action. Order matters: first hit wins. KEYWORDS = [ ("tpose", None), ("t_pose", None), ("bind", None), ("rest", None), + # compound names FIRST — "walk"/"run" below would swallow them + ("crouchwalk", "crouch"), ("crouchrun", "crouch"), + ("sneakwalk", "sneak"), ("sneakrun", "sneak"), ("walk", "walk"), ("run", "run"), ("jog", "run"), ("sprint", "run"), ("idle", "idle"), ("stand", "idle"), ("breath", "idle"), ("jump", "jump"), ("hop", "jump"), ("leap", "jump"), @@ -238,6 +241,85 @@ def axis(role): elif action in HORIZONTAL_OK: upness = 0.7 + # Placeholder-arm gate: game walk/run cycles are often authored with + # STATIC (T-pose) arms — legs stride while the arms stick straight out. + # Retargeted, that reads as a broken clip. If the legs carry real motion + # but BOTH arms are near-frozen, drop the take. + ARM_ROLES = (7, 8, 11, 12) + LEG_ROLES = (15, 16, 19, 20) + if len(quats) > 1 and action not in ("idle", "sit", "sleep"): + # Energy RELATIVE to the chest (role 2): world quats inherit the + # torso's sway, so locally-frozen arms still show world energy. + # rel = chest^-1 * bone isolates the limb's own motion. + def rel(f, r): + c = quats[f][2] + b = quats[f][r] + ci = [-c[0], -c[1], -c[2], c[3]] + return qmul(ci, b) + def group_energy(roles): + tot = 0.0 + for f in range(1, len(quats)): + tot += sum(quat_angle(rel(f - 1, r), rel(f, r)) + for r in roles) / len(roles) + return tot / (len(quats) - 1) + legs_e = group_energy(LEG_ROLES) + arms_e = group_energy(ARM_ROLES) + if legs_e > 0.004 and arms_e < 0.0015: + return 0.0, (f"static placeholder arms (rel arm energy " + f"{arms_e:.4f} vs legs {legs_e:.4f})") + + # Neck-up gate: some rigs export the neck/head bone with an INVERTED + # axis — the extracted direction points down for the whole clip and the + # retargeted head renders thrown back / buried in the torso. For any + # upright action, the neck chain's animated direction must stay roughly + # up. + if action not in HORIZONTAL_OK and rest_world and rest_dir: + for r in (3, 4, 5): # neck, neck1, head + d = vnorm(rest_dir[r]) + if d is None: + continue + q = rest_world[r] + a_s = qrot([-q[0], -q[1], -q[2], q[3]], d) + tot = sum(qrot(quats[f][r], a_s)[1] for f in range(len(quats))) + if tot / max(1, len(quats)) < 0.3: + return 0.0, (f"neck/head direction not upright (role {r} " + f"mean up-dot {tot / max(1, len(quats)):.2f}) — " + "inverted neck axis, head renders thrown back") + break # first resolvable is enough + + # Horizontal-arm gate for plain locomotion: zombie-shamble / T-pose-armed + # walk cycles hold the upper arms near-horizontal for the whole clip + # (mean |up-component| of the upper-arm direction ~0 vs 0.6-0.95 on a + # natural walk). Retargeted onto a generic character under a generic + # "walk" prompt they read broken — drop them from locomotion actions. + RSHO, LSHO = 7, 11 + if action in ("walk", "run", "march") and rest_world and rest_dir: + downdots = [] + for r in (RSHO, LSHO): + d = vnorm(rest_dir[r]) + if d is None: + continue + q = rest_world[r] + a_s = qrot([-q[0], -q[1], -q[2], q[3]], d) + tot = 0.0 + for f in range(len(quats)): + tot += qrot(quats[f][r], a_s)[1] # SIGNED up-component + downdots.append(tot / max(1, len(quats))) + if not downdots: + # Arm roles UNRESOLVED: the retarget leaves the target's arms at + # its bind pose — a literal T-pose held for the whole clip. + return 0.0, (f"arm roles unresolved — target arms would freeze " + f"in the bind T-pose during {action}") + # judge each arm separately, on the SIGNED up-component: a natural + # locomotion arm hangs (mean Y ~ -0.6..-0.95). Horizontal zombie arms + # (~0) AND raised arms (+) both read broken — an abs() gate passed a + # straight-up arm as if it were hanging (several corpus rigs carry a + # per-bone axis inversion that renders one arm skyward). + if max(downdots) > -0.25: + return 0.0, (f"arm(s) not hanging (upper-arm signed up-dots " + f"{[round(u, 2) for u in downdots]}) — zombie/" + f"T-pose/raised style, not a generic {action}") + # Energy band: below = a pose, way above = spasm/mis-mapped. lo, hi, cap = min_energy * 2.0, 0.10, 0.20 if mean_energy <= lo: diff --git a/scripts/prep-t2m-v5.py b/scripts/prep-t2m-v5.py new file mode 100644 index 00000000..e7a387f2 --- /dev/null +++ b/scripts/prep-t2m-v5.py @@ -0,0 +1,398 @@ +#!/usr/bin/env python3 +# ruff: noqa: E702, E741 +"""Build the v5 text-to-motion training cache from the motion corpus (#858). + +ONE-TIME OFFLINE dev tool — NOT shipped. Successor to prep-t2m-v4.py, fixing +its structural flaw: v4 trained on raw per-rig world quats, so every rig's +bone-axis convention leaked into the data (the "convention mush" that made +the CVAE average modes into gentle motion). v5 CANONICALIZES every clip into +a rig-independent representation before windowing: + + For each canonical joint c with source reference triple (restWorld Q_r, + restDir d_r) — the same triple the bind-referenced retarget (PR #843) + consumes — the source bone's world direction trajectory is + d(f) = W(f) · (Q_r⁻¹ · d̂_r) (canonical axes) + and its roll about the bone is the swing/twist residual + Δ(f) = W(f) · Q_r⁻¹ + aim = arc(d̂_r → d(f)) + θ(f) = signed angle of (aim⁻¹ · Δ(f)) about d̂_r, unwrapped over f. + The training quat is rebuilt against a FIXED canonical T-pose direction + set D (spine +Y, left arm +X, right arm −X, legs −Y, feet +Z): + Q'(f) = twist(θ(f), d(f)) · arc(D_c → d(f)) + + Q' is identical for every rig performing the same motion — and it is + EXACTLY what the runtime consumes: with the model's reference triple + (restWorld = identity, restDir = D), applyMotionClip recovers + d(f) = Q'(f)·D_c and (with #857 twist transport) θ(f) losslessly. The + model therefore rides the SAME retarget path as v5 template clips, no + synthetic-standing-pose shim. + +Sources: + --corpus ~/motion_corpus: the *.canonical.json sidecar dumps written by + build-motion-library-v5.py / `qtmesh anim --dump-canonical` + (#838/#839). Labels via the SAME action keyword table as the + template library, so both motion sources cover the same vocab. + --bvh CMU BVH conversion (optional): FK world quats (rest = identity, + restDir from the skeleton's rest joint positions), labels from + the public CMU index — the v4 path, canonicalized the same way. + +Output npz: mo[N,T,22,4] canonical quats, msk[N,22] per-joint validity, +tk[N,V] one-hot, vocab, fps, canonRestDir[22,3] (= D, for the vocab json). + +Usage: + python3 scripts/prep-t2m-v5.py --corpus ~/motion_corpus \ + [--bvh /tmp/cmu-mocap-bvh/data --index /tmp/cmu_index.txt] \ + --out /tmp/t2m_v5.npz [--T 60] +""" +import argparse +import glob +import importlib.util +import json +import os +import re +import sys + +import numpy as np + +HERE = os.path.dirname(os.path.abspath(__file__)) + + +def load_module(name, fname): + spec = importlib.util.spec_from_file_location(name, os.path.join(HERE, fname)) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +libv5 = load_module("libv5", "build-motion-library-v5.py") # action_for, quality +prep4 = load_module("prep4", "prep-t2m-v4.py") # BVH FK, index, windows + +CANON = prep4.CANON +J = 22 +FPS = 30 +# canonical chain parent (AnimationMerger kParentCanon) +PARENT = [-1, 0, 1, 2, 3, 4, 2, 6, 7, 8, 2, 10, 11, 12, 0, 14, 15, 16, 0, 18, 19, 20] +CHILD = {} +for _c, _p in enumerate(PARENT): # chain child = FIRST child (hip→abdomen, + if _p >= 0: # chest→neck), matching the extractor + CHILD.setdefault(_p, _c) + +# FIXED canonical T-pose bone directions D (canonical axes: +Y up, +Z fwd, +# +X left — the CMU frame convention every dump is normalized into). +D_CANON = np.array([ + [0, 1, 0], [0, 1, 0], [0, 1, 0], [0, 1, 0], [0, 1, 0], [0, 1, 0], + [-1, 0, 0], [-1, 0, 0], [-1, 0, 0], [-1, 0, 0], + [1, 0, 0], [1, 0, 0], [1, 0, 0], [1, 0, 0], + [0, -1, 0], [0, -1, 0], [0, -1, 0], [0, 0, 1], + [0, -1, 0], [0, -1, 0], [0, -1, 0], [0, 0, 1], +], np.float32) + + +# ---------- quat math, (x,y,z,w), vectorised over leading dims ---------- +def qmul(a, b): + ax, ay, az, aw = np.moveaxis(a, -1, 0) + bx, by, bz, bw = np.moveaxis(b, -1, 0) + return np.stack([aw*bx + ax*bw + ay*bz - az*by, + aw*by - ax*bz + ay*bw + az*bx, + aw*bz + ax*by - ay*bx + az*bw, + aw*bw - ax*bx - ay*by - az*bz], -1) + + +def qconj(q): + return q * np.array([-1, -1, -1, 1], q.dtype) + + +def qrot(q, v): + """Rotate vectors v[...,3] by quats q[...,4].""" + qv = q[..., :3] + uv = np.cross(qv, v) + uuv = np.cross(qv, uv) + return v + 2.0 * (q[..., 3:4] * uv + uuv) + + +def qnorm(q): + return q / (np.linalg.norm(q, axis=-1, keepdims=True) + 1e-12) + + +def arc(a, b): + """Shortest-arc quats rotating unit vectors a[...,3] onto b[...,3].""" + d = (a * b).sum(-1, keepdims=True) + axis = np.cross(a, b) + w = 1.0 + d + q = np.concatenate([axis, w], -1) + # antiparallel: rotate π about any perpendicular of a + bad = (w[..., 0] < 1e-6) + if np.any(bad): + ab = a[bad] + perp = np.cross(ab, np.broadcast_to([1.0, 0, 0], ab.shape)) + n = np.linalg.norm(perp, axis=-1, keepdims=True) + alt = np.cross(ab, np.broadcast_to([0, 1.0, 0], ab.shape)) + perp = np.where(n > 1e-6, perp, alt) + perp /= (np.linalg.norm(perp, axis=-1, keepdims=True) + 1e-12) + q[bad] = np.concatenate([perp, np.zeros_like(perp[..., :1])], -1) + return qnorm(q) + + +def axis_angle(axis, ang): + h = ang[..., None] * 0.5 + return np.concatenate([axis * np.sin(h), np.cos(h)], -1) + + +def canonicalize(wq, rest_world, rest_dir): + """wq[T,J,4] world quats + reference triple → (Q'[T,J,4], valid[J]). + + Q' is the rig-independent training quat; invalid joints (zero restDir) + are identity with valid=0. + """ + T = wq.shape[0] + out = np.zeros((T, J, 4), np.float32) + out[..., 3] = 1.0 + valid = np.zeros(J, np.float32) + for c in range(J): + dr = np.asarray(rest_dir[c], np.float32) + n = np.linalg.norm(dr) + if n < 1e-6: + continue + dr = dr / n + qr = qnorm(np.asarray(rest_world[c], np.float32)) + a_s = qrot(qconj(qr)[None], dr[None])[0] # bone's local axis + W = qnorm(wq[:, c]) # [T,4] + d = qrot(W, np.broadcast_to(a_s, (T, 3))) # [T,3] world dir + d = d / (np.linalg.norm(d, axis=-1, keepdims=True) + 1e-12) + delta = qmul(W, np.broadcast_to(qconj(qr), (T, 4))) + aim_s = arc(np.broadcast_to(dr, (T, 3)), d) + tw = qmul(qconj(aim_s), delta) # twist about dr + th = 2.0 * np.arctan2((tw[:, :3] * dr).sum(-1), tw[:, 3]) + th = np.unwrap(th) + # cap runaway unwrap drift (numeric noise on near-degenerate aims) + th = np.clip(th, -2.5 * np.pi, 2.5 * np.pi) + aim_c = arc(np.broadcast_to(D_CANON[c], (T, 3)), d) + out[:, c] = qmul(axis_angle(d, th), aim_c) + valid[c] = 1.0 + return qnorm(out), valid + + +# ---------- corpus source ---------- +def corpus_clips(corpus, min_roles): + """Yield (action, Q'[T,J,4], valid[J], src) from every sidecar dump.""" + manifest = {} + mpath = os.path.join(corpus, "manifest.json") + if os.path.exists(mpath): + manifest = json.load(open(mpath)) + dumps = sorted(glob.glob(os.path.join(corpus, "raw", "**", + "*.canonical.json"), recursive=True)) + for dpath in dumps: + try: + dump = json.load(open(dpath)) + except Exception: + continue + asset = os.path.basename(os.path.dirname(dpath)) + prov = libv5.manifest_lookup(manifest, asset) or {} + tags = prov.get("tags", []) + title = prov.get("title", asset) + for c in dump.get("clips", []): + if c.get("resolvedRoles", 0) < min_roles: + continue + q = c.get("quats", []) + rw = c.get("restWorld", []) + rd = c.get("restDir", []) + if len(q) < 12 or len(rw) != J or len(rd) != J: + continue + action = libv5.action_for(c.get("animation", ""), tags) + if not action: + continue + # the library's uprightness/energy gate (#855) — same hygiene bar + e = libv5.frame_energy(q) + me = sum(e) / max(1, len(e)) + if me < 0.004: + continue + quality, drop = libv5.clip_quality( + action, q, rw, rd, c.get("resolvedRoles", 0), me, 0.004) + if drop: + continue + wq = np.asarray(q, np.float32) # [T,J,4] + cq, valid = canonicalize(wq, rw, rd) + yield action, cq, valid, f"{title} — {c.get('animation')}" + + +# ---------- CMU source ---------- +def bvh_rest(path): + """Rest joint positions from a BVH's OFFSET tree → restDir per canonical + joint (identity rest rotations ⇒ world dir = offset-chain direction).""" + from bvh import Bvh + with open(path) as f: + m = Bvh(f.read()) + cmap = prep4.resolve_canon(m.get_joints_names()) + if cmap is None: + return None + pos = {} + for name in m.get_joints_names(): + p, cur = np.zeros(3), name + while cur is not None: + p = p + np.asarray(m.joint_offset(cur), np.float64) + try: + par = m.joint_parent(cur) + cur = par.name if par else None + except Exception: + cur = None + pos[name] = p + rd = np.zeros((J, 3), np.float32) + for c in range(J): + if c in CHILD: + v = pos[cmap[CANON[CHILD[c]]]] - pos[cmap[CANON[c]]] + else: # tips: head +Y, hands follow the forearm, feet +Z + if c == 5: + v = np.array([0, 1.0, 0]) + elif c in (9, 13): + v = pos[cmap[CANON[c]]] - pos[cmap[CANON[PARENT[c]]]] + else: + v = np.array([0, 0, 1.0]) + n = np.linalg.norm(v) + rd[c] = (v / n) if n > 1e-6 else 0.0 + return rd + + +def cmu_clips(bvh_dir, index): + labels = prep4.load_index(index) + files = sorted(glob.glob(os.path.join(bvh_dir, "**/*.bvh"), recursive=True)) + ident = np.zeros((J, 4), np.float32) + ident[:, 3] = 1.0 + for path in files: + mid_m = re.match(r"(\d+_\d+)", os.path.basename(path)) + mid = mid_m.group(1) if mid_m else None + if mid is None or mid not in labels: + continue + r = prep4.parse_bvh(path) + if r is None: + continue + _lq, wq = r + rd = bvh_rest(path) + if rd is None: + continue + cq, valid = canonicalize(wq, ident, rd) + yield labels[mid], cq, valid, f"CMU {mid}" + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--corpus", default="") + ap.add_argument("--bvh", default="") + ap.add_argument("--index", default="") + ap.add_argument("--out", default="/tmp/t2m_v5.npz") + ap.add_argument("--T", type=int, default=40) + ap.add_argument("--min-roles", type=int, default=12) + ap.add_argument("--min-action-windows", type=int, default=8) + a = ap.parse_args() + + mo, msk, acts, srcs = [], [], [], [] + + def angdist(x, y): + d = np.abs((x * y).sum(-1)).clip(0, 1) + return 2.0 * np.arccos(d) + + LOCOMOTION = {"walk", "run", "march"} + + def posture_ok(action, cq, valid): + """Posture gates (#837 quality follow-up): the v5 model learned a + head-down hunched walk because unfiltered windows include folded / + idle-contaminated / placeholder-armed source content. In the + CANONICAL rep the checks are trivial — d(f) = Q'(f)·D_c: + spine must stay up, neck/head must stay up, and locomotion arms + must HANG (signed Y, the library-curation lesson: abs() passes a + skyward arm).""" + def mean_y(r): + d = qrot(cq[:, r], np.broadcast_to(D_CANON[r], (len(cq), 3))) + return float(d[:, 1].mean()) + floor = 0.7 if action in LOCOMOTION else 0.5 + for r in (0, 1, 2): # spine chain + if valid[r]: + if mean_y(r) < floor: + return False + break + for r in (3, 4, 5): # neck / head + if valid[r]: + if mean_y(r) < 0.5: + return False + break + if action in LOCOMOTION: + for r in (7, 11): # upper arms hang + if valid[r] and mean_y(r) > -0.25: + return False + return True + + dropped = [0] + + def window(action, cq, valid, src): + # The v4 neutral-start gate is deliberately GONE: model clips now ride + # the bind-referenced direction retarget, which references the + # reference TRIPLE, not clip frame 0 — any phase is a valid window. + T, nF = a.T, cq.shape[0] + if nF < max(16, T // 2): + return + if nF < T: + # pad: loop-tile cyclic clips (end pose ≈ start pose), else hold + cyc = angdist(cq[-1], cq[0]).mean() < 0.25 + reps = [cq] + while sum(r.shape[0] for r in reps) < T: + reps.append(cq if cyc else cq[-1:].repeat(T, 0)) + cq = np.concatenate(reps, 0)[:T] + nF = T + for s in range(0, nF - T + 1, max(1, T // 2)): + w = cq[s:s + T] + if not posture_ok(action, w, valid): + dropped[0] += 1 + continue + mo.append(w) + msk.append(valid) + acts.append(action) + srcs.append(src) + + # Per-item guard: a single malformed clip/BVH must not abort a multi- + # thousand-item batch (offline dev tool — resilience over strictness). + if a.corpus: + n0, bad = 0, 0 + for action, cq, valid, src in corpus_clips( + os.path.expanduser(a.corpus), a.min_roles): + try: + window(action, cq, valid, src); n0 += 1 + except Exception as e: # noqa: BLE001 + bad += 1 + print(f" skip corpus clip ({src}): {e}") + print(f"corpus: {n0} clips → {len(mo)} windows ({bad} skipped)") + if a.bvh and a.index: + n0, w0, bad = 0, len(mo), 0 + for action, cq, valid, src in cmu_clips(a.bvh, a.index): + try: + window(action, cq, valid, src); n0 += 1 + except Exception as e: # noqa: BLE001 + bad += 1 + print(f" skip cmu trial ({src}): {e}") + print(f"cmu: {n0} trials → {len(mo) - w0} windows ({bad} skipped)") + + print(f"posture gates dropped {dropped[0]} windows") + if not mo: + sys.exit("no windows extracted") + + # vocab = actions with enough support + from collections import Counter + cnt = Counter(acts) + vocab = sorted(w for w, n in cnt.items() if n >= a.min_action_windows) + keep = [i for i, w in enumerate(acts) if w in vocab] + if not vocab or not keep: + sys.exit(f"no action reached --min-action-windows " + f"({a.min_action_windows}); per-action counts: {dict(cnt)}") + mo = np.stack([mo[i] for i in keep]).astype(np.float32) + msk = np.stack([msk[i] for i in keep]).astype(np.float32) + tk = np.zeros((len(keep), len(vocab)), np.float32) + for r, i in enumerate(keep): + tk[r, vocab.index(acts[i])] = 1.0 + print(f"windows: {mo.shape[0]} vocab({len(vocab)}):", + {w: int(tk[:, vocab.index(w)].sum()) for w in vocab}) + np.savez_compressed(a.out, mo=mo, msk=msk, tk=tk, + vocab=np.array(vocab), fps=FPS, + canonRestDir=D_CANON) + print(f"wrote {a.out} mo{mo.shape}") + + +if __name__ == "__main__": + main() diff --git a/scripts/prep-t2m-v6.py b/scripts/prep-t2m-v6.py new file mode 100644 index 00000000..85417c67 --- /dev/null +++ b/scripts/prep-t2m-v6.py @@ -0,0 +1,251 @@ +#!/usr/bin/env python3 +# ruff: noqa: E702, E741 +"""Build the v6 text-to-motion training cache — CURATION-GRADE (#837). + +ONE-TIME OFFLINE dev tool — NOT shipped. Successor to prep-t2m-v5.py. + +The v5.1 model still rendered mushy walks: flow matching samples the +distribution it is given, and the v5 distribution contained everything the +extractor produced — turns, pauses, off-poses, style outliers. v6 applies +the SAME quality bar that curates the shipping template library to every +individual training window, folds the curated library takes themselves into +the set, and augments: + + gates (per window, canonical rep — d(f) = Q'(f)·D_c): + - spine + neck/head upright (stricter for locomotion) + - locomotion upper arms HANG, judged per arm on the SIGNED up-component + - energy band: mean joint speed in [lo, hi] (poses and spasms both out) + augmentation: + - sagittal MIRROR: q' = (x, -y, -z, w) + swap L/R roles (doubles data, + teaches symmetry) + - SPEED 0.85x / 1.15x (slerp resample) + +Windows are T=60 @ 30 fps (2 s). Output schema matches prep-t2m-v5 +(mo/msk/tk/vocab/fps/canonRestDir) so train-t2m-flow-v5.py runs unchanged. + +Usage: + python3 scripts/prep-t2m-v6.py --corpus ~/motion_corpus \ + --bvh /data --index /cmu-mocap-index-text.txt \ + --library ~/t2m_v6/motion-library.json --out ~/t2m_v6/t2m_v6.npz +""" +import argparse +import importlib.util +import json +import os +import sys + +import numpy as np + +HERE = os.path.dirname(os.path.abspath(__file__)) + + +def load_module(name, fname): + spec = importlib.util.spec_from_file_location(name, os.path.join(HERE, fname)) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +prep5 = load_module("prep5", "prep-t2m-v5.py") + +J = 22 +FPS = 30 +D_CANON = prep5.D_CANON +# canonical L/R role pairs (AnimationMerger kLR) +LR = [(6, 10), (7, 11), (8, 12), (9, 13), (14, 18), (15, 19), (16, 20), (17, 21)] +MIRROR_PERM = list(range(J)) +for a, b in LR: + MIRROR_PERM[a], MIRROR_PERM[b] = b, a + +LOCOMOTION = {"walk", "run", "march"} +HORIZONTAL_OK = {"death", "crawl", "roll", "swim", "fall", "sleep", "sit"} + + +def qrot(q, v): + qv = q[..., :3] + uv = np.cross(qv, v) + uuv = np.cross(qv, uv) + return v + 2.0 * (q[..., 3:4] * uv + uuv) + + +def mean_dir_y(w, r): + d = qrot(w[:, r], np.broadcast_to(D_CANON[r], (len(w), 3))) + return float(d[:, 1].mean()) + + +# canonical parent chain (AnimationMerger kParentCanon) for FK +PAR = [-1, 0, 1, 2, 3, 4, 2, 6, 7, 8, 2, 10, 11, 12, 0, 14, 15, 16, 0, 18, 19, 20] + + +def foot_travel_ratio(w): + """fwd/side travel ratio of the feet over the window (unit bone lengths). + A clean walk/run steps front-to-back (Z >> X); a splayed/side-step stride + (the v6 model's failure mode) has X ~ Z. Positions come from the same + canonical-direction FK the retarget uses, so this measures exactly what + renders. Returns fwd/side (higher = cleaner).""" + T = len(w) + + def pos(role): + p = np.zeros((T, 3), np.float32) + r = role + while PAR[r] >= 0: + p = p + qrot(w[:, PAR[r]], np.broadcast_to(D_CANON[r], (T, 3))) + r = PAR[r] + return p + side = fwd = 0.0 + for foot in (17, 21): + c = pos(foot) + c = c - c.mean(0, keepdims=True) + side += float(np.abs(c[:, 0]).mean()) + fwd += float(np.abs(c[:, 2]).mean()) + return fwd / (side + 1e-6) + + +def window_quality(action, w, valid): + """True when the window meets the library curation bar.""" + # energy band — mean joint rotation speed (rad/frame) + dq = np.abs((w[1:] * w[:-1]).sum(-1)).clip(0, 1) + e = float((2 * np.arccos(dq)).mean()) + if not (0.004 <= e <= 0.11): + return False + if action in HORIZONTAL_OK: + return True + floor = 0.7 if action in LOCOMOTION else 0.5 + for r in (0, 1, 2): + if valid[r]: + if mean_dir_y(w, r) < floor: + return False + break + for r in (3, 4, 5): + if valid[r]: + if mean_dir_y(w, r) < 0.5: + return False + break + if action in LOCOMOTION: + for r in (7, 11): + if valid[r] and mean_dir_y(w, r) > -0.25: + return False + # Stride-directionality gate (v6.1): the feet must step FORWARD, not + # sideways. 59% of raw CMU walk windows are splayed/side-stepping + # (measured fwd/side < 1.5) — the model faithfully learned that + # majority and walked sideways. Require a clean front-to-back stride. + if valid[17] and valid[21] and foot_travel_ratio(w) < 2.0: + return False + return True + + +def mirror(w, valid): + """Sagittal mirror: reflect each quat (x,-y,-z,w) and swap L/R roles.""" + m = w * np.array([1, -1, -1, 1], np.float32) + return m[:, MIRROR_PERM], valid[MIRROR_PERM] + + +def retime(w, factor): + """Slerp-resample a [T,J,4] window to the same length at `factor` speed.""" + T = w.shape[0] + src = np.clip(np.arange(T, dtype=np.float64) * factor, 0, T - 1.001) + i0 = src.astype(int) + t = (src - i0)[:, None, None].astype(np.float32) + a, b = w[i0], w[np.minimum(i0 + 1, T - 1)] + # hemisphere-align then nlerp (windows are 30fps — angles tiny) + dot = (a * b).sum(-1, keepdims=True) + b = np.where(dot < 0, -b, b) + out = a * (1 - t) + b * t + return out / (np.linalg.norm(out, axis=-1, keepdims=True) + 1e-12) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--corpus", default="") + ap.add_argument("--bvh", default="") + ap.add_argument("--index", default="") + ap.add_argument("--library", default="", + help="curated motion-library.json — takes are folded in " + "as extra (already-curated) source clips") + ap.add_argument("--out", default=os.path.expanduser("~/t2m_v6/t2m_v6.npz")) + ap.add_argument("--T", type=int, default=60) + ap.add_argument("--min-roles", type=int, default=12) + ap.add_argument("--min-action-windows", type=int, default=16) + a = ap.parse_args() + + T = a.T + mo, msk, acts = [], [], [] + gated = [0] + + def add(action, w, valid): + mo.append(w) + msk.append(valid) + acts.append(action) + + def windows(action, cq, valid): + nF = cq.shape[0] + if nF < T // 2: + return + if nF < T: + dq = np.abs((cq[-1] * cq[0]).sum(-1)).clip(0, 1) + cyc = float((2 * np.arccos(dq)).mean()) < 0.25 + reps = [cq] + while sum(r.shape[0] for r in reps) < T: + reps.append(cq if cyc else cq[-1:].repeat(T, 0)) + cq = np.concatenate(reps, 0)[:T] + nF = T + for s in range(0, nF - T + 1, max(1, T // 3)): + w = cq[s:s + T] + if not window_quality(action, w, valid): + gated[0] += 1 + continue + add(action, w, valid) + mw, mv = mirror(w, valid) + add(action, mw, mv) + for f in (0.85, 1.15): + add(action, retime(w, f), valid) + + if a.corpus: + n = 0 + for action, cq, valid, _src in prep5.corpus_clips( + os.path.expanduser(a.corpus), a.min_roles): + windows(action, cq, valid) + n += 1 + print(f"corpus: {n} clips → {len(mo)} windows (cum)") + if a.library: + lib = json.load(open(os.path.expanduser(a.library))) + n = 0 + for c in lib.get("clips", []): + rw, rd = c.get("restWorld"), c.get("restDir") + if not rw or not rd: + continue + cq, valid = prep5.canonicalize( + np.asarray(c["quats"], np.float32), rw, rd) + windows(c["action"], cq, valid) + n += 1 + print(f"library: {n} takes → {len(mo)} windows (cum)") + if a.bvh and a.index: + n = 0 + for action, cq, valid, _src in prep5.cmu_clips(a.bvh, a.index): + windows(action, cq, valid) + n += 1 + print(f"cmu: {n} trials → {len(mo)} windows (cum)") + + print(f"quality gates dropped {gated[0]} base windows") + if not mo: + sys.exit("no windows") + from collections import Counter + cnt = Counter(acts) + vocab = sorted(w for w, k in cnt.items() if k >= a.min_action_windows) + keep = [i for i, w in enumerate(acts) if w in vocab] + mo = np.stack([mo[i] for i in keep]).astype(np.float32) + msk = np.stack([msk[i] for i in keep]).astype(np.float32) + tk = np.zeros((len(keep), len(vocab)), np.float32) + for r, i in enumerate(keep): + tk[r, vocab.index(acts[i])] = 1.0 + print(f"windows: {mo.shape[0]} vocab({len(vocab)}):", + {w: int(tk[:, vocab.index(w)].sum()) for w in vocab}) + os.makedirs(os.path.dirname(os.path.abspath(a.out)), exist_ok=True) + np.savez_compressed(a.out, mo=mo, msk=msk, tk=tk, + vocab=np.array(vocab), fps=FPS, + canonRestDir=D_CANON) + print(f"wrote {a.out} mo{mo.shape}") + + +if __name__ == "__main__": + main() diff --git a/scripts/train-t2m-flow-v5.py b/scripts/train-t2m-flow-v5.py new file mode 100644 index 00000000..ba6c145e --- /dev/null +++ b/scripts/train-t2m-flow-v5.py @@ -0,0 +1,310 @@ +#!/usr/bin/env python3 +# ruff: noqa: E702, E741 +"""Train the v5 FLOW-MATCHING text-to-motion model (#840, epic #837). + +ONE-TIME OFFLINE dev tool — NOT shipped. Replaces the v4 CVAE: a conditional- +mean decoder averages an action's phase-misaligned windows into gentle motion +(the v4 training notes measured this); flow matching samples a transport path +from noise to A SINGLE MODE of the data distribution instead — the one +architectural change that separates MDM-era quality from the CVAE, +independent of data scale (#837). + +Data: /tmp/t2m_v5.npz from prep-t2m-v5.py — CANONICALIZED quats (rig- +independent aim+twist against the fixed canonical T-pose, #858), per-joint +validity masks, one-hot action labels. + +Architecture: small DiT-style transformer v(x_t, t, action): + x[B,T,132] (22 joints × 6D rotation) + sinusoidal frame positions, + conditioned on (flow time t, action embedding) via AdaLN-Zero-lite + (per-layer scale/shift from the conditioning vector). Rectified-flow + objective: x_t=(1−t)x0+t·x1, target v*=x1−x0, masked joint-wise MSE. + +Export: ONNX graph with the EULER SAMPLER UNROLLED INSIDE (N fixed steps), +matching the shipped MotionGenerator contract exactly: + inputs tokens[1,V] one-hot, seed[1,Z] (Z = T·132 flattened noise; + MotionGenerator draws N(0,0.5) — the graph rescales ×2 to unit) + output motion[1,T,220] (per joint: tx,ty,tz=0, + quat x,y,z,w from Gram-Schmidt 6D→R, sx,sy,sz=1) +plus t2m-vocab.json carrying the CANONICAL REFERENCE TRIPLE (restWorld = +identity, restDir = the fixed canonical T-pose directions) so model clips +ride the same bind-referenced direction retarget as v5 template clips — +retiring the synthetic-standing-pose shim (#858). + +Usage: + python3 scripts/train-t2m-flow-v5.py --data /tmp/t2m_v5.npz \ + --out /tmp/t2m_v5_flow --epochs 60 --device mps [--steps 16] +""" +import argparse +import json +import math +import os + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F + +J, D6 = 22, 6 +C6 = J * D6 # 132 + + +# ---------- rotation reps ---------- +def quat_to_6d(q): + """q[...,4] (x,y,z,w) → first two rotation-matrix COLUMNS [...,6].""" + x, y, z, w = q.unbind(-1) + c0 = torch.stack([1 - 2 * (y * y + z * z), + 2 * (x * y + z * w), + 2 * (x * z - y * w)], -1) + c1 = torch.stack([2 * (x * y - z * w), + 1 - 2 * (x * x + z * z), + 2 * (y * z + x * w)], -1) + return torch.cat([c0, c1], -1) + + +def d6_to_quat(d): + """[...,6] → unit quats [...,4] (x,y,z,w) via Gram-Schmidt (ONNX-safe).""" + a, b = d[..., :3], d[..., 3:] + c0 = F.normalize(a, dim=-1, eps=1e-6) + b = b - (c0 * b).sum(-1, keepdim=True) * c0 + c1 = F.normalize(b, dim=-1, eps=1e-6) + c2 = torch.cross(c0, c1, dim=-1) + m00, m10, m20 = c0.unbind(-1) + m01, m11, m21 = c1.unbind(-1) + m02, m12, m22 = c2.unbind(-1) + # branchless Shepperd: build all four candidates, pick the max-norm one + t0 = 1 + m00 + m11 + m22 + t1 = 1 + m00 - m11 - m22 + t2 = 1 - m00 + m11 - m22 + t3 = 1 - m00 - m11 + m22 + eps = 1e-8 + q0 = torch.stack([m21 - m12, m02 - m20, m10 - m01, t0], -1) \ + / (2.0 * torch.sqrt(t0.clamp_min(eps)).unsqueeze(-1)) + q1 = torch.stack([t1, m01 + m10, m02 + m20, m21 - m12], -1) \ + / (2.0 * torch.sqrt(t1.clamp_min(eps)).unsqueeze(-1)) + q2 = torch.stack([m01 + m10, t2, m12 + m21, m02 - m20], -1) \ + / (2.0 * torch.sqrt(t2.clamp_min(eps)).unsqueeze(-1)) + q3 = torch.stack([m02 + m20, m12 + m21, t3, m10 - m01], -1) \ + / (2.0 * torch.sqrt(t3.clamp_min(eps)).unsqueeze(-1)) + ts = torch.stack([t0, t1, t2, t3], -1) + idx = ts.argmax(-1, keepdim=True) + qs = torch.stack([q0, q1, q2, q3], -2) # [...,4cand,4] + q = torch.gather(qs, -2, + idx.unsqueeze(-1).expand(*idx.shape, 4)).squeeze(-2) + return F.normalize(q, dim=-1, eps=1e-6) + + +# ---------- model ---------- +class Block(nn.Module): + def __init__(self, dim, heads): + super().__init__() + self.n1 = nn.LayerNorm(dim, elementwise_affine=False) + self.attn = nn.MultiheadAttention(dim, heads, batch_first=True) + self.n2 = nn.LayerNorm(dim, elementwise_affine=False) + self.mlp = nn.Sequential(nn.Linear(dim, dim * 4), nn.GELU(), + nn.Linear(dim * 4, dim)) + self.ada = nn.Linear(dim, dim * 6) + nn.init.zeros_(self.ada.weight) + nn.init.zeros_(self.ada.bias) + + def forward(self, x, c): + s1, b1, g1, s2, b2, g2 = self.ada(c).unsqueeze(1).chunk(6, -1) + h = self.n1(x) * (1 + s1) + b1 + x = x + g1 * self.attn(h, h, h, need_weights=False)[0] + h = self.n2(x) * (1 + s2) + b2 + return x + g2 * self.mlp(h) + + +class FlowDiT(nn.Module): + def __init__(self, V, T, dim=256, layers=6, heads=8): + super().__init__() + self.T = T + self.inp = nn.Linear(C6, dim) + self.pos = nn.Parameter(torch.randn(1, T, dim) * 0.02) + self.act_emb = nn.Linear(V, dim) + self.t_mlp = nn.Sequential(nn.Linear(dim, dim), nn.SiLU(), + nn.Linear(dim, dim)) + self.blocks = nn.ModuleList([Block(dim, heads) for _ in range(layers)]) + self.out_n = nn.LayerNorm(dim, elementwise_affine=False) + self.out = nn.Linear(dim, C6) + nn.init.zeros_(self.out.weight) + nn.init.zeros_(self.out.bias) + half = dim // 2 + self.register_buffer( + "freqs", torch.exp(-math.log(1e4) + * torch.arange(half).float() / half)) + + def forward(self, x, t, tok): + # x[B,T,C6], t[B] in [0,1], tok[B,V] + ang = t[:, None] * 1000.0 * self.freqs[None] + temb = torch.cat([torch.sin(ang), torch.cos(ang)], -1) + c = self.t_mlp(temb) + self.act_emb(tok) + h = self.inp(x) + self.pos + for blk in self.blocks: + h = blk(h, c) + return self.out(self.out_n(h)) + + +class Sampler(nn.Module): + """Euler flow sampler UNROLLED for ONNX export — MotionGenerator contract.""" + + def __init__(self, net, V, T, steps, guidance=2.0): + super().__init__() + self.net, self.V, self.T, self.steps = net, V, T, steps + self.guidance = guidance + + def forward(self, tokens, seed): + B = 1 + # MotionGenerator draws seed ~ N(0, 0.5) — rescale to unit noise. + x = seed.reshape(B, self.T, C6) * 2.0 + uncond = torch.zeros_like(tokens) + for i in range(self.steps): + t = torch.full((B,), i / self.steps, dtype=x.dtype, + device=x.device) + vc = self.net(x, t, tokens) + vu = self.net(x, t, uncond) + v = vu + self.guidance * (vc - vu) + x = x + v / self.steps + q = d6_to_quat(x.reshape(B, self.T, J, D6)) # [1,T,J,4] + zeros3 = torch.zeros(B, self.T, J, 3, dtype=x.dtype, device=x.device) + ones3 = torch.ones(B, self.T, J, 3, dtype=x.dtype, device=x.device) + motion = torch.cat([zeros3, q, ones3], -1) # [1,T,J,10] + return motion.reshape(B, self.T, J * 10) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--data", default="/tmp/t2m_v5.npz") + ap.add_argument("--out", default="/tmp/t2m_v5_flow") + ap.add_argument("--epochs", type=int, default=60) + ap.add_argument("--batch", type=int, default=256) + ap.add_argument("--lr", type=float, default=2e-4) + ap.add_argument("--dim", type=int, default=256) + ap.add_argument("--layers", type=int, default=6) + ap.add_argument("--steps", type=int, default=16, help="Euler export steps") + # Guidance default 1.0: with posture-FILTERED training data (already + # more upright than the uncond distribution), CFG > 1 extrapolates PAST + # upright into a backward arch — measured on the v5.1 retrain (g2.0 + # arched, g1.0 clean). Raise only for conditioning-starved datasets. + ap.add_argument("--guidance", type=float, default=1.0, + help="CFG scale baked into the exported sampler") + ap.add_argument("--device", default="mps") + ap.add_argument("--resume", action="store_true", + help="resume from /ckpt.pt (long runs survive " + "sleep/restarts)") + a = ap.parse_args() + + d = np.load(a.data, allow_pickle=True) + mo, msk, tk = d["mo"], d["msk"], d["tk"] + vocab = [str(w) for w in d["vocab"]] + fps = int(d["fps"]) + canon_rd = d["canonRestDir"] + N, T = mo.shape[0], mo.shape[1] + V = len(vocab) + print(f"data: {N} windows T={T} V={V} vocab={vocab}") + + dev = torch.device(a.device if (a.device != "mps" + or torch.backends.mps.is_available()) + else "cpu") + x1 = quat_to_6d(torch.from_numpy(mo)).reshape(N, T, C6) # data + m6 = torch.from_numpy(msk).repeat_interleave(D6, -1) # [N,C6] + tok = torch.from_numpy(tk) + + # class-balanced sampling — walk is 64% of windows (v4 lesson) + freq = tk.sum(0) + w = (tk @ (1.0 / np.maximum(freq, 1.0))).astype(np.float64) + sampler = torch.utils.data.WeightedRandomSampler( + torch.from_numpy(w), num_samples=N, replacement=True) + ds = torch.utils.data.TensorDataset(x1, m6, tok) + dl = torch.utils.data.DataLoader(ds, batch_size=a.batch, sampler=sampler, + drop_last=True) + + net = FlowDiT(V, T, dim=a.dim, layers=a.layers).to(dev) + print("params:", sum(p.numel() for p in net.parameters()) / 1e6, "M") + opt = torch.optim.AdamW(net.parameters(), lr=a.lr, weight_decay=1e-4) + sched = torch.optim.lr_scheduler.CosineAnnealingLR( + opt, T_max=a.epochs * max(1, len(dl))) + + os.makedirs(a.out, exist_ok=True) + ckpt_path = os.path.join(a.out, "ckpt.pt") + start_ep = 0 + if a.resume and os.path.exists(ckpt_path): + # weights_only=True: the checkpoint is only state_dicts + an int + # epoch — no pickled objects — so load safely (no code execution). + ck = torch.load(ckpt_path, map_location=dev, weights_only=True) + net.load_state_dict(ck["net"]) + opt.load_state_dict(ck["opt"]) + sched.load_state_dict(ck["sched"]) + start_ep = ck["epoch"] + 1 + print(f"resumed from epoch {start_ep}", flush=True) + + for ep in range(start_ep, a.epochs): + tot, nb = 0.0, 0 + for xb, mb, tb in dl: + xb, mb, tb = xb.to(dev), mb.to(dev), tb.to(dev) + x0 = torch.randn_like(xb) + # classifier-free guidance: drop the action condition 10% of the + # time so the sampler can extrapolate cond vs uncond at export. + drop = (torch.rand(tb.shape[0], device=dev) < 0.1).float() + tb = tb * (1.0 - drop)[:, None] + t = torch.rand(xb.shape[0], device=dev) + xt = (1 - t[:, None, None]) * x0 + t[:, None, None] * xb + v = net(xt, t, tb) + tgt = xb - x0 + mask = mb[:, None, :] # [B,1,C6] + loss = ((v - tgt) ** 2 * mask).sum() / mask.sum() / T + opt.zero_grad(set_to_none=True) + loss.backward() + torch.nn.utils.clip_grad_norm_(net.parameters(), 1.0) + opt.step() + sched.step() + tot += loss.item(); nb += 1 + print(f"ep {ep + 1}/{a.epochs} loss {tot / max(1, nb):.4f}", flush=True) + torch.save({"net": net.state_dict(), "opt": opt.state_dict(), + "sched": sched.state_dict(), "epoch": ep}, ckpt_path) + + torch.save(net.state_dict(), os.path.join(a.out, "flow.pt")) + + # ---- export: sampler-unrolled ONNX + vocab json ---- + net_cpu = FlowDiT(V, T, dim=a.dim, layers=a.layers) + net_cpu.load_state_dict({k: v.cpu() for k, v in net.state_dict().items()}) + net_cpu.eval() + samp = Sampler(net_cpu, V, T, a.steps, a.guidance).eval() + Z = T * C6 + tokens = torch.zeros(1, V); tokens[0, 0] = 1.0 + seed = torch.randn(1, Z) * 0.5 + onnx_path = os.path.join(a.out, "t2m.onnx") + torch.onnx.export(samp, (tokens, seed), onnx_path, + input_names=["tokens", "seed"], + output_names=["motion"], opset_version=17, + dynamo=False) + vj = { + "vocab": vocab, "Z": Z, "T": T, "C": J * 10, "J": J, + "fps": fps, "frame": "world", "version": "v5-flow", + "flowSteps": a.steps, + # #858: the canonical reference triple — model clips ride the same + # bind-referenced direction retarget as v5 template clips. + "restWorld": [[0.0, 0.0, 0.0, 1.0]] * J, + "restDir": [[float(v) for v in row] for row in canon_rd], + } + with open(os.path.join(a.out, "t2m-vocab.json"), "w") as f: + json.dump(vj, f) + print(f"exported {onnx_path} " + f"({os.path.getsize(onnx_path) / 1e6:.1f} MB) + vocab") + + # sanity: run one sample through onnxruntime + try: + import onnxruntime as ort + s = ort.InferenceSession(onnx_path, + providers=["CPUExecutionProvider"]) + out = s.run(None, {"tokens": tokens.numpy(), + "seed": seed.numpy()})[0] + q = out[0, :, 3:7] + nrm = np.linalg.norm(out[0].reshape(T, J, 10)[..., 3:7], axis=-1) + print("onnx ok:", out.shape, "quat norms", + nrm.min().round(4), nrm.max().round(4)) + except Exception as e: # noqa: BLE001 + print("onnx check failed:", e) + + +if __name__ == "__main__": + main() diff --git a/scripts/upload-t2m-v5-model.sh b/scripts/upload-t2m-v5-model.sh new file mode 100755 index 00000000..9c7c06fb --- /dev/null +++ b/scripts/upload-t2m-v5-model.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# Upload the v5 flow-matching text-to-motion model to the QtMeshEditor HF +# models repo (#840/#858, epic #837). ONE-TIME, run by a maintainer with +# write access. +# +# The app downloads these on first use from +# https://huggingface.co/fernandotonon/QtMeshEditor-models/resolve/main/motion/t2m.onnx +# https://huggingface.co/fernandotonon/QtMeshEditor-models/resolve/main/motion/t2m-vocab.json +# The v5 ONNX keeps the exact v4 runtime interface (tokens[1,V], seed[1,Z] +# -> motion[1,T,220]) so replacing the files is backward-compatible: older +# builds run the new model too (they ignore the vocab's restWorld/restDir +# and fall back to the synthetic-standing-pose path). The previous v4 files +# are preserved under versioned names for rollback. +# +# Prereqs: +# pip install -U "huggingface_hub[cli]" +# huggingface-cli login # token with write access +# train-t2m-flow-v5.py already run -> OUT_DIR holds t2m.onnx + t2m-vocab.json +# +# Usage: +# OUT_DIR=/tmp/t2m_v5_flow ./scripts/upload-t2m-v5-model.sh +set -euo pipefail + +REPO="${REPO:-fernandotonon/QtMeshEditor-models}" +OUT_DIR="${OUT_DIR:?set OUT_DIR to the training output dir (t2m.onnx + t2m-vocab.json)}" + +for f in t2m.onnx t2m-vocab.json; do + [[ -f "$OUT_DIR/$f" ]] || { echo "missing $OUT_DIR/$f" >&2; exit 1; } +done + +# keep the previous (v4 CVAE) files for rollback under versioned names. +# IDEMPOTENT: only back up if the v4 rollback does NOT already exist — +# otherwise a re-run would overwrite the real v4 with the current (v5) file +# and destroy the rollback point. +TMP=$(mktemp -d) +for f in t2m.onnx t2m-vocab.json; do + case "$f" in + t2m.onnx) dst="motion/t2m-v4.onnx" ;; + t2m-vocab.json) dst="motion/t2m-vocab-v4.json" ;; + esac + if huggingface-cli download "$REPO" "$dst" --local-dir "$TMP" >/dev/null 2>&1; then + echo "rollback $dst already exists — skipping backup (idempotent)" + continue + fi + if huggingface-cli download "$REPO" "motion/$f" --local-dir "$TMP" >/dev/null 2>&1; then + huggingface-cli upload "$REPO" "$TMP/motion/$f" "$dst" \ + --commit-message "t2m: preserve v4 CVAE as $dst before the v5 flow model (#858)" + fi +done + +huggingface-cli upload "$REPO" "$OUT_DIR/t2m.onnx" motion/t2m.onnx \ + --commit-message "t2m v5: flow-matching model, canonical reference triple (#840/#858)" +huggingface-cli upload "$REPO" "$OUT_DIR/t2m-vocab.json" motion/t2m-vocab.json \ + --commit-message "t2m v5: vocab + canonical reference triple (#840/#858)" +echo "uploaded v5 t2m model + vocab to $REPO/motion/" diff --git a/src/AnimationControlController.cpp b/src/AnimationControlController.cpp index ba3ab5bb..ef2fbb1c 100644 --- a/src/AnimationControlController.cpp +++ b/src/AnimationControlController.cpp @@ -1839,7 +1839,7 @@ double AnimationControlController::currentArmSpace(const QString& animName, QVariantMap AnimationControlController::generateMotion(const QString& prompt, double duration, bool useModel, - double armSpaceDeg) + double armSpaceDeg, bool footPin) { QVariantMap out; out["ok"] = false; @@ -1883,10 +1883,18 @@ QVariantMap AnimationControlController::generateMotion(const QString& prompt, if (mr.ok) { action = mr.matchedAction; quats = mr.clip.quats; fps = mr.clip.fps; worldFrame = mr.worldFrame; clipSource = QStringLiteral("model"); gotClip = true; - // Borrow a template clip's reference directions so the - // retarget synthesizes a BIND-referenced base pose (no - // harvest from the rig's other animations). - clipDirs = MotionLibrary::referenceDirsForPrompt(prompt); + if (!mr.clip.restWorld.empty() && !mr.clip.restDir.empty()) { + // v5 models (#858) ship their canonical reference triple + // — same bind-referenced retarget as template clips. + cmuRest = mr.clip.restWorld; + clipDirs = mr.clip.restDir; + } else { + // Legacy v4: borrow a template clip's reference + // directions so the retarget synthesizes a BIND- + // referenced base pose (no harvest from the rig's + // other animations). + clipDirs = MotionLibrary::referenceDirsForPrompt(prompt); + } } } if (!gotClip) @@ -1934,15 +1942,29 @@ QVariantMap AnimationControlController::generateMotion(const QString& prompt, worldFrame, cmuRest, /*refineWithModel=*/false, /*refineStride=*/8, yaw180, - clipDirs); + clipDirs, + clipSource == QStringLiteral("model")); if (!res.ok) return fail(res.error); out["source"] = clipSource; + // #837 quality post-pass: sparse-bake temporal low-pass (removes + // retarget trembling). Before arm-space/foot-pin so pins stay exact. + AnimationMerger::smoothBakeAnimation(skel.get(), animName, 12, fps); + // #854: optional Mixamo-style arm-space post-process. if (std::abs(armSpaceDeg) > 1e-4) AnimationMerger::adjustArmSpace(skel.get(), animName, static_cast(armSpaceDeg)); + // #856: foot-contact cleanup — ON by default (checkbox opts out). + if (footPin) { + const auto fp = AnimationMerger::pinFeet(skel.get(), animName); + if (fp.ok && fp.spans > 0) + out["footPinSpans"] = fp.spans; + else if (!fp.ok && !fp.error.isEmpty()) + out["footPinError"] = fp.error; // surface why (no leg tracks, etc.) + } + entity->refreshAvailableAnimationState(); // Make the generated clip the ONLY enabled animation. Ogre AVERAGES all // enabled animation states, so leaving the import's auto-enabled clip (or diff --git a/src/AnimationControlController.h b/src/AnimationControlController.h index 41e3d138..7b6b35dd 100644 --- a/src/AnimationControlController.h +++ b/src/AnimationControlController.h @@ -343,10 +343,13 @@ class AnimationControlController : public QObject /// (MotionGenerator/ONNX); it falls back to the template library automatically /// when the model is unavailable or the action isn't in its vocab. Default /// false = the reliable template-clip retarget. + /// `footPin` (default true) runs the #856 foot-contact cleanup on the + /// generated clip (contact detection + two-bone IK pinning). Q_INVOKABLE QVariantMap generateMotion(const QString& prompt, double duration = 0.0, bool useModel = false, - double armSpaceDeg = 0.0); + double armSpaceDeg = 0.0, + bool footPin = true); /// #854: Mixamo-style arm-space post-process on an EXISTING animation of /// the selected entity. Positive `degrees` widens the arms away from the diff --git a/src/AnimationMerger.cpp b/src/AnimationMerger.cpp index 921e2656..698ddbd3 100644 --- a/src/AnimationMerger.cpp +++ b/src/AnimationMerger.cpp @@ -1,5 +1,6 @@ #include "AnimationMerger.h" #include "AutoRig.h" +#include "FootContact.h" #include "MotionInbetween.h" #include #include @@ -1470,6 +1471,274 @@ bool AnimationMerger::adjustArmSpace(Ogre::Skeleton* skel, return true; } +int AnimationMerger::smoothBakeAnimation(Ogre::Skeleton* skel, + const std::string& animName, + int sparseFps, int targetFps) +{ + if (!skel || sparseFps <= 0 || targetFps <= 0) return 0; + if (sparseFps >= targetFps) return 0; // nothing to low-pass + if (bakeAnimationAtFps(skel, animName, sparseFps) == 0) return 0; + return bakeAnimationAtFps(skel, animName, targetFps); +} + +namespace { +std::string footPinKey(const std::string& animName) +{ + return "qtme.footpin." + animName; +} +} // namespace + +AnimationMerger::FootPinResult AnimationMerger::pinFeet( + Ogre::Skeleton* skel, const std::string& animName, int blendFrames) +{ + FootPinResult res; + if (!skel || !skel->hasAnimation(animName)) { + res.error = QStringLiteral("animation not found"); + return res; + } + Ogre::Animation* anim = skel->getAnimation(animName); + + const int nBones = static_cast(skel->getNumBones()); + std::vector boneToCanon(static_cast(nBones), -1); + for (int i = 0; i < nBones; ++i) + boneToCanon[static_cast(i)] = + MotionInbetween::canonicalIndexForBone(QString::fromStdString( + skel->getBone(static_cast(i))->getName())); + const TargetBindFrame tb = readTargetBindFrame(skel, boneToCanon); + // Bind-local POSITIONS (parent-relative) for the manual FK below — the + // skeleton is still in its reset pose after readTargetBindFrame. + std::vector bindLocalPos(static_cast(nBones)); + for (int i = 0; i < nBones; ++i) + bindLocalPos[static_cast(i)] = + skel->getBone(static_cast(i))->getPosition(); + + // Leg chains: thigh / knee / foot roles per side (first bone per role). + struct Leg { int thigh, shin, foot; }; + const Leg legs[2] = { + {tb.roleBoneIdx[15], tb.roleBoneIdx[16], tb.roleBoneIdx[17]}, // right + {tb.roleBoneIdx[19], tb.roleBoneIdx[20], tb.roleBoneIdx[21]}, // left + }; + + auto trackOf = [&](int bone) -> Ogre::NodeAnimationTrack* { + if (bone < 0 || !anim->hasNodeTrack(static_cast(bone))) + return nullptr; + auto* t = anim->getNodeTrack(static_cast(bone)); + return (t && t->getNumKeyFrames() > 0) ? t : nullptr; + }; + + // Keyframe times come from the first available thigh track — generated + // clips write one keyframe per frame on every mapped bone. + Ogre::NodeAnimationTrack* timeSrc = trackOf(legs[0].thigh); + if (!timeSrc) timeSrc = trackOf(legs[1].thigh); + if (!timeSrc) { + res.error = QStringLiteral("no leg tracks on this rig/animation"); + return res; + } + const int nk = static_cast(timeSrc->getNumKeyFrames()); + if (nk < 4) { + res.error = QStringLiteral("too few keyframes to detect contacts"); + return res; + } + std::vector times(static_cast(nk)); + for (int k = 0; k < nk; ++k) + times[static_cast(k)] = + timeSrc->getNodeKeyFrame(static_cast(k))->getTime(); + + // ---- manual FK over the tracks (pure math; the live skeleton — which + // may be a SkeletonInstance driving an on-screen entity — is untouched). + std::vector> Wrot( + static_cast(nk), + std::vector(static_cast(nBones))); + std::vector> Wpos( + static_cast(nk), + std::vector(static_cast(nBones))); + for (int k = 0; k < nk; ++k) { + const Ogre::TimeIndex ti = + anim->_getTimeIndex(times[static_cast(k)]); + for (int i : tb.order) { + Ogre::Quaternion lrot = tb.bindLocal[static_cast(i)]; + Ogre::Vector3 lpos = bindLocalPos[static_cast(i)]; + if (auto* trk = trackOf(i)) { + Ogre::TransformKeyFrame kf(nullptr, 0.0f); + trk->getInterpolatedKeyFrame(ti, &kf); + lrot = lrot * kf.getRotation(); // applyToNode: rotate() + lpos = lpos + kf.getTranslate(); // translate(TS_PARENT) + } + const int pi = tb.parentIdx[static_cast(i)]; + if (pi >= 0) { + Wrot[static_cast(k)][static_cast(i)] = + Wrot[static_cast(k)][static_cast(pi)] * lrot; + Wpos[static_cast(k)][static_cast(i)] = + Wpos[static_cast(k)][static_cast(pi)] + + Wrot[static_cast(k)][static_cast(pi)] * lpos; + } else { + Wrot[static_cast(k)][static_cast(i)] = lrot; + Wpos[static_cast(k)][static_cast(i)] = lpos; + } + } + } + + // Detection runs in the CANONICAL frame (Ct maps rig world → X=left, + // Y=up, Z=forward) so "ground" is a horizontal plane regardless of the + // rig's own axes. + auto toCanon = [&](const Ogre::Vector3& p) { return tb.Ct * p; }; + auto fromCanon = [&](const FootContact::V3& p) { + return tb.Ct.Inverse() * Ogre::Vector3(p[0], p[1], p[2]); + }; + auto v3 = [](const Ogre::Vector3& p) { + return FootContact::V3{p.x, p.y, p.z}; + }; + + for (const Leg& leg : legs) { + if (leg.thigh < 0 || leg.shin < 0 || leg.foot < 0) + continue; + auto* thighTrk = trackOf(leg.thigh); + auto* shinTrk = trackOf(leg.shin); + auto* footTrk = trackOf(leg.foot); + if (!thighTrk || !shinTrk + || static_cast(thighTrk->getNumKeyFrames()) != nk + || static_cast(shinTrk->getNumKeyFrames()) != nk + || (footTrk && static_cast(footTrk->getNumKeyFrames()) != nk)) + continue; // mixed keyframe grids — not a generated clip + // The rewrite below indexes shin/foot keyframes by k and assumes they + // share timeSrc's times. Count alone isn't enough — an authored clip + // can have equal counts at DIFFERENT times, which would write a + // correction computed at time t onto a keyframe at t'. Verify the + // grids actually align (generated clips do by construction). + { + bool aligned = true; + for (int k = 0; k < nk && aligned; ++k) { + const float t = times[static_cast(k)]; + if (std::abs(thighTrk->getNodeKeyFrame( + static_cast(k))->getTime() - t) > 1e-4f + || std::abs(shinTrk->getNodeKeyFrame( + static_cast(k))->getTime() - t) > 1e-4f + || (footTrk && std::abs(footTrk->getNodeKeyFrame( + static_cast(k))->getTime() - t) > 1e-4f)) + aligned = false; + } + if (!aligned) continue; // keyframe times don't match — skip leg + } + + std::vector footC(static_cast(nk)); + for (int k = 0; k < nk; ++k) + footC[static_cast(k)] = v3(toCanon( + Wpos[static_cast(k)][static_cast(leg.foot)])); + const float legLen = + (Wpos[0][static_cast(leg.shin)] + - Wpos[0][static_cast(leg.thigh)]).length() + + (Wpos[0][static_cast(leg.foot)] + - Wpos[0][static_cast(leg.shin)]).length(); + auto spans = FootContact::detectContacts(footC, legLen); + // Coverage guard: a single "contact" spanning most of the clip is a + // misdetection (loose thresholds on a moving clip) — pinning it + // freezes the leg for the whole animation. Genuine stance phases in + // gait are well under this. + spans.erase(std::remove_if(spans.begin(), spans.end(), + [&](const FootContact::Span& sp) { + return (sp.end - sp.start + 1) + > (nk * 55) / 100; + }), + spans.end()); + if (spans.empty()) + continue; + res.spans += static_cast(spans.size()); + + for (const auto& span : spans) { + const FootContact::V3 anchor = footC[static_cast(span.start)]; + for (int k = span.start; k <= span.end; ++k) { + const float w = FootContact::blendWeight(span, k, blendFrames); + if (w <= 0.0f) + continue; + const size_t kc = static_cast(k); + const Ogre::Vector3 hipW = Wpos[kc][static_cast(leg.thigh)]; + const Ogre::Vector3 kneeW = Wpos[kc][static_cast(leg.shin)]; + const Ogre::Vector3 footW = Wpos[kc][static_cast(leg.foot)]; + // pinned target: blend the current foot toward the anchor + const FootContact::V3 cur = footC[kc]; + const FootContact::V3 tgtC{ + cur[0] + (anchor[0] - cur[0]) * w, + cur[1] + (anchor[1] - cur[1]) * w, + cur[2] + (anchor[2] - cur[2]) * w}; + const FootContact::V3 kneeNewC = FootContact::solveKnee( + v3(toCanon(hipW)), v3(toCanon(kneeW)), v3(toCanon(footW)), + tgtC); + const Ogre::Vector3 kneeNew = fromCanon(kneeNewC); + const Ogre::Vector3 tgt = fromCanon(tgtC); + + Ogre::Vector3 dThighOld = kneeW - hipW; + Ogre::Vector3 dThighNew = kneeNew - hipW; + Ogre::Vector3 dShinOld = footW - kneeW; + Ogre::Vector3 dShinNew = tgt - kneeNew; + if (dThighOld.squaredLength() < 1e-12f + || dThighNew.squaredLength() < 1e-12f + || dShinOld.squaredLength() < 1e-12f + || dShinNew.squaredLength() < 1e-12f) + continue; + dThighOld.normalise(); dThighNew.normalise(); + dShinOld.normalise(); dShinNew.normalise(); + const Ogre::Quaternion S1 = dThighOld.getRotationTo(dThighNew); + const Ogre::Quaternion S2 = + (S1 * dShinOld).getRotationTo(dShinNew); + + // Rewrite the three keyframes as world premultipliers folded + // into parent-relative deltas (keyframes compose as + // local = bindLocal · kf; world = Wp · local). + const Ogre::Quaternion& WpT = + (tb.parentIdx[static_cast(leg.thigh)] >= 0) + ? Wrot[kc][static_cast( + tb.parentIdx[static_cast(leg.thigh)])] + : Ogre::Quaternion::IDENTITY; + const Ogre::Quaternion Wthigh = + Wrot[kc][static_cast(leg.thigh)]; + const Ogre::Quaternion WthighNew = S1 * Wthigh; + auto* kfT = thighTrk->getNodeKeyFrame( + static_cast(k)); + kfT->setRotation( + tb.bindLocal[static_cast(leg.thigh)].Inverse() + * WpT.Inverse() * WthighNew); + + const Ogre::Quaternion& WpS = + Wrot[kc][static_cast( + tb.parentIdx[static_cast(leg.shin)])]; + const Ogre::Quaternion Wshin = + Wrot[kc][static_cast(leg.shin)]; + const Ogre::Quaternion WshinNew = S2 * S1 * Wshin; + auto* kfS = shinTrk->getNodeKeyFrame( + static_cast(k)); + kfS->setRotation( + tb.bindLocal[static_cast(leg.shin)].Inverse() + * (S1 * WpS).Inverse() * WshinNew); + + if (footTrk) { + // foot keeps its ORIGINAL world orientation (no toe pop + // inherited from the parent corrections) + const Ogre::Quaternion& WpF = + Wrot[kc][static_cast( + tb.parentIdx[static_cast(leg.foot)])]; + auto* kfF = footTrk->getNodeKeyFrame( + static_cast(k)); + kfF->setRotation( + tb.bindLocal[static_cast(leg.foot)].Inverse() + * (S2 * S1 * WpF).Inverse() + * Wrot[kc][static_cast(leg.foot)]); + } + ++res.keyframesAdjusted; + } + } + // setRotation does NOT invalidate the track caches (#854 lesson). + thighTrk->_keyFrameDataChanged(); + shinTrk->_keyFrameDataChanged(); + if (footTrk) footTrk->_keyFrameDataChanged(); + } + + if (skel->getNumBones() > 0) + skel->getBone(0)->getUserObjectBindings().setUserAny( + footPinKey(animName), Ogre::Any(true)); + res.ok = true; + return res; +} + AnimationMerger::ApplyMotionResult AnimationMerger::applyMotionClip( Ogre::Skeleton* skel, const std::string& animName, @@ -1480,7 +1749,8 @@ AnimationMerger::ApplyMotionResult AnimationMerger::applyMotionClip( bool refineWithModel, int refineStride, bool yaw180, - const std::vector>& clipRestDir) + const std::vector>& clipRestDir, + bool modelClip) { ApplyMotionResult res; if (!skel) { res.error = QStringLiteral("no skeleton"); return res; } @@ -1570,9 +1840,13 @@ AnimationMerger::ApplyMotionResult AnimationMerger::applyMotionClip( // stored angle from a prior generation of the same name would make a // re-request of that same angle a no-op (delta 0) on the new keyframes — // forget it. - if (skel->getNumBones() > 0) + if (skel->getNumBones() > 0) { skel->getBone(0)->getUserObjectBindings().eraseUserAny( armSpaceKey(animName)); + // #856: same for the foot-pin marker — a regenerated clip is unpinned. + skel->getBone(0)->getUserObjectBindings().eraseUserAny( + "qtme.footpin." + animName); + } Ogre::Animation* anim = skel->createAnimation(animName, length); anim->setInterpolationMode(Ogre::Animation::IM_LINEAR); anim->setRotationInterpolationMode(Ogre::Animation::RIM_LINEAR); @@ -1587,13 +1861,17 @@ AnimationMerger::ApplyMotionResult AnimationMerger::applyMotionClip( // Generated clips therefore reference the TARGET BIND — no pose from any // other animation is involved (the standing-pose harvest below is only // the fallback for restWorld-less clips, e.g. the CMU-built libraries). + // A reference orientation is PRESENT when it is a valid (unit-norm) quat; + // unresolved roles in dumps are zero-filled. Identity counts — the v5 + // model's canonical triple (#858) is restWorld = identity ×22 by + // construction, and treating it as "absent" would silently demote model + // clips to the synthetic-standing-pose path. bool haveRestWorld = false; if (worldFrame && cmuRestWorld.size() == static_cast( MotionInbetween::canonicalJointCount())) { for (const auto& q : cmuRestWorld) - if (std::abs(q[0]) > 1e-5f || std::abs(q[1]) > 1e-5f - || std::abs(q[2]) > 1e-5f || std::abs(1.f - std::abs(q[3])) > 1e-5f) { + if (q[0] * q[0] + q[1] * q[1] + q[2] * q[2] + q[3] * q[3] > 0.25f) { haveRestWorld = true; break; } @@ -1643,6 +1921,146 @@ AnimationMerger::ApplyMotionResult AnimationMerger::applyMotionClip( const auto& q = clipQuats[frame][joint]; return Ogre::Quaternion(q[3], q[0], q[1], q[2]); // (w,x,y,z) }; + // #857: STABILIZED TWIST TRANSPORT. The aim above transports the + // bone's DIRECTION but inherits the target bind's roll — gesture- + // heavy clips (dance, wave) lose forearm/spine roll and read + // flat. Decompose the source's frame rotation into swing (the + // direction, already transported) + twist about the bone axis: + // Δ(f) = Ws(f) · Ws_bind⁻¹ (canonical axes) + // aim(f) = arc(d_bind → d(f)) + // θ(f) = signed angle of aim(f)⁻¹·Δ(f) about d_bind + // θ is wrapped to (−π,π] then UNWRAPPED across frames (a ±180° + // pop near the shortest-arc degeneracy would otherwise flip the + // roll direction mid-clip once a gain ≠ 1 scales it) and composed + // on the target about the SAME pole the source decomposed about: + // Qbase = arc(dt_bind → d_ref) · Wt_bind (once per bone) + // Wt(f) = twist(θ·gain, ds) · arc(d_ref → ds) · Qbase + // Recomposing per-frame from the TARGET BIND direction instead + // (the pre-#857 form) decomposes about a different pole than the + // source did — the roll component "between" the two poles leaks + // into swing and inflates amplitude (measured: Mixamo self-parity + // arm amp +38%, elbow error 3.3°→6.3°). With matched poles the + // per-frame relative motion is the source's world delta exactly, + // conjugated into target axes — direction stays absolute, roll + // transports losslessly, self-parity drops below the aim-only + // baseline. Per-role gains: collars damped (they share the + // shoulder line's roll), everything else transports fully. + static const float kTwistGain[22] = { + 1.f, 1.f, 1.f, 1.f, 1.f, 1.f, // spine chain + head + 0.5f, 1.f, 1.f, 1.f, // rcollar damped, right arm + 0.5f, 1.f, 1.f, 1.f, // lcollar damped, left arm + 1.f, 1.f, 1.f, 1.f, // right leg + foot + 1.f, 1.f, 1.f, 1.f }; // left leg + foot + // Per-role twist caps (radians). The single generous 150° cap + // let noisy source roll through on spine/neck/head — takes whose + // roll was invisible pre-#857 (dropped) came back with flailing + // arms and a thrown-back head. Roll matters most on forearms; + // the axial chain needs very little. Hip keeps a wide cap: it + // carries genuine facing turns (salsa). + // MODEL clips get tight per-role caps (the from-scratch model's + // roll is noisy — uncapped it flails). AUTHORED/self-parity clips + // carry legitimate large roll and must NOT be capped (measured: + // caps collapse mouse elbow 180°→124°, parity 5.1°→3.1° off). + static const float kTwistCapModel[22] = { + 2.62f, 0.79f, 0.79f, 0.52f, 0.52f, 0.52f, // hip, spine 45°, neck/head 30° + 0.52f, 1.57f, 1.57f, 1.57f, // rcollar 30°, right arm 90° + 0.52f, 1.57f, 1.57f, 1.57f, // lcollar 30°, left arm 90° + 1.05f, 1.05f, 1.05f, 0.79f, // right leg 60°, foot 45° + 1.05f, 1.05f, 1.05f, 0.79f }; // left leg 60°, foot 45° + static const float kTwistCapOpen[22] = { + 3.15f, 3.15f, 3.15f, 3.15f, 3.15f, 3.15f, + 3.15f, 3.15f, 3.15f, 3.15f, + 3.15f, 3.15f, 3.15f, 3.15f, + 3.15f, 3.15f, 3.15f, 3.15f, + 3.15f, 3.15f, 3.15f, 3.15f }; + const float* kTwistCapRole = modelClip ? kTwistCapModel + : kTwistCapOpen; + std::vector> twistTheta( + static_cast(frames), + std::vector(static_cast(Jc), 0.0f)); + { + constexpr float kTau = 2.0f * static_cast(M_PI); + std::vector prev(static_cast(Jc), 0.0f); + std::vector has(static_cast(Jc), 0); + for (int f = 0; f < frames; ++f) + for (int c = 0; c < Jc; ++c) { + const Ogre::Vector3& as = + srcLocalAxis[static_cast(c)]; + if (as.squaredLength() <= 1e-8f) continue; + const auto& rq = cmuRestWorld[static_cast(c)]; + const Ogre::Quaternion restQ(rq[3], rq[0], rq[1], rq[2]); + const auto& sd = clipRestDir[static_cast(c)]; + Ogre::Vector3 dbind(sd[0], sd[1], sd[2]); + dbind.normalise(); + const Ogre::Quaternion Wf = clipQ(f, c); + const Ogre::Vector3 d = Wf * as; + const Ogre::Quaternion delta = Wf * restQ.Inverse(); + const Ogre::Quaternion aim = dbind.getRotationTo(d); + const Ogre::Quaternion tw = aim.Inverse() * delta; + float th = 2.0f * std::atan2( + tw.x * dbind.x + tw.y * dbind.y + tw.z * dbind.z, + tw.w); + th = std::remainder(th, kTau); + if (has[static_cast(c)]) + th += kTau * std::round( + (prev[static_cast(c)] - th) / kTau); + prev[static_cast(c)] = th; + has[static_cast(c)] = 1; + twistTheta[static_cast(f)] + [static_cast(c)] = std::clamp( + th, -kTwistCapRole[c], kTwistCapRole[c]); + } + // Low-pass the twist trajectories (5-tap binomial). θ comes + // from per-frame shortest-arc decompositions and jitters near + // degenerate aims — transporting it raw renders as trembling. + // Directions are untouched; only the roll is smoothed. + if (frames >= 5) { + for (int c = 0; c < Jc; ++c) { + std::vector src(static_cast(frames)); + for (int f = 0; f < frames; ++f) + src[static_cast(f)] = + twistTheta[static_cast(f)] + [static_cast(c)]; + static const float k[5] = {1.f, 4.f, 6.f, 4.f, 1.f}; + for (int f = 0; f < frames; ++f) { + float acc = 0.0f, wsum = 0.0f; + for (int o = -2; o <= 2; ++o) { + const int j = f + o; + if (j < 0 || j >= frames) continue; + acc += k[o + 2] * src[static_cast(j)]; + wsum += k[o + 2]; + } + twistTheta[static_cast(f)] + [static_cast(c)] = acc / wsum; + } + } + } + } + // Reference-aligned roll baseline per bone: aim the target bind + // at the source's REFERENCE direction once, so every per-frame + // swing below decomposes about the source's own pole. + std::vector dref(static_cast(Jc), + Ogre::Vector3::ZERO); + for (int c = 0; c < Jc; ++c) { + const auto& sd = clipRestDir[static_cast(c)]; + Ogre::Vector3 v(sd[0], sd[1], sd[2]); + if (v.squaredLength() > 1e-8f) { + v.normalise(); + dref[static_cast(c)] = CtInv * v; + } + } + std::vector Qbase(static_cast(nBones), + Ogre::Quaternion::IDENTITY); + for (int i = 0; i < nBones; ++i) { + const int c = boneToCanon[i]; + if (c >= 0 && c < Jc + && srcLocalAxis[static_cast(c)].squaredLength() + > 1e-8f) + Qbase[static_cast(i)] = + tb.tgtBindDir[static_cast(c)] + .getRotationTo(dref[static_cast(c)]) + * tb.bindWorld[static_cast(i)]; + } std::vector tracks( static_cast(nBones), nullptr); for (int i = 0; i < nBones; ++i) @@ -1675,10 +2093,19 @@ AnimationMerger::ApplyMotionResult AnimationMerger::applyMotionClip( (clipQ(f, c) * srcLocalAxis[static_cast(c)]); const Ogre::Quaternion R = - tb.tgtBindDir[static_cast(c)] - .getRotationTo(ds); - const Ogre::Quaternion Wt = - R * tb.bindWorld[static_cast(i)]; + dref[static_cast(c)].getRotationTo(ds); + Ogre::Quaternion Wt = + R * Qbase[static_cast(i)]; + // #857: re-apply the source's roll about the aimed + // direction (the swing above deliberately dropped it). + const float th = twistTheta[static_cast(f)] + [static_cast(c)] + * kTwistGain[c]; + if (std::abs(th) > 1e-5f) { + Ogre::Vector3 axis = ds; + axis.normalise(); + Wt = Ogre::Quaternion(Ogre::Radian(th), axis) * Wt; + } local = Wp.Inverse() * Wt; W[static_cast(i)] = Wt; } else { diff --git a/src/AnimationMerger.h b/src/AnimationMerger.h index 543e72be..87834853 100644 --- a/src/AnimationMerger.h +++ b/src/AnimationMerger.h @@ -181,7 +181,14 @@ class AnimationMerger { bool refineWithModel = false, int refineStride = 8, bool yaw180 = false, - const std::vector>& clipRestDir = {}); + const std::vector>& clipRestDir = {}, + // #837: tight per-role twist caps damp the from-scratch MODEL's noisy + // roll (flailing arms / thrown-back head). Authored template + self- + // parity clips carry legitimate large roll (arms up to 180°); capping + // them collapses real motion (measured: mouse elbow 180°→124°, total + // parity 5.1°→3.1° with caps off). So default = relaxed; the model + // path passes modelClip=true to re-enable the tight caps. + bool modelClip = false); /// One skeletal animation extracted onto the 22-joint canonical skeleton /// (#839, the REVERSE of applyMotionClip's world-frame path): per frame, @@ -260,6 +267,47 @@ class AnimationMerger { const std::string& oldAnim, const std::string& newAnim); + /// #837 quality post-pass: re-grid the animation to a SPARSE keyframe + /// rate, then back to `targetFps` — a temporal low-pass that removes + /// retarget jitter ("trembling") while preserving the silhouette and the + /// clip length (both passes keep endpoints). Codifies the field-proven + /// trick of baking sparse and re-baking at 30 FPS. Returns the final + /// keyframe count (0 = animation missing / invalid fps). + static int smoothBakeAnimation(Ogre::Skeleton* skel, + const std::string& animName, + int sparseFps = 12, int targetFps = 30); + + /// Outcome of pinFeet. + struct FootPinResult { + bool ok = false; + QString error; + int spans = 0; ///< contact spans pinned (both feet) + int keyframesAdjusted = 0; + }; + + /// #856 — foot-contact cleanup. Retargeted clips slide/float feet on rigs + /// whose proportions differ from the source (the direction retarget + /// transfers bone DIRECTIONS, not world foot positions). Per foot role, + /// detect contact spans (foot near the clip's ground level AND nearly + /// stationary horizontally — FootContact::detectContacts, canonical-frame, + /// leg-length-scaled thresholds) and lock the foot's world position to its + /// span-start position with an analytic two-bone hip–knee–foot IK + /// (FootContact::solveKnee — keeps segment lengths and the pose's own + /// bend plane), blending in/out over `blendFrames` at span edges so knees + /// don't pop. Rewrites ONLY the thigh/shin/foot keyframes (foot keeps its + /// original world orientation); everything else untouched. Pure track + /// math — nothing is applied to the live skeleton. + /// + /// Effectively idempotent: a second run detects the already-planted spans + /// and re-pins to the same targets (near-no-op). The application is + /// recorded on bone[0]'s UserObjectBindings ("qtme.footpin.") so a + /// UI can reflect state; applyMotionClip clears it on clip regeneration. + /// Designed for generated clips (dense uniform keyframes); sparse + /// authored clips get keyframe-rate detection (approximate). + static FootPinResult pinFeet(Ogre::Skeleton* skel, + const std::string& animName, + int blendFrames = 3); + /// Sample every (or one) skeletal animation of `entity` at `fps` and /// express each canonical joint's world orientation per frame. Bone→role /// mapping is MotionInbetween::canonicalIndexForBone — the same matcher diff --git a/src/AnimationMerger_test.cpp b/src/AnimationMerger_test.cpp index e4f8a2ac..a7e8dbb1 100644 --- a/src/AnimationMerger_test.cpp +++ b/src/AnimationMerger_test.cpp @@ -1139,3 +1139,206 @@ TEST_F(AnimationMergerTest, ArmSpaceFollowsAnimationRename) "walk_wide"); EXPECT_GT(degBetween(wide, neutral), 15.0f); // it actually moved back } + +// ── #857: twist transport in the bind-referenced direction retarget ───────── + +namespace { +// Canonical-clip inputs for a virtual SOURCE rig whose bind is rotated 90° +// about Z (a deliberately foreign convention — exercises the restWorld +// conjugation): restDir = clean canonical T-pose directions, restWorld = the +// same non-identity quat everywhere, and identity motion W(f) = restQ. +constexpr int kJc = 22; +const Ogre::Quaternion kSrcRest(Ogre::Degree(90), Ogre::Vector3::UNIT_Z); + +std::vector> canonRestDirs() +{ + static const float d[kJc][3] = { + {0,1,0},{0,1,0},{0,1,0},{0,1,0},{0,1,0},{0,1,0}, + {-1,0,0},{-1,0,0},{-1,0,0},{-1,0,0}, + {1,0,0},{1,0,0},{1,0,0},{1,0,0}, + {0,-1,0},{0,-1,0},{0,-1,0},{0,0,1}, + {0,-1,0},{0,-1,0},{0,-1,0},{0,0,1}}; + std::vector> out(kJc); + for (int c = 0; c < kJc; ++c) out[c] = {d[c][0], d[c][1], d[c][2]}; + return out; +} + +std::vector> srcRestWorld() +{ + return std::vector>( + kJc, {kSrcRest.x, kSrcRest.y, kSrcRest.z, kSrcRest.w}); +} + +// frames of identity motion (every joint sits at the source bind) +std::vector>> identityClip(int frames) +{ + return std::vector>>( + frames, std::vector>( + kJc, {kSrcRest.x, kSrcRest.y, kSrcRest.z, kSrcRest.w})); +} + +// signed rotation angle of world-orientation delta `q` about unit axis `ax` +float twistDegAbout(const Ogre::Quaternion& q, const Ogre::Vector3& ax) +{ + const float s = q.x * ax.x + q.y * ax.y + q.z * ax.z; + return 2.0f * std::atan2(s, q.w) * 180.0f / static_cast(M_PI); +} +} // namespace + +TEST_F(AnimationMergerTest, TwistTransportCarriesBoneRoll) +{ + // makeArmRigEntity resolves only 9/22 roles — below applyMotionClip's + // humanoid gate (>= 11) — so build a fuller rig (13 roles: + hands/feet). + auto skelRes = Ogre::SkeletonManager::getSingleton().create( + "twist_roll_skel", + Ogre::ResourceGroupManager::DEFAULT_RESOURCE_GROUP_NAME); + unsigned short h = 0; + auto bone = [&](const char* n, const Ogre::Vector3& p, Ogre::Bone* par) { + auto* b = skelRes->createBone(n, h++); + b->setPosition(p); + if (par) par->addChild(b); + return b; + }; + auto* hips = bone("Hips", {0, 1.0f, 0}, nullptr); + auto* spine = bone("Spine", {0, 0.3f, 0}, hips); + bone("Head", {0, 0.4f, 0}, spine); + auto* lleg = bone("LeftUpLeg", {0.15f, -0.1f, 0}, hips); + bone("LeftFoot", {0, -0.8f, 0}, lleg); + auto* rleg = bone("RightUpLeg", {-0.15f, -0.1f, 0}, hips); + bone("RightFoot", {0, -0.8f, 0}, rleg); + auto* rsh = bone("RightArm", {-0.2f, 0.1f, 0}, spine); + auto* rfa = bone("RightForeArm", {-0.3f, 0, 0}, rsh); + bone("RightHand", {-0.25f, 0, 0}, rfa); + auto* lsh = bone("LeftArm", {0.2f, 0.1f, 0}, spine); + auto* lfa = bone("LeftForeArm", {0.3f, 0, 0}, lsh); + bone("LeftHand", {0.25f, 0, 0}, lfa); + skelRes->setBindingPose(); + auto mesh = createInMemoryMesh("twist_roll_mesh", skelRes); + auto* sm = Manager::getSingleton()->getSceneMgr(); + Ogre::Entity* ent = sm->createEntity("twist_roll_ent", mesh); + ASSERT_NE(ent, nullptr); + Ogre::SkeletonInstance* skel = ent->getSkeleton(); + + // Source: left upper arm (role 11, +X) ROLLS 60° about its own axis over + // the clip while its direction stays put. Pre-#857 the retarget dropped + // this entirely (aim-only) — the target bone never moved. + const int frames = 31; + auto quats = identityClip(frames); + for (int f = 0; f < frames; ++f) { + const float a = 60.0f * static_cast(f) / (frames - 1); + const Ogre::Quaternion w = + Ogre::Quaternion(Ogre::Degree(a), Ogre::Vector3::UNIT_X) + * kSrcRest; + quats[f][11] = {w.x, w.y, w.z, w.w}; + } + const auto res = AnimationMerger::applyMotionClip( + skel, "twistclip", quats, 30, /*worldFrame=*/true, srcRestWorld(), + false, 8, false, canonRestDirs()); + ASSERT_TRUE(res.ok) << res.error.toStdString(); + + // Direction is invariant under a pure roll… + const Ogre::Vector3 dir0(1, 0, 0); + skel->reset(true); + skel->getAnimation("twistclip")->apply(skel, 1.0f); + skel->_updateTransforms(); + const Ogre::Vector3 a = + skel->getBone("LeftArm")->_getDerivedPosition(); + const Ogre::Vector3 b = + skel->getBone("LeftForeArm")->_getDerivedPosition(); + EXPECT_GT((b - a).normalisedCopy().dotProduct(dir0), 0.99f); + + // …but the bone's world orientation now carries the 60° roll about it. + const Ogre::Quaternion w = + skel->getBone("LeftArm")->_getDerivedOrientation(); + EXPECT_NEAR(twistDegAbout(w, dir0), 60.0f, 4.0f); + + // Legs saw identity source motion — they must not pick up any rotation. + const Ogre::Quaternion leg = + skel->getBone("LeftUpLeg")->_getDerivedOrientation(); + EXPECT_NEAR(std::abs(leg.w), 1.0f, 1e-3f); + + sm->destroyEntity(ent); +} + +TEST_F(AnimationMergerTest, TwistUnwrapKeepsDampedCollarContinuous) +{ + // Rig WITH clavicles (Mixamo "Shoulder" → collar roles 6/10). + auto skelRes = Ogre::SkeletonManager::getSingleton().create( + "twist_collar_skel", + Ogre::ResourceGroupManager::DEFAULT_RESOURCE_GROUP_NAME); + unsigned short h = 0; + auto bone = [&](const char* n, const Ogre::Vector3& p, Ogre::Bone* par) { + auto* b = skelRes->createBone(n, h++); + b->setPosition(p); + if (par) par->addChild(b); + return b; + }; + auto* hips = bone("Hips", {0, 1.0f, 0}, nullptr); + auto* spine = bone("Spine", {0, 0.3f, 0}, hips); + bone("Head", {0, 0.4f, 0}, spine); + bone("LeftUpLeg", {0.15f, -0.1f, 0}, hips); + bone("RightUpLeg", {-0.15f, -0.1f, 0}, hips); + auto* lcol = bone("LeftShoulder", {0.05f, 0.25f, 0}, spine); + auto* larm = bone("LeftArm", {0.15f, 0, 0}, lcol); + bone("LeftForeArm", {0.3f, 0, 0}, larm); + auto* rcol = bone("RightShoulder", {-0.05f, 0.25f, 0}, spine); + auto* rarm = bone("RightArm", {-0.15f, 0, 0}, rcol); + bone("RightForeArm", {-0.3f, 0, 0}, rarm); + skelRes->setBindingPose(); + auto mesh = createInMemoryMesh("twist_collar_mesh", skelRes); + auto* sm = Manager::getSingleton()->getSceneMgr(); + Ogre::Entity* ent = sm->createEntity("twist_collar_ent", mesh); + ASSERT_NE(ent, nullptr); + Ogre::SkeletonInstance* skel = ent->getSkeleton(); + ASSERT_NE(skel, nullptr); + + // Source: left collar (role 10, +X) rolls 0 → 240° — past the ±180° wrap. + // The gain table damps collars to 0.5×, which is exactly where a missing + // unwrap explodes: wrapped −120° would scale to −60° instead of the + // capped +150°'s half — a mid-clip snap. + const int frames = 61; + auto quats = identityClip(frames); + for (int f = 0; f < frames; ++f) { + const float a = 240.0f * static_cast(f) / (frames - 1); + const Ogre::Quaternion w = + Ogre::Quaternion(Ogre::Degree(a), Ogre::Vector3::UNIT_X) + * kSrcRest; + quats[f][10] = {w.x, w.y, w.z, w.w}; + } + // modelClip=true: the per-role twist caps only apply on the model path + // (authored/self clips run uncapped since #837 review). This test asserts + // the capped collar behavior, so it exercises the model path explicitly. + const auto res = AnimationMerger::applyMotionClip( + skel, "collarclip", quats, 30, true, srcRestWorld(), + false, 8, false, canonRestDirs(), /*modelClip=*/true); + ASSERT_TRUE(res.ok) << res.error.toStdString(); + + // Sample densely: the collar's world orientation must move CONTINUOUSLY + // (no wrap snap) and end near 240° × 0.5 = 120° about +X. + Ogre::Quaternion prev = Ogre::Quaternion::IDENTITY; + float maxStepDeg = 0.0f; + Ogre::Quaternion last; + auto* anim = skel->getAnimation("collarclip"); + const float len = anim->getLength(); + for (int s = 0; s <= 60; ++s) { + skel->reset(true); + anim->apply(skel, len * static_cast(s) / 60.0f); + skel->_updateTransforms(); + const Ogre::Quaternion w = + skel->getBone("LeftShoulder")->_getDerivedOrientation(); + if (s > 0) { + const Ogre::Quaternion d = w * prev.Inverse(); + const float step = 2.0f * std::acos(std::min( + 1.0f, std::abs(d.w))) * 180.0f / static_cast(M_PI); + maxStepDeg = std::max(maxStepDeg, step); + } + prev = w; + last = w; + } + EXPECT_LT(maxStepDeg, 15.0f) << "collar roll snapped mid-clip (unwrap)"; + // 240° source twist hits the 150° runaway-unwrap cap FIRST, then the + // 0.5× collar gain: 150 × 0.5 = 75°. + EXPECT_NEAR(twistDegAbout(last, Ogre::Vector3::UNIT_X), 75.0f, 8.0f); + + sm->destroyEntity(ent); +} diff --git a/src/CLIPipeline.cpp b/src/CLIPipeline.cpp index 5e098e0a..94ee9287 100644 --- a/src/CLIPipeline.cpp +++ b/src/CLIPipeline.cpp @@ -2036,7 +2036,8 @@ int CLIPipeline::cmdFix(int argc, char* argv[]) int CLIPipeline::cmdAnimGenerate(const QString& filePath, const QString& prompt, float duration, const QString& outputPath, bool jsonOutput, bool useModel, - float armSpaceDeg) + float armSpaceDeg, bool footPin, + int smoothFps) { // #411 text-to-motion (template-clip MVP): match the prompt to a permissive // CMU motion clip from the downloadable library, retarget it onto the mesh's @@ -2075,12 +2076,21 @@ int CLIPipeline::cmdAnimGenerate(const QString& filePath, const QString& prompt, action = mr.matchedAction; quats = mr.clip.quats; fps = mr.clip.fps; - worldFrame = mr.worldFrame; // v4 models: world-frame quats - // Model clips carry no reference triple — borrow a template - // clip's canonical bone directions so the retarget can - // synthesize a BIND-referenced base pose instead of - // harvesting the rig's other animations (contamination). - clipDirs = MotionLibrary::referenceDirsForPrompt(prompt); + worldFrame = mr.worldFrame; // v4/v5 models: world-frame quats + if (!mr.clip.restWorld.empty() && !mr.clip.restDir.empty()) { + // v5 models (#858) ship their canonical reference triple + // in the vocab — the clip rides the same bind-referenced + // direction retarget as v5 template clips. + cmuRest = mr.clip.restWorld; + clipDirs = mr.clip.restDir; + } else { + // Legacy v4 models carry no reference triple — borrow a + // template clip's canonical bone directions so the + // retarget can synthesize a BIND-referenced base pose + // instead of harvesting the rig's other animations + // (contamination). + clipDirs = MotionLibrary::referenceDirsForPrompt(prompt); + } clipSource = QStringLiteral("model"); gotClip = true; } else { @@ -2148,12 +2158,24 @@ int CLIPipeline::cmdAnimGenerate(const QString& filePath, const QString& prompt, worldFrame, cmuRest, /*refineWithModel=*/false, /*refineStride=*/8, yaw180, - clipDirs); + clipDirs, + clipSource == QStringLiteral("model")); if (!res.ok) { err() << "Error: " << res.error << Qt::endl; return 1; } err() << "(source: " << clipSource << ")" << Qt::endl; + // #837 quality post-pass (ON by default, --no-smooth-bake disables, + // --smooth-fps N tunes): bake sparse -> re-bake at clip rate. A temporal + // low-pass that removes retarget trembling; runs BEFORE arm-space and + // foot pinning so the pin targets stay exact. + if (smoothFps > 0) { + if (AnimationMerger::smoothBakeAnimation(skel.get(), animName, + smoothFps, fps) > 0) + err() << "(smooth-bake: " << smoothFps << " -> " << fps + << " fps)" << Qt::endl; + } + // #854: optional Mixamo-style arm-space post-process before export. if (std::abs(armSpaceDeg) > 1e-4f) { if (AnimationMerger::adjustArmSpace(skel.get(), animName, armSpaceDeg)) { @@ -2166,6 +2188,19 @@ int CLIPipeline::cmdAnimGenerate(const QString& filePath, const QString& prompt, << Qt::endl; } + // #856: foot-contact cleanup — ON by default (--no-foot-pin disables). + if (footPin) { + const auto fp = AnimationMerger::pinFeet(skel.get(), animName); + if (fp.ok && fp.spans > 0) { + SentryReporter::addBreadcrumb( + QStringLiteral("ai.tool_call"), + QStringLiteral("foot_pin %1 spans").arg(fp.spans)); + err() << "(foot-pin: " << fp.spans << " contact span(s), " + << fp.keyframesAdjusted << " keyframes)" << Qt::endl; + } else if (!fp.ok) + err() << "Note: foot-pin skipped (" << fp.error << ")." << Qt::endl; + } + auto* node = entity->getParentSceneNode(); const QString fmt = formatForExtension(outputPath); if (MeshImporterExporter::exporter(node, QFileInfo(outputPath).absoluteFilePath(), fmt) != 0) { @@ -2213,12 +2248,17 @@ int CLIPipeline::cmdAnim(int argc, char* argv[]) bool inbetweenNoModel = false; // --no-model → force spline fallback bool dumpCanonicalMode = false; // #839: rig→canonical clip extraction QString dumpCanonicalPath; // --dump-canonical + bool applyCanonicalMode = false; // #837 parity harness: apply canonical json + QString applyCanonicalPath; // --apply-canonical bool generateMode = false; // #411: text-to-motion (template-clip MVP) QString generatePrompt; // --generate "" float generateDuration = 0.0f; // --duration N (seconds; 0 = clip's native length) bool generateUseModel = false; // --model → experimental trained t2m model (template fallback) float armSpaceDeg = 0.0f; // #854: Mixamo-style arm-space swing (degrees) bool armSpaceSet = false; // --arm-space given (standalone post-adjust) + bool generateFootPin = true; // #856: pin feet after --generate (default ON) + bool footPinSet = false; // --foot-pin given (standalone post-process) + int generateSmoothFps = 12; // #837: sparse-bake low-pass (0 = off) bool jsonOutput = false; int resampleCount = 0; int decimateStep = 0; @@ -2294,6 +2334,17 @@ int CLIPipeline::cmdAnim(int argc, char* argv[]) dumpCanonicalPath = QString(argv[++i]); continue; } + // #837 retarget parity harness (dev/test): apply an arbitrary + // canonical clip JSON (as written by --dump-canonical) through the + // SAME bind-referenced retarget the model path uses — no model, no + // conditioning. Lets a self-retarget round-trip (dump a rig's own + // clip → apply-canonical back onto it → re-dump → compare) measure + // per-bone retarget parity in isolation. + if (arg == "--apply-canonical" && i + 1 < argc) { + applyCanonicalMode = true; + applyCanonicalPath = QString(argv[++i]); + continue; + } if (arg == "--generate" && i + 1 < argc) { generateMode = true; generatePrompt = QString::fromLocal8Bit(argv[++i]); @@ -2312,6 +2363,18 @@ int CLIPipeline::cmdAnim(int argc, char* argv[]) armSpaceSet = true; continue; } + // #856 foot-contact cleanup: ON by default during --generate + // (--no-foot-pin opts out); --foot-pin alone post-processes an + // EXISTING animation (standalone, needs --animation). + if (arg == "--no-foot-pin") { generateFootPin = false; continue; } + if (arg == "--foot-pin") { footPinSet = true; continue; } + // #837 smooth-bake post-pass on --generate: bake sparse then back to + // the clip rate (temporal low-pass, kills retarget trembling). + if (arg == "--no-smooth-bake") { generateSmoothFps = 0; continue; } + if (arg == "--smooth-fps" && i + 1 < argc) { + generateSmoothFps = QString(argv[++i]).toInt(); + continue; + } if (arg == "--simplify") { simplifyMode = true; continue; } if (arg == "--analyze") { analyzeMode = true; continue; } if (arg == "--preset" && i + 1 < argc) { @@ -2407,7 +2470,70 @@ int CLIPipeline::cmdAnim(int argc, char* argv[]) } return cmdAnimGenerate(filePath, generatePrompt, generateDuration, outputPath.isEmpty() ? filePath : outputPath, jsonOutput, - generateUseModel, armSpaceDeg); + generateUseModel, armSpaceDeg, generateFootPin, + generateSmoothFps); + } + + // #837 parity harness: apply a canonical clip JSON through the pure + // bind-referenced retarget (no model, no conditioning, no smooth/pin/ + // arm-space) and export. Used only for retarget self-parity testing. + if (applyCanonicalMode) { + if (!initOgreHeadless()) return 1; + QFile jf(applyCanonicalPath); + if (!jf.open(QIODevice::ReadOnly)) { + err() << "Error: cannot open " << applyCanonicalPath << Qt::endl; + return 1; + } + const QJsonObject root = + QJsonDocument::fromJson(jf.readAll()).object(); + const QJsonArray clips = root.value("clips").toArray(); + if (clips.isEmpty()) { err() << "Error: no clips in json" << Qt::endl; return 1; } + const QJsonObject clip = clips.first().toObject(); + std::vector>> q; + for (const QJsonValue& fv : clip.value("quats").toArray()) { + const QJsonArray fa = fv.toArray(); + std::vector> frame; + for (const QJsonValue& jv : fa) { + const QJsonArray a = jv.toArray(); + frame.push_back({float(a[0].toDouble()), float(a[1].toDouble()), + float(a[2].toDouble()), float(a[3].toDouble())}); + } + q.push_back(std::move(frame)); + } + std::vector> rw; + for (const QJsonValue& v : clip.value("restWorld").toArray()) { + const QJsonArray a = v.toArray(); + rw.push_back({float(a[0].toDouble()), float(a[1].toDouble()), + float(a[2].toDouble()), float(a[3].toDouble())}); + } + std::vector> rd; + for (const QJsonValue& v : clip.value("restDir").toArray()) { + const QJsonArray a = v.toArray(); + rd.push_back({float(a[0].toDouble()), float(a[1].toDouble()), + float(a[2].toDouble())}); + } + const int cfps = root.value("fps").toInt(30); + MeshImporterExporter::importer({QFileInfo(filePath).absoluteFilePath()}); + Ogre::Entity* ent = nullptr; + for (auto* e : Manager::getSingleton()->getEntities()) + if (e && e->getMovableType() == "Entity" && e->hasSkeleton()) { ent = e; break; } + if (!ent) { err() << "Error: no skinned mesh" << Qt::endl; return 1; } + Ogre::SkeletonPtr skel = ent->getMesh()->getSkeleton(); + const auto res = AnimationMerger::applyMotionClip( + skel.get(), "generated_parity", q, cfps, + /*worldFrame=*/true, rw, /*refineWithModel=*/false, + /*refineStride=*/8, /*yaw180=*/false, rd); + if (!res.ok) { err() << "Error: " << res.error << Qt::endl; return 1; } + const QString out = outputPath.isEmpty() + ? QStringLiteral("/tmp/parity_out.glb") : outputPath; + auto* node = ent->getParentSceneNode(); + if (MeshImporterExporter::exporter(node, QFileInfo(out).absoluteFilePath(), + formatForExtension(out)) != 0) { + err() << "Error: export failed." << Qt::endl; return 1; + } + cliWrite(QString("applied canonical → %1 (%2 frames)\n") + .arg(QFileInfo(out).fileName()).arg(res.frames)); + return 0; } // #854 standalone: post-adjust the arm space of an EXISTING animation @@ -2453,6 +2579,45 @@ int CLIPipeline::cmdAnim(int argc, char* argv[]) return 0; } + // #856 standalone: pin the feet of an EXISTING animation (no --generate). + // `qtmesh anim --foot-pin --animation -o out`. + if (footPinSet) { + if (animationFilter.isEmpty()) { + err() << "Error: --foot-pin (standalone) requires --animation ." + << Qt::endl; + return 2; + } + if (!initOgreHeadless()) return 1; + MeshImporterExporter::importer({QFileInfo(filePath).absoluteFilePath()}); + Ogre::Entity* entity = nullptr; + for (auto* e : Manager::getSingleton()->getEntities()) + if (e && e->getMovableType() == "Entity" && e->hasSkeleton()) { entity = e; break; } + if (!entity) { + err() << "Error: " << filePath << " has no skinned mesh." << Qt::endl; + return 1; + } + Ogre::SkeletonPtr skel = entity->getMesh()->getSkeleton(); + const auto fp = AnimationMerger::pinFeet( + skel.get(), animationFilter.toStdString()); + if (!fp.ok) { + err() << "Error: foot-pin failed: " << fp.error << Qt::endl; + return 1; + } + SentryReporter::addBreadcrumb(QStringLiteral("ai.tool_call"), + QStringLiteral("foot_pin %1 spans").arg(fp.spans)); + const QString out = outputPath.isEmpty() ? filePath : outputPath; + auto* node = entity->getParentSceneNode(); + if (MeshImporterExporter::exporter(node, QFileInfo(out).absoluteFilePath(), + formatForExtension(out)) != 0) { + err() << "Error: export failed." << Qt::endl; return 1; + } + cliWrite(QString("Pinned feet of '%1' (%2 span(s), %3 keyframes) → %4\n") + .arg(animationFilter).arg(fp.spans) + .arg(fp.keyframesAdjusted) + .arg(QFileInfo(out).fileName())); + return 0; + } + if (!listMode && !renameMode && !mergeMode && !resampleMode && !decimateMode && !simplifyMode && !analyzeMode && !bakeFpsMode && !inbetweenMode && !dumpCanonicalMode) { diff --git a/src/CLIPipeline.h b/src/CLIPipeline.h index 969e4277..3d057d17 100644 --- a/src/CLIPipeline.h +++ b/src/CLIPipeline.h @@ -93,7 +93,8 @@ class CLIPipeline { static int cmdAnimGenerate(const QString& filePath, const QString& prompt, float duration, const QString& outputPath, bool jsonOutput, bool useModel = false, - float armSpaceDeg = 0.0f); + float armSpaceDeg = 0.0f, bool footPin = true, + int smoothFps = 12); static int cmdValidate(int argc, char* argv[]); static int cmdLod(int argc, char* argv[]); static int cmdPose(int argc, char* argv[]); diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index b7bfd493..f910afbe 100755 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -65,6 +65,7 @@ SentryReporter.cpp BoneWeightOverlay.cpp NormalVisualizer.cpp AnimationMerger.cpp +FootContact.cpp CLIPipeline.cpp ConsoleLogSanitize.cpp MeshInfoOverlay.cpp @@ -290,6 +291,7 @@ SentryReporter.h BoneWeightOverlay.h NormalVisualizer.h AnimationMerger.h +FootContact.h CLIPipeline.h MeshInfoOverlay.h RTShaderHelper.h diff --git a/src/FootContact.cpp b/src/FootContact.cpp new file mode 100644 index 00000000..a3a2ffbb --- /dev/null +++ b/src/FootContact.cpp @@ -0,0 +1,115 @@ +#include "FootContact.h" + +#include +#include + +namespace FootContact { + +namespace { + +V3 sub(const V3& a, const V3& b) { return {a[0] - b[0], a[1] - b[1], a[2] - b[2]}; } +V3 add(const V3& a, const V3& b) { return {a[0] + b[0], a[1] + b[1], a[2] + b[2]}; } +V3 mul(const V3& a, float s) { return {a[0] * s, a[1] * s, a[2] * s}; } +float dot(const V3& a, const V3& b) { return a[0]*b[0] + a[1]*b[1] + a[2]*b[2]; } +V3 cross(const V3& a, const V3& b) +{ + return {a[1]*b[2] - a[2]*b[1], a[2]*b[0] - a[0]*b[2], a[0]*b[1] - a[1]*b[0]}; +} +float len(const V3& a) { return std::sqrt(dot(a, a)); } +V3 normed(const V3& a) +{ + const float l = len(a); + return l > 1e-9f ? mul(a, 1.0f / l) : V3{0, 0, 0}; +} + +} // namespace + +float groundLevel(const std::vector& foot, int upAxis) +{ + if (foot.empty()) return 0.0f; + std::vector h(foot.size()); + for (size_t i = 0; i < foot.size(); ++i) h[i] = foot[i][upAxis]; + const size_t k = std::min(foot.size() - 1, + static_cast(foot.size() / 10)); + std::nth_element(h.begin(), h.begin() + static_cast(k), h.end()); + return h[k]; +} + +std::vector detectContacts(const std::vector& foot, + float legLength, + const DetectOptions& opt) +{ + std::vector spans; + const int n = static_cast(foot.size()); + if (n < opt.minFrames || legLength <= 1e-6f) return spans; + const int up = std::clamp(opt.upAxis, 0, 2); + const float ground = groundLevel(foot, up); + const float band = ground + opt.heightBandFrac * legLength; + const float vmax = opt.velThreshFrac * legLength; + + auto horizSpeed = [&](int f) { + // central difference where possible; edges take the one-sided step + const int a = std::max(0, f - 1); + const int b = std::min(n - 1, f + 1); + V3 d = sub(foot[b], foot[a]); + d[up] = 0.0f; + return len(d) / static_cast(std::max(1, b - a)); + }; + + int start = -1; + for (int f = 0; f <= n; ++f) { + const bool contact = f < n && foot[f][up] <= band && horizSpeed(f) <= vmax; + if (contact && start < 0) start = f; + if (!contact && start >= 0) { + if (f - start >= opt.minFrames) spans.push_back({start, f - 1}); + start = -1; + } + } + return spans; +} + +V3 solveKnee(const V3& hip, const V3& knee, const V3& foot, const V3& target) +{ + const float l1 = len(sub(knee, hip)); + const float l2 = len(sub(foot, knee)); + if (l1 <= 1e-6f || l2 <= 1e-6f) return knee; + + V3 toT = sub(target, hip); + float d = len(toT); + const float dMin = std::abs(l1 - l2) + 1e-4f; + const float dMax = l1 + l2 - 1e-4f; + if (d < 1e-6f) return knee; // target on the hip: give up + const V3 dirT = mul(toT, 1.0f / d); + d = std::clamp(d, dMin, dMax); + + // Bend direction: the current knee's offset from the hip→foot line keeps + // the pose's own bend plane (the knee axis of the existing pose). + V3 bend = sub(knee, hip); + bend = sub(bend, mul(dirT, dot(bend, dirT))); + if (dot(bend, bend) < 1e-10f) { + // straight leg — bend forward of the current chain plane, or any + // perpendicular when even that is degenerate + bend = cross(dirT, cross(sub(foot, hip), sub(knee, hip))); + if (dot(bend, bend) < 1e-10f) + bend = cross(dirT, std::abs(dirT[1]) < 0.9f ? V3{0, 1, 0} + : V3{1, 0, 0}); + } + bend = normed(bend); + + // law of cosines: knee at distance l1 from hip, l2 from target + const float cosA = std::clamp((l1 * l1 + d * d - l2 * l2) + / (2.0f * l1 * d), -1.0f, 1.0f); + const float sinA = std::sqrt(std::max(0.0f, 1.0f - cosA * cosA)); + return add(hip, add(mul(dirT, l1 * cosA), mul(bend, l1 * sinA))); +} + +float blendWeight(const Span& s, int f, int blend) +{ + if (f < s.start || f > s.end) return 0.0f; + if (blend <= 0) return 1.0f; + const float in = static_cast(f - s.start + 1) / blend; + const float out = static_cast(s.end - f + 1) / blend; + return std::clamp(std::min(in, out), 0.0f, 1.0f); +} + +} // namespace FootContact diff --git a/src/FootContact.h b/src/FootContact.h new file mode 100644 index 00000000..5ccd9a39 --- /dev/null +++ b/src/FootContact.h @@ -0,0 +1,68 @@ +#ifndef FOOT_CONTACT_H +#define FOOT_CONTACT_H + +#include +#include + +// #856 — foot-contact cleanup, pure-data core (Ogre-free, unit-tested). +// +// Retargeted clips slide/float feet on rigs whose proportions differ from the +// source: the bind-referenced direction retarget (PR #843) transfers bone +// DIRECTIONS, not world foot positions. This core provides the two pieces the +// track post-process (AnimationMerger::pinFeet) composes: +// 1. contact detection — frames where a foot is near the clip's ground +// level AND nearly stationary horizontally → contact spans, +// 2. an analytic two-bone IK step — where must the knee sit so the foot +// reaches its pinned position, preserving the pose's own bend plane, +// plus the smooth edge-blend weight that avoids knee pops at span borders. +namespace FootContact { + +using V3 = std::array; + +/// Inclusive frame range during which a foot is planted. +struct Span { + int start = 0; + int end = 0; +}; + +struct DetectOptions { + /// Height band above the detected ground level counting as "on the + /// ground", as a fraction of leg length. Deliberately TIGHT: a walking + /// foot lifts only ~5-15% of leg length, and a generous band swallows + /// most of the swing phase — the whole clip pins and the character + /// shuffles (legs frozen while the upper body keeps full rate). + float heightBandFrac = 0.06f; + /// Max horizontal speed (units/frame) counting as "stationary", as a + /// fraction of leg length. + float velThreshFrac = 0.012f; + /// Spans shorter than this are noise, not contacts. + int minFrames = 4; + /// Up axis: 0=X, 1=Y, 2=Z (canonical rigs are +Y-up). + int upAxis = 1; +}; + +/// Ground level = low percentile of the foot's height trajectory (robust to +/// a clip that never plants, e.g. a jump apex-only window). +float groundLevel(const std::vector& foot, int upAxis); + +/// Contact spans of one foot trajectory. `legLength` scales both thresholds +/// so detection is rig-size-independent. +std::vector detectContacts(const std::vector& foot, + float legLength, + const DetectOptions& opt = {}); + +/// Analytic two-bone IK: given the CURRENT chain (hip → knee → foot) and a +/// new foot `target`, return the knee position that keeps both segment +/// lengths and stays in the pose's own bend plane (pole derived from the +/// current knee). Targets beyond reach are clamped along hip→target; +/// degenerate poses (straight leg) bend toward the current knee offset or, +/// failing that, any perpendicular. +V3 solveKnee(const V3& hip, const V3& knee, const V3& foot, const V3& target); + +/// Smooth 0→1→0 weight for frame `f` inside span `s`, ramping linearly over +/// `blend` frames at each edge (1 in the interior, 0 outside). +float blendWeight(const Span& s, int f, int blend); + +} // namespace FootContact + +#endif // FOOT_CONTACT_H diff --git a/src/FootContact_test.cpp b/src/FootContact_test.cpp new file mode 100644 index 00000000..907dc60d --- /dev/null +++ b/src/FootContact_test.cpp @@ -0,0 +1,128 @@ +#include + +#include + +#include "FootContact.h" + +// #856 pure-data core tests — no Ogre/GL needed. + +using FootContact::V3; + +namespace { +float dist(const V3& a, const V3& b) +{ + const float dx = a[0] - b[0], dy = a[1] - b[1], dz = a[2] - b[2]; + return std::sqrt(dx * dx + dy * dy + dz * dz); +} +} // namespace + +TEST(FootContactTest, DetectsPlantAndSwingPhases) +{ + // Gait-like trajectory: planted at x=0 for 10 frames, swing forward over + // 10 frames (lifted), planted at x=1 for 10 frames. + std::vector foot; + for (int f = 0; f < 10; ++f) foot.push_back({0.0f, 0.0f, 0.0f}); + for (int f = 0; f < 10; ++f) { + const float t = (f + 1) / 11.0f; + foot.push_back({t, 0.25f * std::sin(t * 3.14159f), 0.0f}); + } + for (int f = 0; f < 10; ++f) foot.push_back({1.0f, 0.0f, 0.0f}); + + const auto spans = FootContact::detectContacts(foot, /*legLength=*/1.0f); + ASSERT_EQ(spans.size(), 2u); + EXPECT_EQ(spans[0].start, 0); + EXPECT_GE(spans[0].end, 7); // plant 1 covers most of frames 0..9 + EXPECT_LE(spans[0].end, 10); + EXPECT_GE(spans[1].start, 19); // plant 2 starts after the swing + EXPECT_EQ(spans[1].end, 29); +} + +TEST(FootContactTest, SlidingFootIsNotAContact) +{ + // On the ground the whole time but translating fast — foot skate, the + // exact artifact we're pinning. Must NOT be detected as one long contact. + std::vector foot; + for (int f = 0; f < 30; ++f) + foot.push_back({0.1f * f, 0.0f, 0.0f}); // 0.1 leg-lengths/frame + const auto spans = FootContact::detectContacts(foot, 1.0f); + EXPECT_TRUE(spans.empty()); +} + +TEST(FootContactTest, ShortBlipsAreIgnored) +{ + // Foot is airborne (0.6) most of the clip, dipping to the ground for + // three SEPARATE single frames. The low-percentile ground level lands + // near 0, so each dip is an isolated on-ground frame — none forms a + // run of minFrames consecutive, so no contact span registers. + std::vector foot; + for (int f = 0; f < 20; ++f) + foot.push_back({0.0f, (f == 3 || f == 10 || f == 17) ? 0.0f : 0.6f, 0.0f}); + FootContact::DetectOptions opt; + opt.minFrames = 3; + const auto spans = FootContact::detectContacts(foot, 1.0f, opt); + EXPECT_TRUE(spans.empty()); // isolated 1-frame touches < minFrames +} + +TEST(FootContactTest, SolveKneePreservesLengthsAndReachesTarget) +{ + const V3 hip{0, 1.0f, 0}; + const V3 knee{0.1f, 0.5f, 0.15f}; // bent slightly forward + const V3 foot{0, 0.0f, 0.05f}; + const float l1 = dist(hip, knee), l2 = dist(knee, foot); + + const V3 target{0.05f, 0.02f, -0.2f}; // pin slightly behind + const V3 k2 = FootContact::solveKnee(hip, knee, foot, target); + EXPECT_NEAR(dist(hip, k2), l1, 1e-4f); + EXPECT_NEAR(dist(k2, target), l2, 1e-4f); + // knee keeps bending roughly forward (pose's own bend plane, no flip) + EXPECT_GT(k2[2], -0.05f); +} + +TEST(FootContactTest, SolveKneeClampsUnreachableTarget) +{ + const V3 hip{0, 1.0f, 0}; + const V3 knee{0, 0.5f, 0.1f}; + const V3 foot{0, 0.0f, 0}; + const float l1 = dist(hip, knee), l2 = dist(knee, foot); + + const V3 far{3.0f, -2.0f, 0}; // beyond l1+l2 + const V3 k2 = FootContact::solveKnee(hip, knee, foot, far); + EXPECT_NEAR(dist(hip, k2), l1, 1e-3f); + // the chain extends straight toward the target: knee sits on hip→far + const float reach = dist(hip, k2) + l2; + EXPECT_NEAR(reach, l1 + l2, 1e-3f); +} + +TEST(FootContactTest, SolveKneeStraightLegPicksStableBend) +{ + const V3 hip{0, 1.0f, 0}; + const V3 knee{0, 0.5f, 0}; // perfectly straight leg + const V3 foot{0, 0.0f, 0}; + const V3 target{0, 0.2f, 0}; // shorten: must bend somewhere + const V3 k2 = FootContact::solveKnee(hip, knee, foot, target); + EXPECT_NEAR(dist(hip, k2), 0.5f, 1e-4f); + EXPECT_NEAR(dist(k2, target), 0.5f, 1e-4f); +} + +TEST(FootContactTest, BlendWeightRampsAtSpanEdges) +{ + const FootContact::Span s{10, 20}; + EXPECT_FLOAT_EQ(FootContact::blendWeight(s, 9, 3), 0.0f); + EXPECT_FLOAT_EQ(FootContact::blendWeight(s, 21, 3), 0.0f); + EXPECT_NEAR(FootContact::blendWeight(s, 10, 3), 1.0f / 3.0f, 1e-5f); + EXPECT_NEAR(FootContact::blendWeight(s, 11, 3), 2.0f / 3.0f, 1e-5f); + EXPECT_FLOAT_EQ(FootContact::blendWeight(s, 15, 3), 1.0f); + EXPECT_NEAR(FootContact::blendWeight(s, 20, 3), 1.0f / 3.0f, 1e-5f); + // zero blend = hard pin + EXPECT_FLOAT_EQ(FootContact::blendWeight(s, 10, 0), 1.0f); +} + +TEST(FootContactTest, GroundLevelIsRobustToAirTime) +{ + // Foot spends most of the clip in the air (jump) — ground level must + // still track the low plateau, not the mean. + std::vector foot; + for (int f = 0; f < 40; ++f) + foot.push_back({0.0f, (f < 6) ? 0.02f : 0.8f, 0.0f}); + EXPECT_NEAR(FootContact::groundLevel(foot, 1), 0.02f, 1e-4f); +} diff --git a/src/MCPServer.cpp b/src/MCPServer.cpp index 550536d6..cfb1b888 100644 --- a/src/MCPServer.cpp +++ b/src/MCPServer.cpp @@ -655,6 +655,7 @@ const QMap& MCPServer::toolHandlers() {QStringLiteral("motion_in_between"), &MCPServer::toolMotionInBetween}, {QStringLiteral("generate_motion"), &MCPServer::toolGenerateMotion}, {QStringLiteral("adjust_arm_space"), &MCPServer::toolAdjustArmSpace}, + {QStringLiteral("pin_feet"), &MCPServer::toolPinFeet}, {QStringLiteral("segment_mesh"), &MCPServer::toolSegmentMesh}, {QStringLiteral("generate_mesh_from_image"), &MCPServer::toolGenerateMeshFromImage}, {QStringLiteral("save_scene"), &MCPServer::toolSaveScene}, @@ -4195,10 +4196,18 @@ QJsonObject MCPServer::toolGenerateMotion(const QJsonObject &args) if (mr.ok) { action = mr.matchedAction; quats = mr.clip.quats; fps = mr.clip.fps; worldFrame = mr.worldFrame; clipSource = QStringLiteral("model"); gotClip = true; - // Borrow a template clip's reference directions so the - // retarget synthesizes a BIND-referenced base pose (no - // harvest from the rig's other animations). - clipDirs = MotionLibrary::referenceDirsForPrompt(prompt); + if (!mr.clip.restWorld.empty() && !mr.clip.restDir.empty()) { + // v5 models (#858) ship their canonical reference + // triple — same bind-referenced retarget as templates. + cmuRest = mr.clip.restWorld; + clipDirs = mr.clip.restDir; + } else { + // Legacy v4: borrow a template clip's reference + // directions so the retarget synthesizes a + // BIND-referenced base pose (no harvest from the + // rig's other animations). + clipDirs = MotionLibrary::referenceDirsForPrompt(prompt); + } } } } @@ -4242,9 +4251,19 @@ QJsonObject MCPServer::toolGenerateMotion(const QJsonObject &args) worldFrame, cmuRest, /*refineWithModel=*/false, /*refineStride=*/8, yaw180, - clipDirs); + clipDirs, + clipSource == QStringLiteral("model")); if (!r.ok) return makeErrorResult(QString("Error: %1").arg(r.error)); + // #837 quality post-pass (ON by default): sparse-bake temporal + // low-pass — removes retarget trembling. Runs before arm-space and + // foot pinning so the pin targets stay exact. + const int smoothFps = args.value("smooth_bake").toBool(true) + ? args.value("smooth_fps").toInt(12) : 0; + if (smoothFps > 0) + AnimationMerger::smoothBakeAnimation(skel.get(), animName, + smoothFps, fps); + // #854: optional Mixamo-style arm-space post-process. Echo whether it // took effect so an MCP caller can tell the rig had no arm roles // (rather than silently getting an unadjusted clip). @@ -4254,6 +4273,13 @@ QJsonObject MCPServer::toolGenerateMotion(const QJsonObject &args) armSpaceApplied = AnimationMerger::adjustArmSpace( skel.get(), animName, static_cast(armSpace)); + // #856: foot-contact cleanup — ON by default (foot_pin:false opts out). + int footPinSpans = -1; + if (args.value("foot_pin").toBool(true)) { + const auto fp = AnimationMerger::pinFeet(skel.get(), animName); + footPinSpans = fp.ok ? fp.spans : -1; + } + entity->refreshAvailableAnimationState(); // Exclusively enable the generated clip — enabled states BLEND in // Ogre, and mixing with the import's auto-enabled animation renders @@ -4289,6 +4315,7 @@ QJsonObject MCPServer::toolGenerateMotion(const QJsonObject &args) content["tracks_written"] = r.tracksWritten; content["canonical_joints"] = r.canonicalJoints; content["entity"] = QString::fromStdString(entity->getName()); if (std::abs(armSpace) > 1e-4) content["arm_space_applied"] = armSpaceApplied; + if (footPinSpans >= 0) content["foot_pin_spans"] = footPinSpans; if (!outPath.isEmpty()) content["exported"] = outPath; return makeSuccessResult( QString::fromUtf8(QJsonDocument(content).toJson(QJsonDocument::Indented))); @@ -4374,6 +4401,76 @@ QJsonObject MCPServer::toolAdjustArmSpace(const QJsonObject &args) } } +QJsonObject MCPServer::toolPinFeet(const QJsonObject &args) +{ + try { + SentryReporter::addBreadcrumb(QStringLiteral("ai.tool_call"), + QStringLiteral("MCP pin_feet")); + + Manager* mgr = Manager::getSingletonPtr(); + if (!mgr) return makeErrorResult("Error: Manager not available"); + + const QString animName = args.value("animation_name").toString(); + if (animName.isEmpty()) + return makeErrorResult("Error: animation_name is required."); + + const QString entityName = args.value("entity_name").toString(); + Ogre::Entity* entity = nullptr; + for (auto* ent : mgr->getEntities()) { + if (!ent || ent->getMovableType() != "Entity" || !ent->hasSkeleton()) + continue; + if (entityName.isEmpty() + || QString::fromStdString(ent->getName()) == entityName) { + entity = ent; break; + } + } + if (!entity) + return makeErrorResult(entityName.isEmpty() + ? QString("Error: no skinned mesh found.") + : QString("Error: skinned entity '%1' not found.").arg(entityName)); + + // Edit the mesh's MASTER skeleton (exporter serializes the master; + // animations are shared, so the live viewport still updates). + Ogre::SkeletonPtr skel = entity->getMesh()->getSkeleton(); + const std::string an = animName.toStdString(); + if (!skel || !skel->hasAnimation(an)) + return makeErrorResult( + QString("Error: animation '%1' not found on entity.").arg(animName)); + + const auto fp = AnimationMerger::pinFeet(skel.get(), an); + if (!fp.ok) + return makeErrorResult( + QString("Error: foot-pin failed: %1").arg(fp.error)); + + if (auto* acc = AnimationControlController::instance()) + acc->notifyExternalAnimationEdit(); + + const QString outPath = args.value("output_path").toString(); + if (!outPath.isEmpty()) { + auto* node = entity->getParentSceneNode(); + if (MeshImporterExporter::exporter( + node, outPath, CLIPipeline::formatForExtension(outPath)) != 0) + return makeErrorResult( + QString("Error: pinned feet but export to %1 failed") + .arg(outPath)); + } + + QJsonObject content; + content["ok"] = true; + content["animation"] = animName; + content["spans"] = fp.spans; + content["keyframes_adjusted"] = fp.keyframesAdjusted; + content["entity"] = QString::fromStdString(entity->getName()); + if (!outPath.isEmpty()) content["exported"] = outPath; + return makeSuccessResult( + QString::fromUtf8(QJsonDocument(content).toJson(QJsonDocument::Indented))); + } catch (Ogre::Exception& e) { + return makeErrorResult(QString("Error: Ogre exception — %1").arg(e.getFullDescription().c_str())); + } catch (std::exception& e) { + return makeErrorResult(QString("Error: %1").arg(e.what())); + } +} + QJsonObject MCPServer::toolSegmentMesh(const QJsonObject &args) { try { @@ -8297,6 +8394,9 @@ QJsonArray MCPServer::buildToolsList() props["output_path"] = QJsonObject{{"type", "string"}, {"description", "Optional path to re-export the mesh with the new animation (e.g. /tmp/out.glb). If omitted, the animation is applied in-session only."}}; props["model"] = QJsonObject{{"type", "boolean"}, {"description", "EXPERIMENTAL: use the trained from-scratch text-to-motion ONNX model instead of the template clip. Falls back to the template library automatically if the model is unavailable or the action isn't in its vocabulary. Default false (template). Quality is action-dependent (locomotion better than gestures)."}}; props["arm_space"] = QJsonObject{{"type", "number"}, {"description", "Optional Mixamo-style arm-space post-process in degrees (#854): positive widens the arms away from the body, negative tucks them in. Default 0. Rescues arm-into-torso clipping on rigs whose proportions differ from the clip."}}; + props["foot_pin"] = QJsonObject{{"type", "boolean"}, {"description", "Foot-contact cleanup (#856): detect ground-contact spans and IK-pin the feet so they plant instead of skating. Default true; set false to keep the raw retarget."}}; + props["smooth_bake"] = QJsonObject{{"type", "boolean"}, {"description", "Temporal low-pass post-pass: bake the clip sparse then back to its native rate, removing retarget trembling. Default true."}}; + props["smooth_fps"] = QJsonObject{{"type", "number"}, {"description", "Sparse keyframe rate for the smooth-bake pass. Lower = smoother but softer motion. Default 12."}}; appendTool( "generate_motion", "AI text-to-motion (#411, experimental): generate a skeletal animation from a text prompt and " @@ -8329,6 +8429,24 @@ QJsonArray MCPServer::buildToolsList() ); } + // pin_feet + { + QJsonObject props; + props["animation_name"] = QJsonObject{{"type", "string"}, {"description", "Name of the animation to clean up, e.g. \"generated_walk\"."}}; + props["entity_name"] = QJsonObject{{"type", "string"}, {"description", "Name of the rigged entity. If omitted, uses the first skinned entity."}}; + props["output_path"] = QJsonObject{{"type", "string"}, {"description", "Optional path to re-export the mesh with the cleaned animation. If omitted, applied in-session only."}}; + appendTool( + "pin_feet", + "Foot-contact cleanup (#856): detect frames where each foot is near the clip's ground level and " + "nearly stationary (contact spans) and lock the foot's world position to its span-start position " + "with an analytic two-bone hip-knee-foot IK, blending in/out at span edges so knees don't pop. " + "Fixes foot skating/floating on retargeted clips whose rig proportions differ from the source. " + "Only the thigh/shin/foot keyframes are rewritten; effectively idempotent.", + props, + QJsonArray{"animation_name"} + ); + } + // segment_mesh { QJsonObject props; diff --git a/src/MCPServer.h b/src/MCPServer.h index 82657ff3..7400a820 100644 --- a/src/MCPServer.h +++ b/src/MCPServer.h @@ -214,6 +214,7 @@ private slots: QJsonObject toolMotionInBetween(const QJsonObject &args); QJsonObject toolGenerateMotion(const QJsonObject &args); // #411 text-to-motion QJsonObject toolAdjustArmSpace(const QJsonObject &args); // #854 arm-space + QJsonObject toolPinFeet(const QJsonObject &args); // #856 foot-contact pin QJsonObject toolSegmentMesh(const QJsonObject &args); QJsonObject toolGenerateMeshFromImage(const QJsonObject &args); // #764 image-to-3D QJsonObject toolSaveScene(const QJsonObject &args); diff --git a/src/MotionGenerator.cpp b/src/MotionGenerator.cpp index 5e7f973d..eafff638 100644 --- a/src/MotionGenerator.cpp +++ b/src/MotionGenerator.cpp @@ -191,6 +191,39 @@ MotionGenerator::Result MotionGenerator::generate( const bool worldFrame = vj.value("frame").toString() == QLatin1String("world"); const int fps = vj.value("fps").toInt(30); + // v5 models (#858) train on CANONICALIZED quats and ship their + // reference triple in the vocab: restWorld (identity ×22) + restDir + // (the fixed canonical T-pose directions). With it, model clips ride + // the SAME bind-referenced direction retarget as v5 template clips — + // no synthetic-standing-pose shim. + std::vector> vocabRestWorld; + std::vector> vocabRestDir; + { + const QJsonArray rw = vj.value("restWorld").toArray(); + const QJsonArray rd = vj.value("restDir").toArray(); + if (rw.size() == J && rd.size() == J) { + for (const QJsonValue& v : rw) { + const QJsonArray q = v.toArray(); + if (q.size() != 4) { vocabRestWorld.clear(); break; } + vocabRestWorld.push_back({float(q[0].toDouble()), + float(q[1].toDouble()), + float(q[2].toDouble()), + float(q[3].toDouble())}); + } + for (const QJsonValue& v : rd) { + const QJsonArray d = v.toArray(); + if (d.size() != 3) { vocabRestDir.clear(); break; } + vocabRestDir.push_back({float(d[0].toDouble()), + float(d[1].toDouble()), + float(d[2].toDouble())}); + } + if (vocabRestWorld.size() != static_cast(J) + || vocabRestDir.size() != static_cast(J)) { + vocabRestWorld.clear(); + vocabRestDir.clear(); + } + } + } const int V = vocab.size(); if (V == 0 || T <= 0 || C != J * 10) { r.error = QStringLiteral("t2m vocab json malformed"); return r; @@ -276,10 +309,34 @@ MotionGenerator::Result MotionGenerator::generate( // NOSONAR — non-crypto: seeds latent-noise sampling for motion variety. std::mt19937 rng(QRandomGenerator::global()->generate()); // NOSONAR std::normal_distribution gauss(0.0f, 0.5f); - constexpr int kCandidates = 8; + constexpr int kCandidates = 16; constexpr float kTargetStep = 0.048f; // rad/frame — real-clip locals constexpr float kMaxArtic = 1.5f; // rad from the starting pose + // Posture-aware ranking (#837 quality follow-up): v5+ models are + // CANONICAL (restWorld = identity), so a bone's world direction is + // simply q·restDir — the same measurements that curate the template + // library rank the candidates here: spine and head must stay up, + // and locomotion upper arms must HANG (signed Y — an abs() check + // would pass a skyward arm). + const bool haveCanonDirs = !vocabRestDir.empty(); + const bool locomotion = matched == QLatin1String("walk") + || matched == QLatin1String("run") + || matched == QLatin1String("march"); + auto qRotDir = [&](const Q4& q, int role) -> std::array { + const auto& d = vocabRestDir[static_cast(role)]; + const float qx = q[0], qy = q[1], qz = q[2], qw = q[3]; + const float ux = qy * d[2] - qz * d[1]; + const float uy = qz * d[0] - qx * d[2]; + const float uz = qx * d[1] - qy * d[0]; + const float vx = qy * uz - qz * uy; + const float vy = qz * ux - qx * uz; + const float vz = qx * uy - qy * ux; + return { d[0] + 2.0f * (qw * ux + vx), + d[1] + 2.0f * (qw * uy + vy), + d[2] + 2.0f * (qw * uz + vz) }; + }; + std::vector> best; // [T][J] float bestScore = -1e30f; for (int cand = 0; cand < kCandidates; ++cand) { @@ -338,9 +395,47 @@ MotionGenerator::Result MotionGenerator::generate( } const float step = static_cast(stepSum / ((T - 1) * (J - 1))); const float coh = cohN ? static_cast(cohSum / cohN) : 0.0f; - const float score = -std::abs(step - kTargetStep) * 40.0f - + coh * 2.0f - - std::max(0.0f, maxArtic - kMaxArtic) * 4.0f; + float score = -std::abs(step - kTargetStep) * 40.0f + + coh * 2.0f + - std::max(0.0f, maxArtic - kMaxArtic) * 4.0f; + // Requires the full 22-joint canonical layout — the posture roles + // (17/21 feet, 11 larm) index up to 21. Guard J so a smaller vocab + // can't read out of bounds. + if (haveCanonDirs && worldFrame && J >= 22) { + // Posture penalties DOMINATE the energy/coherence terms: a + // bad sample (head thrown back, an arm reaching out) is worse + // than a slightly-off-tempo good one, so the weights here are + // ~10x the motion terms. This is the "best-of-N must never + // pick a broken pose" guarantee — the model produces good + // walks most of the time, and the scorer's job is to reject + // the occasional flailing draw. + auto meanDir = [&](int role) -> std::array { + float ax = 0.f, ay = 0.f, az = 0.f; + for (int f = 0; f < T; ++f) { + const auto d = qRotDir(w[f][role], role); + ax += d[0]; ay += d[1]; az += d[2]; + } + const float inv = 1.0f / static_cast(T); + return { ax * inv, ay * inv, az * inv }; + }; + const float spineY = meanDir(1)[1]; // abdomen up-ness + const auto head = meanDir(5); // head direction + // head must point UP; penalize both drooping AND tipping back + // (|Z| forward/back lean of the head axis). + score -= std::max(0.0f, 0.85f - head[1]) * 60.0f; + score -= std::abs(head[2]) * 25.0f; + score -= std::max(0.0f, 0.85f - spineY) * 60.0f; + if (locomotion) { + // Each arm's upper-arm must HANG (signed Y strongly + // negative). An arm reaching out/forward sits near Y≈0 + // and is heavily penalized; the worst arm dominates so a + // single flung arm can't hide behind the other. + const float ry = meanDir(7)[1]; + const float ly = meanDir(11)[1]; + score -= std::max(0.0f, ry + 0.35f) * 40.0f; + score -= std::max(0.0f, ly + 0.35f) * 40.0f; + } + } if (score > bestScore) { bestScore = score; best = std::move(w); } } if (best.empty()) { @@ -375,6 +470,26 @@ MotionGenerator::Result MotionGenerator::generate( clip.frames = want; } + clip.restWorld = vocabRestWorld; + clip.restDir = vocabRestDir; + if (!clip.restDir.empty()) { + // Ballerina-feet guard: feet are low-variance joints the model + // under-fits — near-static predicted feet aim the target's foot + // at the canonical FORWARD axis and point the toes. When a foot + // role barely articulates, drop its reference direction so the + // retarget leaves those bones at the rig's own bind pitch. + for (const int foot : {17, 21}) { + float maxDev = 0.0f; + const Q4& q0 = best[0][static_cast(foot)]; + for (int f = 1; f < T; ++f) { + const Q4 d = qMul(qConj(q0), + best[f][static_cast(foot)]); + maxDev = std::max(maxDev, qAngle(d)); + } + if (maxDev < 0.21f) // < ~12 degrees over the whole clip + clip.restDir[static_cast(foot)] = {0.f, 0.f, 0.f}; + } + } r.clip = std::move(clip); r.worldFrame = worldFrame; r.ok = true; diff --git a/src/MotionGenerator.h b/src/MotionGenerator.h index 081802f0..a353b5f3 100644 --- a/src/MotionGenerator.h +++ b/src/MotionGenerator.h @@ -22,11 +22,17 @@ // model is unavailable (no ENABLE_ONNX, model not downloaded, prompt action not // in the model's vocab, or inference fails). // -// ONNX contract (must match scripts/train-t2m-onnx-v3.py export): +// ONNX contract (v5: scripts/train-t2m-flow-v5.py; v4: train-t2m-onnx-v4.py): // input "tokens" float32 [1, V] one-hot/bag over the fixed action vocab -// input "seed" float32 [1, Z] latent (we pass zeros for the mean clip) +// input "seed" float32 [1, Z] latent noise (sampled; best-of-N scored) // output "motion" float32 [1, T, C] C = 22*10 per-joint [t.xyz, q.xyzw, s.xyz] -// The accompanying t2m-vocab.json gives {vocab, Z, T, C, J, joints}. +// The accompanying t2m-vocab.json gives {vocab, Z, T, C, J, fps, frame}. +// v5 FLOW-MATCHING models (#840/#858) keep this exact interface: the Euler +// sampler is unrolled INSIDE the exported graph (seed = the flattened noise +// tensor, Z = T·132), and the vocab additionally carries the model's +// canonical reference triple ("restWorld" identity ×22 + "restDir" canonical +// T-pose directions) so generated clips ride the same bind-referenced +// direction retarget as v5 template clips (no standing-pose shim). class MotionGenerator { public: MotionGenerator() = delete; diff --git a/src/MotionInbetween.cpp b/src/MotionInbetween.cpp index e612adcc..ecf23d34 100644 --- a/src/MotionInbetween.cpp +++ b/src/MotionInbetween.cpp @@ -153,7 +153,7 @@ int MotionInbetween::canonicalIndexForBone(const QString& boneName) if (has({"elbow", "forearm", "lowerarm"})) return 8; // relbow if (has({"hand", "wrist"})) return 9; // rhand if (has({"buttock"})) return 14; // rbuttock - if (has({"upleg", "thigh", "hip", "femur"})) return 15; // rhip + if (has({"upleg", "upperleg", "thigh", "hip", "femur"})) return 15; // rhip if (has({"knee", "leg", "shin", "calf"}) && !has({"upleg","thigh"})) return 16; // rknee if (has({"foot", "ankle"})) return 17; // rfoot } else if (side == 'l') { @@ -163,7 +163,7 @@ int MotionInbetween::canonicalIndexForBone(const QString& boneName) if (has({"elbow", "forearm", "lowerarm"})) return 12; // lelbow if (has({"hand", "wrist"})) return 13; // lhand if (has({"buttock"})) return 18; // lbuttock - if (has({"upleg", "thigh", "hip", "femur"})) return 19; // lhip + if (has({"upleg", "upperleg", "thigh", "hip", "femur"})) return 19; // lhip if (has({"knee", "leg", "shin", "calf"}) && !has({"upleg","thigh"})) return 20; // lknee if (has({"foot", "ankle"})) return 21; // lfoot } diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 05899fbd..49fb5958 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -77,6 +77,7 @@ if(BUILD_TESTS) ${CMAKE_CURRENT_SOURCE_DIR}/../src/BoneWeightOverlay.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../src/NormalVisualizer.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../src/AnimationMerger.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../src/FootContact.cpp # Face auto-rig (#889): MCPServer/CLIPipeline/mainwindow reference # these — without them the per-suite test executables fail to link. ${CMAKE_CURRENT_SOURCE_DIR}/../src/FaceRigController.cpp