From 98fa2c54efc8b1a012f1eedbe4cba4cc9c251b73 Mon Sep 17 00:00:00 2001 From: Fernando Date: Mon, 13 Jul 2026 08:50:41 -0300 Subject: [PATCH 01/23] feat(anim): stabilized twist transport in the direction retarget (#857) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The bind-referenced direction retarget aimed each bone but inherited the target bind's roll — gesture clips read flat and the roll component between mismatched decomposition poles leaked into swing (arm amplitude +38% on Mixamo self-retarget). Now the source's swing/twist split is recomposed about the SAME pole on the target: Qbase = arc(dt_bind -> d_ref) * Wt_bind (once per bone) Wt(f) = twist(theta*gain, ds) * arc(d_ref -> ds) * Qbase theta is the source's signed roll about its reference bone direction, wrapped to (-pi,pi] and unwrapped across frames (a +-180deg pop would flip the roll once a gain != 1 scales it), capped at 150deg; collars are damped 0.5x (they share the shoulder line's roll). Direction tracking stays absolute, so cross-rig robustness is unchanged. Mixamo Walk self-parity: 2.22deg -> 0.03deg mean joint delta (target was <=1.6deg); spine amplitude restored (abdomen 2.3deg -> 9.0deg amp), legs exact; Rumba walk/dance/wave and Quaternius Knight renders upright with natural hanging arms. Co-Authored-By: Claude Fable 5 --- src/AnimationMerger.cpp | 109 ++++++++++++++++++++++- src/AnimationMerger_test.cpp | 166 +++++++++++++++++++++++++++++++++++ 2 files changed, 271 insertions(+), 4 deletions(-) diff --git a/src/AnimationMerger.cpp b/src/AnimationMerger.cpp index 921e2656..9013fc36 100644 --- a/src/AnimationMerger.cpp +++ b/src/AnimationMerger.cpp @@ -1643,6 +1643,98 @@ AnimationMerger::ApplyMotionResult AnimationMerger::applyMotionClip( const auto& q = clipQuats[frame][joint]; return Ogre::Quaternion(q[3], q[0], q[1], q[2]); // (w,x,y,z) }; + // #857: STABILIZED TWIST TRANSPORT. The aim above transports the + // bone's DIRECTION but inherits the target bind's roll — gesture- + // heavy clips (dance, wave) lose forearm/spine roll and read + // flat. Decompose the source's frame rotation into swing (the + // direction, already transported) + twist about the bone axis: + // Δ(f) = Ws(f) · Ws_bind⁻¹ (canonical axes) + // aim(f) = arc(d_bind → d(f)) + // θ(f) = signed angle of aim(f)⁻¹·Δ(f) about d_bind + // θ is wrapped to (−π,π] then UNWRAPPED across frames (a ±180° + // pop near the shortest-arc degeneracy would otherwise flip the + // roll direction mid-clip once a gain ≠ 1 scales it) and composed + // on the target about the SAME pole the source decomposed about: + // Qbase = arc(dt_bind → d_ref) · Wt_bind (once per bone) + // Wt(f) = twist(θ·gain, ds) · arc(d_ref → ds) · Qbase + // Recomposing per-frame from the TARGET BIND direction instead + // (the pre-#857 form) decomposes about a different pole than the + // source did — the roll component "between" the two poles leaks + // into swing and inflates amplitude (measured: Mixamo self-parity + // arm amp +38%, elbow error 3.3°→6.3°). With matched poles the + // per-frame relative motion is the source's world delta exactly, + // conjugated into target axes — direction stays absolute, roll + // transports losslessly, self-parity drops below the aim-only + // baseline. Per-role gains: collars damped (they share the + // shoulder line's roll), everything else transports fully. + static const float kTwistGain[22] = { + 1.f, 1.f, 1.f, 1.f, 1.f, 1.f, // spine chain + head + 0.5f, 1.f, 1.f, 1.f, // rcollar damped, right arm + 0.5f, 1.f, 1.f, 1.f, // lcollar damped, left arm + 1.f, 1.f, 1.f, 1.f, // right leg + foot + 1.f, 1.f, 1.f, 1.f }; // left leg + foot + constexpr float kTwistCap = 2.618f; // 150° — runaway-unwrap guard + std::vector> twistTheta( + static_cast(frames), + std::vector(static_cast(Jc), 0.0f)); + { + constexpr float kTau = 2.0f * static_cast(M_PI); + std::vector prev(static_cast(Jc), 0.0f); + std::vector has(static_cast(Jc), 0); + for (int f = 0; f < frames; ++f) + for (int c = 0; c < Jc; ++c) { + const Ogre::Vector3& as = + srcLocalAxis[static_cast(c)]; + if (as.squaredLength() <= 1e-8f) continue; + const auto& rq = cmuRestWorld[static_cast(c)]; + const Ogre::Quaternion restQ(rq[3], rq[0], rq[1], rq[2]); + const auto& sd = clipRestDir[static_cast(c)]; + Ogre::Vector3 dbind(sd[0], sd[1], sd[2]); + dbind.normalise(); + const Ogre::Quaternion Wf = clipQ(f, c); + const Ogre::Vector3 d = Wf * as; + const Ogre::Quaternion delta = Wf * restQ.Inverse(); + const Ogre::Quaternion aim = dbind.getRotationTo(d); + const Ogre::Quaternion tw = aim.Inverse() * delta; + float th = 2.0f * std::atan2( + tw.x * dbind.x + tw.y * dbind.y + tw.z * dbind.z, + tw.w); + th = std::remainder(th, kTau); + if (has[static_cast(c)]) + th += kTau * std::round( + (prev[static_cast(c)] - th) / kTau); + prev[static_cast(c)] = th; + has[static_cast(c)] = 1; + twistTheta[static_cast(f)] + [static_cast(c)] = + std::clamp(th, -kTwistCap, kTwistCap); + } + } + // Reference-aligned roll baseline per bone: aim the target bind + // at the source's REFERENCE direction once, so every per-frame + // swing below decomposes about the source's own pole. + std::vector dref(static_cast(Jc), + Ogre::Vector3::ZERO); + for (int c = 0; c < Jc; ++c) { + const auto& sd = clipRestDir[static_cast(c)]; + Ogre::Vector3 v(sd[0], sd[1], sd[2]); + if (v.squaredLength() > 1e-8f) { + v.normalise(); + dref[static_cast(c)] = CtInv * v; + } + } + std::vector Qbase(static_cast(nBones), + Ogre::Quaternion::IDENTITY); + for (int i = 0; i < nBones; ++i) { + const int c = boneToCanon[i]; + if (c >= 0 && c < Jc + && srcLocalAxis[static_cast(c)].squaredLength() + > 1e-8f) + Qbase[static_cast(i)] = + tb.tgtBindDir[static_cast(c)] + .getRotationTo(dref[static_cast(c)]) + * tb.bindWorld[static_cast(i)]; + } std::vector tracks( static_cast(nBones), nullptr); for (int i = 0; i < nBones; ++i) @@ -1675,10 +1767,19 @@ AnimationMerger::ApplyMotionResult AnimationMerger::applyMotionClip( (clipQ(f, c) * srcLocalAxis[static_cast(c)]); const Ogre::Quaternion R = - tb.tgtBindDir[static_cast(c)] - .getRotationTo(ds); - const Ogre::Quaternion Wt = - R * tb.bindWorld[static_cast(i)]; + dref[static_cast(c)].getRotationTo(ds); + Ogre::Quaternion Wt = + R * Qbase[static_cast(i)]; + // #857: re-apply the source's roll about the aimed + // direction (the swing above deliberately dropped it). + const float th = twistTheta[static_cast(f)] + [static_cast(c)] + * kTwistGain[c]; + if (std::abs(th) > 1e-5f) { + Ogre::Vector3 axis = ds; + axis.normalise(); + Wt = Ogre::Quaternion(Ogre::Radian(th), axis) * Wt; + } local = Wp.Inverse() * Wt; W[static_cast(i)] = Wt; } else { diff --git a/src/AnimationMerger_test.cpp b/src/AnimationMerger_test.cpp index e4f8a2ac..07e55bf9 100644 --- a/src/AnimationMerger_test.cpp +++ b/src/AnimationMerger_test.cpp @@ -1139,3 +1139,169 @@ TEST_F(AnimationMergerTest, ArmSpaceFollowsAnimationRename) "walk_wide"); EXPECT_GT(degBetween(wide, neutral), 15.0f); // it actually moved back } + +// ── #857: twist transport in the bind-referenced direction retarget ───────── + +namespace { +// Canonical-clip inputs for a virtual SOURCE rig whose bind is rotated 90° +// about Z (a deliberately foreign convention — exercises the restWorld +// conjugation): restDir = clean canonical T-pose directions, restWorld = the +// same non-identity quat everywhere, and identity motion W(f) = restQ. +constexpr int kJc = 22; +const Ogre::Quaternion kSrcRest(Ogre::Degree(90), Ogre::Vector3::UNIT_Z); + +std::vector> canonRestDirs() +{ + static const float d[kJc][3] = { + {0,1,0},{0,1,0},{0,1,0},{0,1,0},{0,1,0},{0,1,0}, + {-1,0,0},{-1,0,0},{-1,0,0},{-1,0,0}, + {1,0,0},{1,0,0},{1,0,0},{1,0,0}, + {0,-1,0},{0,-1,0},{0,-1,0},{0,0,1}, + {0,-1,0},{0,-1,0},{0,-1,0},{0,0,1}}; + std::vector> out(kJc); + for (int c = 0; c < kJc; ++c) out[c] = {d[c][0], d[c][1], d[c][2]}; + return out; +} + +std::vector> srcRestWorld() +{ + return std::vector>( + kJc, {kSrcRest.x, kSrcRest.y, kSrcRest.z, kSrcRest.w}); +} + +// frames of identity motion (every joint sits at the source bind) +std::vector>> identityClip(int frames) +{ + return std::vector>>( + frames, std::vector>( + kJc, {kSrcRest.x, kSrcRest.y, kSrcRest.z, kSrcRest.w})); +} + +// signed rotation angle of world-orientation delta `q` about unit axis `ax` +float twistDegAbout(const Ogre::Quaternion& q, const Ogre::Vector3& ax) +{ + const float s = q.x * ax.x + q.y * ax.y + q.z * ax.z; + return 2.0f * std::atan2(s, q.w) * 180.0f / static_cast(M_PI); +} +} // namespace + +TEST_F(AnimationMergerTest, TwistTransportCarriesBoneRoll) +{ + Ogre::Entity* ent = makeArmRigEntity("twist_roll"); + ASSERT_NE(ent, nullptr); + Ogre::SkeletonInstance* skel = ent->getSkeleton(); + + // Source: left upper arm (role 11, +X) ROLLS 60° about its own axis over + // the clip while its direction stays put. Pre-#857 the retarget dropped + // this entirely (aim-only) — the target bone never moved. + const int frames = 31; + auto quats = identityClip(frames); + for (int f = 0; f < frames; ++f) { + const float a = 60.0f * static_cast(f) / (frames - 1); + const Ogre::Quaternion w = + Ogre::Quaternion(Ogre::Degree(a), Ogre::Vector3::UNIT_X) + * kSrcRest; + quats[f][11] = {w.x, w.y, w.z, w.w}; + } + const auto res = AnimationMerger::applyMotionClip( + skel, "twistclip", quats, 30, /*worldFrame=*/true, srcRestWorld(), + false, 8, false, canonRestDirs()); + ASSERT_TRUE(res.ok) << res.error.toStdString(); + + // Direction is invariant under a pure roll… + const Ogre::Vector3 dir0(1, 0, 0); + skel->reset(true); + skel->getAnimation("twistclip")->apply(skel, 1.0f); + skel->_updateTransforms(); + const Ogre::Vector3 a = + skel->getBone("LeftArm")->_getDerivedPosition(); + const Ogre::Vector3 b = + skel->getBone("LeftForeArm")->_getDerivedPosition(); + EXPECT_GT((b - a).normalisedCopy().dotProduct(dir0), 0.99f); + + // …but the bone's world orientation now carries the 60° roll about it. + const Ogre::Quaternion w = + skel->getBone("LeftArm")->_getDerivedOrientation(); + EXPECT_NEAR(twistDegAbout(w, dir0), 60.0f, 4.0f); + + // Legs saw identity source motion — they must not pick up any rotation. + const Ogre::Quaternion leg = + skel->getBone("LeftUpLeg")->_getDerivedOrientation(); + EXPECT_NEAR(std::abs(leg.w), 1.0f, 1e-3f); +} + +TEST_F(AnimationMergerTest, TwistUnwrapKeepsDampedCollarContinuous) +{ + // Rig WITH clavicles (Mixamo "Shoulder" → collar roles 6/10). + auto skelRes = Ogre::SkeletonManager::getSingleton().create( + "twist_collar_skel", + Ogre::ResourceGroupManager::DEFAULT_RESOURCE_GROUP_NAME); + unsigned short h = 0; + auto bone = [&](const char* n, const Ogre::Vector3& p, Ogre::Bone* par) { + auto* b = skelRes->createBone(n, h++); + b->setPosition(p); + if (par) par->addChild(b); + return b; + }; + auto* hips = bone("Hips", {0, 1.0f, 0}, nullptr); + auto* spine = bone("Spine", {0, 0.3f, 0}, hips); + bone("Head", {0, 0.4f, 0}, spine); + bone("LeftUpLeg", {0.15f, -0.1f, 0}, hips); + bone("RightUpLeg", {-0.15f, -0.1f, 0}, hips); + auto* lcol = bone("LeftShoulder", {0.05f, 0.25f, 0}, spine); + auto* larm = bone("LeftArm", {0.15f, 0, 0}, lcol); + bone("LeftForeArm", {0.3f, 0, 0}, larm); + auto* rcol = bone("RightShoulder", {-0.05f, 0.25f, 0}, spine); + auto* rarm = bone("RightArm", {-0.15f, 0, 0}, rcol); + bone("RightForeArm", {-0.3f, 0, 0}, rarm); + skelRes->setBindingPose(); + auto mesh = createInMemoryMesh("twist_collar_mesh", skelRes); + auto* sm = Manager::getSingleton()->getSceneMgr(); + Ogre::Entity* ent = sm->createEntity("twist_collar_ent", mesh); + Ogre::SkeletonInstance* skel = ent->getSkeleton(); + + // Source: left collar (role 10, +X) rolls 0 → 240° — past the ±180° wrap. + // The gain table damps collars to 0.5×, which is exactly where a missing + // unwrap explodes: wrapped −120° would scale to −60° instead of +120°'s + // half — a mid-clip snap. + const int frames = 61; + auto quats = identityClip(frames); + for (int f = 0; f < frames; ++f) { + const float a = 240.0f * static_cast(f) / (frames - 1); + const Ogre::Quaternion w = + Ogre::Quaternion(Ogre::Degree(a), Ogre::Vector3::UNIT_X) + * kSrcRest; + quats[f][10] = {w.x, w.y, w.z, w.w}; + } + const auto res = AnimationMerger::applyMotionClip( + skel, "collarclip", quats, 30, true, srcRestWorld(), + false, 8, false, canonRestDirs()); + ASSERT_TRUE(res.ok) << res.error.toStdString(); + + // Sample densely: the collar's world orientation must move CONTINUOUSLY + // (no wrap snap) and end near 240° × 0.5 = 120° about +X. + Ogre::Quaternion prev = Ogre::Quaternion::IDENTITY; + float maxStepDeg = 0.0f; + Ogre::Quaternion last; + auto* anim = skel->getAnimation("collarclip"); + const float len = anim->getLength(); + for (int s = 0; s <= 60; ++s) { + skel->reset(true); + anim->apply(skel, len * static_cast(s) / 60.0f); + skel->_updateTransforms(); + const Ogre::Quaternion w = + skel->getBone("LeftShoulder")->_getDerivedOrientation(); + if (s > 0) { + const Ogre::Quaternion d = w * prev.Inverse(); + const float step = 2.0f * std::acos(std::min( + 1.0f, std::abs(d.w))) * 180.0f / static_cast(M_PI); + maxStepDeg = std::max(maxStepDeg, step); + } + prev = w; + last = w; + } + EXPECT_LT(maxStepDeg, 15.0f) << "collar roll snapped mid-clip (unwrap)"; + EXPECT_NEAR(twistDegAbout(last, Ogre::Vector3::UNIT_X), 120.0f, 8.0f); + + sm->destroyEntity(ent); +} From 23123de44373795855bbab261f0f72b9e33b8e9f Mon Sep 17 00:00:00 2001 From: Fernando Date: Mon, 13 Jul 2026 09:00:12 -0300 Subject: [PATCH 02/23] =?UTF-8?q?feat(anim):=20v5=20flow-matching=20t2m=20?= =?UTF-8?q?model=20=E2=80=94=20canonical=20data,=20in-graph=20sampler,=20r?= =?UTF-8?q?eference=20triple=20(#840,=20#858)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Data (#858): scripts/prep-t2m-v5.py canonicalizes every clip (corpus *.canonical.json dumps + CMU BVH) into a rig-independent representation before windowing — per joint, aim-from-canonical-T-pose composed with the swing/twist residual: Q'(f) = twist(theta, d(f)) * arc(D_c -> d(f)). Identical for every rig performing the same motion, and exactly what the bind-referenced retarget consumes given the canonical reference triple (restWorld = identity, restDir = D_c). 17,286 windows / 18 actions (corpus 517 + CMU 16,789), replacing the v4 per-rig convention mush. Model (#840): scripts/train-t2m-flow-v5.py — small DiT-style velocity transformer (7.3M params, 6D rotations, AdaLN-Zero conditioning) trained with the rectified-flow objective + classifier-free guidance; flow matching samples a transport path to a single data mode instead of decoding a conditional mean (the CVAE's structural weakness). The Euler sampler (+CFG) is UNROLLED INSIDE the exported ONNX graph, so the shipped MotionGenerator contract (tokens[1,V], seed[1,Z] -> motion [1,T,220]) is unchanged — seed is the flattened noise tensor. Wiring (#858): MotionGenerator parses the vocab's restWorld/restDir into the generated clip; CLI/MCP/GUI model paths pass it to applyMotionClip, so model clips ride the SAME bind-referenced direction retarget as v5 template clips — the synthetic-standing-pose shim remains only as the legacy-v4 fallback. applyMotionClip now treats any valid unit-norm reference quat as present (identity counts; only zero-filled entries mean absent), which the v5 identity triple requires. Render-verified on the Rumba rig: model walk/run/wave/dance all upright and temporally coherent end-to-end (the v4 CVAE folded or tumbled); templates remain the default + automatic fallback. Co-Authored-By: Claude Fable 5 --- scripts/prep-t2m-v5.py | 350 +++++++++++++++++++++++++++++ scripts/train-t2m-flow-v5.py | 289 ++++++++++++++++++++++++ src/AnimationControlController.cpp | 16 +- src/AnimationMerger.cpp | 8 +- src/CLIPipeline.cpp | 21 +- src/MCPServer.cpp | 16 +- src/MotionGenerator.cpp | 35 +++ src/MotionGenerator.h | 12 +- 8 files changed, 728 insertions(+), 19 deletions(-) create mode 100644 scripts/prep-t2m-v5.py create mode 100644 scripts/train-t2m-flow-v5.py diff --git a/scripts/prep-t2m-v5.py b/scripts/prep-t2m-v5.py new file mode 100644 index 00000000..4b0a07a6 --- /dev/null +++ b/scripts/prep-t2m-v5.py @@ -0,0 +1,350 @@ +#!/usr/bin/env python3 +# ruff: noqa: E702, E741 +"""Build the v5 text-to-motion training cache from the motion corpus (#858). + +ONE-TIME OFFLINE dev tool — NOT shipped. Successor to prep-t2m-v4.py, fixing +its structural flaw: v4 trained on raw per-rig world quats, so every rig's +bone-axis convention leaked into the data (the "convention mush" that made +the CVAE average modes into gentle motion). v5 CANONICALIZES every clip into +a rig-independent representation before windowing: + + For each canonical joint c with source reference triple (restWorld Q_r, + restDir d_r) — the same triple the bind-referenced retarget (PR #843) + consumes — the source bone's world direction trajectory is + d(f) = W(f) · (Q_r⁻¹ · d̂_r) (canonical axes) + and its roll about the bone is the swing/twist residual + Δ(f) = W(f) · Q_r⁻¹ + aim = arc(d̂_r → d(f)) + θ(f) = signed angle of (aim⁻¹ · Δ(f)) about d̂_r, unwrapped over f. + The training quat is rebuilt against a FIXED canonical T-pose direction + set D (spine +Y, left arm +X, right arm −X, legs −Y, feet +Z): + Q'(f) = twist(θ(f), d(f)) · arc(D_c → d(f)) + + Q' is identical for every rig performing the same motion — and it is + EXACTLY what the runtime consumes: with the model's reference triple + (restWorld = identity, restDir = D), applyMotionClip recovers + d(f) = Q'(f)·D_c and (with #857 twist transport) θ(f) losslessly. The + model therefore rides the SAME retarget path as v5 template clips, no + synthetic-standing-pose shim. + +Sources: + --corpus ~/motion_corpus: the *.canonical.json sidecar dumps written by + build-motion-library-v5.py / `qtmesh anim --dump-canonical` + (#838/#839). Labels via the SAME action keyword table as the + template library, so both motion sources cover the same vocab. + --bvh CMU BVH conversion (optional): FK world quats (rest = identity, + restDir from the skeleton's rest joint positions), labels from + the public CMU index — the v4 path, canonicalized the same way. + +Output npz: mo[N,T,22,4] canonical quats, msk[N,22] per-joint validity, +tk[N,V] one-hot, vocab, fps, canonRestDir[22,3] (= D, for the vocab json). + +Usage: + python3 scripts/prep-t2m-v5.py --corpus ~/motion_corpus \ + [--bvh /tmp/cmu-mocap-bvh/data --index /tmp/cmu_index.txt] \ + --out /tmp/t2m_v5.npz [--T 60] +""" +import argparse +import glob +import importlib.util +import json +import os +import re +import sys + +import numpy as np + +HERE = os.path.dirname(os.path.abspath(__file__)) + + +def load_module(name, fname): + spec = importlib.util.spec_from_file_location(name, os.path.join(HERE, fname)) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +libv5 = load_module("libv5", "build-motion-library-v5.py") # action_for, quality +prep4 = load_module("prep4", "prep-t2m-v4.py") # BVH FK, index, windows + +CANON = prep4.CANON +J = 22 +FPS = 30 +# canonical chain parent (AnimationMerger kParentCanon) +PARENT = [-1, 0, 1, 2, 3, 4, 2, 6, 7, 8, 2, 10, 11, 12, 0, 14, 15, 16, 0, 18, 19, 20] +CHILD = {} +for _c, _p in enumerate(PARENT): # chain child = FIRST child (hip→abdomen, + if _p >= 0: # chest→neck), matching the extractor + CHILD.setdefault(_p, _c) + +# FIXED canonical T-pose bone directions D (canonical axes: +Y up, +Z fwd, +# +X left — the CMU frame convention every dump is normalized into). +D_CANON = np.array([ + [0, 1, 0], [0, 1, 0], [0, 1, 0], [0, 1, 0], [0, 1, 0], [0, 1, 0], + [-1, 0, 0], [-1, 0, 0], [-1, 0, 0], [-1, 0, 0], + [1, 0, 0], [1, 0, 0], [1, 0, 0], [1, 0, 0], + [0, -1, 0], [0, -1, 0], [0, -1, 0], [0, 0, 1], + [0, -1, 0], [0, -1, 0], [0, -1, 0], [0, 0, 1], +], np.float32) + + +# ---------- quat math, (x,y,z,w), vectorised over leading dims ---------- +def qmul(a, b): + ax, ay, az, aw = np.moveaxis(a, -1, 0) + bx, by, bz, bw = np.moveaxis(b, -1, 0) + return np.stack([aw*bx + ax*bw + ay*bz - az*by, + aw*by - ax*bz + ay*bw + az*bx, + aw*bz + ax*by - ay*bx + az*bw, + aw*bw - ax*bx - ay*by - az*bz], -1) + + +def qconj(q): + return q * np.array([-1, -1, -1, 1], q.dtype) + + +def qrot(q, v): + """Rotate vectors v[...,3] by quats q[...,4].""" + qv = q[..., :3] + uv = np.cross(qv, v) + uuv = np.cross(qv, uv) + return v + 2.0 * (q[..., 3:4] * uv + uuv) + + +def qnorm(q): + return q / (np.linalg.norm(q, axis=-1, keepdims=True) + 1e-12) + + +def arc(a, b): + """Shortest-arc quats rotating unit vectors a[...,3] onto b[...,3].""" + d = (a * b).sum(-1, keepdims=True) + axis = np.cross(a, b) + w = 1.0 + d + q = np.concatenate([axis, w], -1) + # antiparallel: rotate π about any perpendicular of a + bad = (w[..., 0] < 1e-6) + if np.any(bad): + ab = a[bad] + perp = np.cross(ab, np.broadcast_to([1.0, 0, 0], ab.shape)) + n = np.linalg.norm(perp, axis=-1, keepdims=True) + alt = np.cross(ab, np.broadcast_to([0, 1.0, 0], ab.shape)) + perp = np.where(n > 1e-6, perp, alt) + perp /= (np.linalg.norm(perp, axis=-1, keepdims=True) + 1e-12) + q[bad] = np.concatenate([perp, np.zeros_like(perp[..., :1])], -1) + return qnorm(q) + + +def axis_angle(axis, ang): + h = ang[..., None] * 0.5 + return np.concatenate([axis * np.sin(h), np.cos(h)], -1) + + +def canonicalize(wq, rest_world, rest_dir): + """wq[T,J,4] world quats + reference triple → (Q'[T,J,4], valid[J]). + + Q' is the rig-independent training quat; invalid joints (zero restDir) + are identity with valid=0. + """ + T = wq.shape[0] + out = np.zeros((T, J, 4), np.float32) + out[..., 3] = 1.0 + valid = np.zeros(J, np.float32) + for c in range(J): + dr = np.asarray(rest_dir[c], np.float32) + n = np.linalg.norm(dr) + if n < 1e-6: + continue + dr = dr / n + qr = qnorm(np.asarray(rest_world[c], np.float32)) + a_s = qrot(qconj(qr)[None], dr[None])[0] # bone's local axis + W = qnorm(wq[:, c]) # [T,4] + d = qrot(W, np.broadcast_to(a_s, (T, 3))) # [T,3] world dir + d = d / (np.linalg.norm(d, axis=-1, keepdims=True) + 1e-12) + delta = qmul(W, np.broadcast_to(qconj(qr), (T, 4))) + aim_s = arc(np.broadcast_to(dr, (T, 3)), d) + tw = qmul(qconj(aim_s), delta) # twist about dr + th = 2.0 * np.arctan2((tw[:, :3] * dr).sum(-1), tw[:, 3]) + th = np.unwrap(th) + # cap runaway unwrap drift (numeric noise on near-degenerate aims) + th = np.clip(th, -2.5 * np.pi, 2.5 * np.pi) + aim_c = arc(np.broadcast_to(D_CANON[c], (T, 3)), d) + out[:, c] = qmul(axis_angle(d, th), aim_c) + valid[c] = 1.0 + return qnorm(out), valid + + +# ---------- corpus source ---------- +def corpus_clips(corpus, min_roles): + """Yield (action, Q'[T,J,4], valid[J], src) from every sidecar dump.""" + manifest = {} + mpath = os.path.join(corpus, "manifest.json") + if os.path.exists(mpath): + manifest = json.load(open(mpath)) + dumps = sorted(glob.glob(os.path.join(corpus, "raw", "**", + "*.canonical.json"), recursive=True)) + for dpath in dumps: + try: + dump = json.load(open(dpath)) + except Exception: + continue + asset = os.path.basename(os.path.dirname(dpath)) + prov = libv5.manifest_lookup(manifest, asset) or {} + tags = prov.get("tags", []) + title = prov.get("title", asset) + for c in dump.get("clips", []): + if c.get("resolvedRoles", 0) < min_roles: + continue + q = c.get("quats", []) + rw = c.get("restWorld", []) + rd = c.get("restDir", []) + if len(q) < 12 or len(rw) != J or len(rd) != J: + continue + action = libv5.action_for(c.get("animation", ""), tags) + if not action: + continue + # the library's uprightness/energy gate (#855) — same hygiene bar + e = libv5.frame_energy(q) + me = sum(e) / max(1, len(e)) + if me < 0.004: + continue + quality, drop = libv5.clip_quality( + action, q, rw, rd, c.get("resolvedRoles", 0), me, 0.004) + if drop: + continue + wq = np.asarray(q, np.float32) # [T,J,4] + cq, valid = canonicalize(wq, rw, rd) + yield action, cq, valid, f"{title} — {c.get('animation')}" + + +# ---------- CMU source ---------- +def bvh_rest(path): + """Rest joint positions from a BVH's OFFSET tree → restDir per canonical + joint (identity rest rotations ⇒ world dir = offset-chain direction).""" + from bvh import Bvh + with open(path) as f: + m = Bvh(f.read()) + cmap = prep4.resolve_canon(m.get_joints_names()) + if cmap is None: + return None + pos = {} + for name in m.get_joints_names(): + p, cur = np.zeros(3), name + while cur is not None: + p = p + np.asarray(m.joint_offset(cur), np.float64) + try: + par = m.joint_parent(cur) + cur = par.name if par else None + except Exception: + cur = None + pos[name] = p + rd = np.zeros((J, 3), np.float32) + for c in range(J): + if c in CHILD: + v = pos[cmap[CANON[CHILD[c]]]] - pos[cmap[CANON[c]]] + else: # tips: head +Y, hands follow the forearm, feet +Z + if c == 5: + v = np.array([0, 1.0, 0]) + elif c in (9, 13): + v = pos[cmap[CANON[c]]] - pos[cmap[CANON[PARENT[c]]]] + else: + v = np.array([0, 0, 1.0]) + n = np.linalg.norm(v) + rd[c] = (v / n) if n > 1e-6 else 0.0 + return rd + + +def cmu_clips(bvh_dir, index): + labels = prep4.load_index(index) + files = sorted(glob.glob(os.path.join(bvh_dir, "**/*.bvh"), recursive=True)) + ident = np.zeros((J, 4), np.float32) + ident[:, 3] = 1.0 + for path in files: + mid_m = re.match(r"(\d+_\d+)", os.path.basename(path)) + mid = mid_m.group(1) if mid_m else None + if mid is None or mid not in labels: + continue + r = prep4.parse_bvh(path) + if r is None: + continue + _lq, wq = r + rd = bvh_rest(path) + if rd is None: + continue + cq, valid = canonicalize(wq, ident, rd) + yield labels[mid], cq, valid, f"CMU {mid}" + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--corpus", default="") + ap.add_argument("--bvh", default="") + ap.add_argument("--index", default="") + ap.add_argument("--out", default="/tmp/t2m_v5.npz") + ap.add_argument("--T", type=int, default=40) + ap.add_argument("--min-roles", type=int, default=12) + ap.add_argument("--min-action-windows", type=int, default=8) + a = ap.parse_args() + + mo, msk, acts, srcs = [], [], [], [] + + def angdist(x, y): + d = np.abs((x * y).sum(-1)).clip(0, 1) + return 2.0 * np.arccos(d) + + def window(action, cq, valid, src): + # The v4 neutral-start gate is deliberately GONE: model clips now ride + # the bind-referenced direction retarget, which references the + # reference TRIPLE, not clip frame 0 — any phase is a valid window. + T, nF = a.T, cq.shape[0] + if nF < max(16, T // 2): + return + if nF < T: + # pad: loop-tile cyclic clips (end pose ≈ start pose), else hold + cyc = angdist(cq[-1], cq[0]).mean() < 0.25 + reps = [cq] + while sum(r.shape[0] for r in reps) < T: + reps.append(cq if cyc else cq[-1:].repeat(T, 0)) + cq = np.concatenate(reps, 0)[:T] + nF = T + for s in range(0, nF - T + 1, max(1, T // 2)): + mo.append(cq[s:s + T]) + msk.append(valid) + acts.append(action) + srcs.append(src) + + if a.corpus: + n0 = 0 + for action, cq, valid, src in corpus_clips( + os.path.expanduser(a.corpus), a.min_roles): + window(action, cq, valid, src) + n0 += 1 + print(f"corpus: {n0} clips → {len(mo)} windows") + if a.bvh and a.index: + n0, w0 = 0, len(mo) + for action, cq, valid, src in cmu_clips(a.bvh, a.index): + window(action, cq, valid, src) + n0 += 1 + print(f"cmu: {n0} trials → {len(mo) - w0} windows") + + if not mo: + sys.exit("no windows extracted") + + # vocab = actions with enough support + from collections import Counter + cnt = Counter(acts) + vocab = sorted(w for w, n in cnt.items() if n >= a.min_action_windows) + keep = [i for i, w in enumerate(acts) if w in vocab] + mo = np.stack([mo[i] for i in keep]).astype(np.float32) + msk = np.stack([msk[i] for i in keep]).astype(np.float32) + tk = np.zeros((len(keep), len(vocab)), np.float32) + for r, i in enumerate(keep): + tk[r, vocab.index(acts[i])] = 1.0 + print(f"windows: {mo.shape[0]} vocab({len(vocab)}):", + {w: int(tk[:, vocab.index(w)].sum()) for w in vocab}) + np.savez_compressed(a.out, mo=mo, msk=msk, tk=tk, + vocab=np.array(vocab), fps=FPS, + canonRestDir=D_CANON) + print(f"wrote {a.out} mo{mo.shape}") + + +if __name__ == "__main__": + main() diff --git a/scripts/train-t2m-flow-v5.py b/scripts/train-t2m-flow-v5.py new file mode 100644 index 00000000..2c5754da --- /dev/null +++ b/scripts/train-t2m-flow-v5.py @@ -0,0 +1,289 @@ +#!/usr/bin/env python3 +# ruff: noqa: E702, E741 +"""Train the v5 FLOW-MATCHING text-to-motion model (#840, epic #837). + +ONE-TIME OFFLINE dev tool — NOT shipped. Replaces the v4 CVAE: a conditional- +mean decoder averages an action's phase-misaligned windows into gentle motion +(the v4 training notes measured this); flow matching samples a transport path +from noise to A SINGLE MODE of the data distribution instead — the one +architectural change that separates MDM-era quality from the CVAE, +independent of data scale (#837). + +Data: /tmp/t2m_v5.npz from prep-t2m-v5.py — CANONICALIZED quats (rig- +independent aim+twist against the fixed canonical T-pose, #858), per-joint +validity masks, one-hot action labels. + +Architecture: small DiT-style transformer v(x_t, t, action): + x[B,T,132] (22 joints × 6D rotation) + sinusoidal frame positions, + conditioned on (flow time t, action embedding) via AdaLN-Zero-lite + (per-layer scale/shift from the conditioning vector). Rectified-flow + objective: x_t=(1−t)x0+t·x1, target v*=x1−x0, masked joint-wise MSE. + +Export: ONNX graph with the EULER SAMPLER UNROLLED INSIDE (N fixed steps), +matching the shipped MotionGenerator contract exactly: + inputs tokens[1,V] one-hot, seed[1,Z] (Z = T·132 flattened noise; + MotionGenerator draws N(0,0.5) — the graph rescales ×2 to unit) + output motion[1,T,220] (per joint: tx,ty,tz=0, + quat x,y,z,w from Gram-Schmidt 6D→R, sx,sy,sz=1) +plus t2m-vocab.json carrying the CANONICAL REFERENCE TRIPLE (restWorld = +identity, restDir = the fixed canonical T-pose directions) so model clips +ride the same bind-referenced direction retarget as v5 template clips — +retiring the synthetic-standing-pose shim (#858). + +Usage: + python3 scripts/train-t2m-flow-v5.py --data /tmp/t2m_v5.npz \ + --out /tmp/t2m_v5_flow --epochs 60 --device mps [--steps 16] +""" +import argparse +import json +import math +import os + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F + +J, D6 = 22, 6 +C6 = J * D6 # 132 + + +# ---------- rotation reps ---------- +def quat_to_6d(q): + """q[...,4] (x,y,z,w) → first two rotation-matrix COLUMNS [...,6].""" + x, y, z, w = q.unbind(-1) + c0 = torch.stack([1 - 2 * (y * y + z * z), + 2 * (x * y + z * w), + 2 * (x * z - y * w)], -1) + c1 = torch.stack([2 * (x * y - z * w), + 1 - 2 * (x * x + z * z), + 2 * (y * z + x * w)], -1) + return torch.cat([c0, c1], -1) + + +def d6_to_quat(d): + """[...,6] → unit quats [...,4] (x,y,z,w) via Gram-Schmidt (ONNX-safe).""" + a, b = d[..., :3], d[..., 3:] + c0 = F.normalize(a, dim=-1, eps=1e-6) + b = b - (c0 * b).sum(-1, keepdim=True) * c0 + c1 = F.normalize(b, dim=-1, eps=1e-6) + c2 = torch.cross(c0, c1, dim=-1) + m00, m10, m20 = c0.unbind(-1) + m01, m11, m21 = c1.unbind(-1) + m02, m12, m22 = c2.unbind(-1) + # branchless Shepperd: build all four candidates, pick the max-norm one + t0 = 1 + m00 + m11 + m22 + t1 = 1 + m00 - m11 - m22 + t2 = 1 - m00 + m11 - m22 + t3 = 1 - m00 - m11 + m22 + eps = 1e-8 + q0 = torch.stack([m21 - m12, m02 - m20, m10 - m01, t0], -1) \ + / (2.0 * torch.sqrt(t0.clamp_min(eps)).unsqueeze(-1)) + q1 = torch.stack([t1, m01 + m10, m02 + m20, m21 - m12], -1) \ + / (2.0 * torch.sqrt(t1.clamp_min(eps)).unsqueeze(-1)) + q2 = torch.stack([m01 + m10, t2, m12 + m21, m02 - m20], -1) \ + / (2.0 * torch.sqrt(t2.clamp_min(eps)).unsqueeze(-1)) + q3 = torch.stack([m02 + m20, m12 + m21, t3, m10 - m01], -1) \ + / (2.0 * torch.sqrt(t3.clamp_min(eps)).unsqueeze(-1)) + ts = torch.stack([t0, t1, t2, t3], -1) + idx = ts.argmax(-1, keepdim=True) + qs = torch.stack([q0, q1, q2, q3], -2) # [...,4cand,4] + q = torch.gather(qs, -2, + idx.unsqueeze(-1).expand(*idx.shape, 4)).squeeze(-2) + return F.normalize(q, dim=-1, eps=1e-6) + + +# ---------- model ---------- +class Block(nn.Module): + def __init__(self, dim, heads): + super().__init__() + self.n1 = nn.LayerNorm(dim, elementwise_affine=False) + self.attn = nn.MultiheadAttention(dim, heads, batch_first=True) + self.n2 = nn.LayerNorm(dim, elementwise_affine=False) + self.mlp = nn.Sequential(nn.Linear(dim, dim * 4), nn.GELU(), + nn.Linear(dim * 4, dim)) + self.ada = nn.Linear(dim, dim * 6) + nn.init.zeros_(self.ada.weight) + nn.init.zeros_(self.ada.bias) + + def forward(self, x, c): + s1, b1, g1, s2, b2, g2 = self.ada(c).unsqueeze(1).chunk(6, -1) + h = self.n1(x) * (1 + s1) + b1 + x = x + g1 * self.attn(h, h, h, need_weights=False)[0] + h = self.n2(x) * (1 + s2) + b2 + return x + g2 * self.mlp(h) + + +class FlowDiT(nn.Module): + def __init__(self, V, T, dim=256, layers=6, heads=8): + super().__init__() + self.T = T + self.inp = nn.Linear(C6, dim) + self.pos = nn.Parameter(torch.randn(1, T, dim) * 0.02) + self.act_emb = nn.Linear(V, dim) + self.t_mlp = nn.Sequential(nn.Linear(dim, dim), nn.SiLU(), + nn.Linear(dim, dim)) + self.blocks = nn.ModuleList([Block(dim, heads) for _ in range(layers)]) + self.out_n = nn.LayerNorm(dim, elementwise_affine=False) + self.out = nn.Linear(dim, C6) + nn.init.zeros_(self.out.weight) + nn.init.zeros_(self.out.bias) + half = dim // 2 + self.register_buffer( + "freqs", torch.exp(-math.log(1e4) + * torch.arange(half).float() / half)) + + def forward(self, x, t, tok): + # x[B,T,C6], t[B] in [0,1], tok[B,V] + ang = t[:, None] * 1000.0 * self.freqs[None] + temb = torch.cat([torch.sin(ang), torch.cos(ang)], -1) + c = self.t_mlp(temb) + self.act_emb(tok) + h = self.inp(x) + self.pos + for blk in self.blocks: + h = blk(h, c) + return self.out(self.out_n(h)) + + +class Sampler(nn.Module): + """Euler flow sampler UNROLLED for ONNX export — MotionGenerator contract.""" + + def __init__(self, net, V, T, steps, guidance=2.0): + super().__init__() + self.net, self.V, self.T, self.steps = net, V, T, steps + self.guidance = guidance + + def forward(self, tokens, seed): + B = 1 + # MotionGenerator draws seed ~ N(0, 0.5) — rescale to unit noise. + x = seed.reshape(B, self.T, C6) * 2.0 + uncond = torch.zeros_like(tokens) + for i in range(self.steps): + t = torch.full((B,), i / self.steps, dtype=x.dtype, + device=x.device) + vc = self.net(x, t, tokens) + vu = self.net(x, t, uncond) + v = vu + self.guidance * (vc - vu) + x = x + v / self.steps + q = d6_to_quat(x.reshape(B, self.T, J, D6)) # [1,T,J,4] + zeros3 = torch.zeros(B, self.T, J, 3, dtype=x.dtype, device=x.device) + ones3 = torch.ones(B, self.T, J, 3, dtype=x.dtype, device=x.device) + motion = torch.cat([zeros3, q, ones3], -1) # [1,T,J,10] + return motion.reshape(B, self.T, J * 10) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--data", default="/tmp/t2m_v5.npz") + ap.add_argument("--out", default="/tmp/t2m_v5_flow") + ap.add_argument("--epochs", type=int, default=60) + ap.add_argument("--batch", type=int, default=256) + ap.add_argument("--lr", type=float, default=2e-4) + ap.add_argument("--dim", type=int, default=256) + ap.add_argument("--layers", type=int, default=6) + ap.add_argument("--steps", type=int, default=16, help="Euler export steps") + ap.add_argument("--guidance", type=float, default=2.0, + help="CFG scale baked into the exported sampler") + ap.add_argument("--device", default="mps") + a = ap.parse_args() + + d = np.load(a.data, allow_pickle=True) + mo, msk, tk = d["mo"], d["msk"], d["tk"] + vocab = [str(w) for w in d["vocab"]] + fps = int(d["fps"]) + canon_rd = d["canonRestDir"] + N, T = mo.shape[0], mo.shape[1] + V = len(vocab) + print(f"data: {N} windows T={T} V={V} vocab={vocab}") + + dev = torch.device(a.device if (a.device != "mps" + or torch.backends.mps.is_available()) + else "cpu") + x1 = quat_to_6d(torch.from_numpy(mo)).reshape(N, T, C6) # data + m6 = torch.from_numpy(msk).repeat_interleave(D6, -1) # [N,C6] + tok = torch.from_numpy(tk) + + # class-balanced sampling — walk is 64% of windows (v4 lesson) + freq = tk.sum(0) + w = (tk @ (1.0 / np.maximum(freq, 1.0))).astype(np.float64) + sampler = torch.utils.data.WeightedRandomSampler( + torch.from_numpy(w), num_samples=N, replacement=True) + ds = torch.utils.data.TensorDataset(x1, m6, tok) + dl = torch.utils.data.DataLoader(ds, batch_size=a.batch, sampler=sampler, + drop_last=True) + + net = FlowDiT(V, T, dim=a.dim, layers=a.layers).to(dev) + print("params:", sum(p.numel() for p in net.parameters()) / 1e6, "M") + opt = torch.optim.AdamW(net.parameters(), lr=a.lr, weight_decay=1e-4) + sched = torch.optim.lr_scheduler.CosineAnnealingLR( + opt, T_max=a.epochs * max(1, len(dl))) + + for ep in range(a.epochs): + tot, nb = 0.0, 0 + for xb, mb, tb in dl: + xb, mb, tb = xb.to(dev), mb.to(dev), tb.to(dev) + x0 = torch.randn_like(xb) + # classifier-free guidance: drop the action condition 10% of the + # time so the sampler can extrapolate cond vs uncond at export. + drop = (torch.rand(tb.shape[0], device=dev) < 0.1).float() + tb = tb * (1.0 - drop)[:, None] + t = torch.rand(xb.shape[0], device=dev) + xt = (1 - t[:, None, None]) * x0 + t[:, None, None] * xb + v = net(xt, t, tb) + tgt = xb - x0 + mask = mb[:, None, :] # [B,1,C6] + loss = ((v - tgt) ** 2 * mask).sum() / mask.sum() / T + opt.zero_grad(set_to_none=True) + loss.backward() + torch.nn.utils.clip_grad_norm_(net.parameters(), 1.0) + opt.step() + sched.step() + tot += loss.item(); nb += 1 + print(f"ep {ep + 1}/{a.epochs} loss {tot / max(1, nb):.4f}", flush=True) + + os.makedirs(a.out, exist_ok=True) + torch.save(net.state_dict(), os.path.join(a.out, "flow.pt")) + + # ---- export: sampler-unrolled ONNX + vocab json ---- + net_cpu = FlowDiT(V, T, dim=a.dim, layers=a.layers) + net_cpu.load_state_dict({k: v.cpu() for k, v in net.state_dict().items()}) + net_cpu.eval() + samp = Sampler(net_cpu, V, T, a.steps, a.guidance).eval() + Z = T * C6 + tokens = torch.zeros(1, V); tokens[0, 0] = 1.0 + seed = torch.randn(1, Z) * 0.5 + onnx_path = os.path.join(a.out, "t2m.onnx") + torch.onnx.export(samp, (tokens, seed), onnx_path, + input_names=["tokens", "seed"], + output_names=["motion"], opset_version=17, + dynamo=False) + vj = { + "vocab": vocab, "Z": Z, "T": T, "C": J * 10, "J": J, + "fps": fps, "frame": "world", "version": "v5-flow", + "flowSteps": a.steps, + # #858: the canonical reference triple — model clips ride the same + # bind-referenced direction retarget as v5 template clips. + "restWorld": [[0.0, 0.0, 0.0, 1.0]] * J, + "restDir": [[float(v) for v in row] for row in canon_rd], + } + with open(os.path.join(a.out, "t2m-vocab.json"), "w") as f: + json.dump(vj, f) + print(f"exported {onnx_path} " + f"({os.path.getsize(onnx_path) / 1e6:.1f} MB) + vocab") + + # sanity: run one sample through onnxruntime + try: + import onnxruntime as ort + s = ort.InferenceSession(onnx_path, + providers=["CPUExecutionProvider"]) + out = s.run(None, {"tokens": tokens.numpy(), + "seed": seed.numpy()})[0] + q = out[0, :, 3:7] + nrm = np.linalg.norm(out[0].reshape(T, J, 10)[..., 3:7], axis=-1) + print("onnx ok:", out.shape, "quat norms", + nrm.min().round(4), nrm.max().round(4)) + except Exception as e: # noqa: BLE001 + print("onnx check failed:", e) + + +if __name__ == "__main__": + main() diff --git a/src/AnimationControlController.cpp b/src/AnimationControlController.cpp index fce0914a..5c307b91 100644 --- a/src/AnimationControlController.cpp +++ b/src/AnimationControlController.cpp @@ -1877,10 +1877,18 @@ QVariantMap AnimationControlController::generateMotion(const QString& prompt, if (mr.ok) { action = mr.matchedAction; quats = mr.clip.quats; fps = mr.clip.fps; worldFrame = mr.worldFrame; clipSource = QStringLiteral("model"); gotClip = true; - // Borrow a template clip's reference directions so the - // retarget synthesizes a BIND-referenced base pose (no - // harvest from the rig's other animations). - clipDirs = MotionLibrary::referenceDirsForPrompt(prompt); + if (!mr.clip.restWorld.empty() && !mr.clip.restDir.empty()) { + // v5 models (#858) ship their canonical reference triple + // — same bind-referenced retarget as template clips. + cmuRest = mr.clip.restWorld; + clipDirs = mr.clip.restDir; + } else { + // Legacy v4: borrow a template clip's reference + // directions so the retarget synthesizes a BIND- + // referenced base pose (no harvest from the rig's + // other animations). + clipDirs = MotionLibrary::referenceDirsForPrompt(prompt); + } } } if (!gotClip) diff --git a/src/AnimationMerger.cpp b/src/AnimationMerger.cpp index 9013fc36..4f7ecd91 100644 --- a/src/AnimationMerger.cpp +++ b/src/AnimationMerger.cpp @@ -1587,13 +1587,17 @@ AnimationMerger::ApplyMotionResult AnimationMerger::applyMotionClip( // Generated clips therefore reference the TARGET BIND — no pose from any // other animation is involved (the standing-pose harvest below is only // the fallback for restWorld-less clips, e.g. the CMU-built libraries). + // A reference orientation is PRESENT when it is a valid (unit-norm) quat; + // unresolved roles in dumps are zero-filled. Identity counts — the v5 + // model's canonical triple (#858) is restWorld = identity ×22 by + // construction, and treating it as "absent" would silently demote model + // clips to the synthetic-standing-pose path. bool haveRestWorld = false; if (worldFrame && cmuRestWorld.size() == static_cast( MotionInbetween::canonicalJointCount())) { for (const auto& q : cmuRestWorld) - if (std::abs(q[0]) > 1e-5f || std::abs(q[1]) > 1e-5f - || std::abs(q[2]) > 1e-5f || std::abs(1.f - std::abs(q[3])) > 1e-5f) { + if (q[0] * q[0] + q[1] * q[1] + q[2] * q[2] + q[3] * q[3] > 0.25f) { haveRestWorld = true; break; } diff --git a/src/CLIPipeline.cpp b/src/CLIPipeline.cpp index 4344a978..147c718d 100644 --- a/src/CLIPipeline.cpp +++ b/src/CLIPipeline.cpp @@ -2056,12 +2056,21 @@ int CLIPipeline::cmdAnimGenerate(const QString& filePath, const QString& prompt, action = mr.matchedAction; quats = mr.clip.quats; fps = mr.clip.fps; - worldFrame = mr.worldFrame; // v4 models: world-frame quats - // Model clips carry no reference triple — borrow a template - // clip's canonical bone directions so the retarget can - // synthesize a BIND-referenced base pose instead of - // harvesting the rig's other animations (contamination). - clipDirs = MotionLibrary::referenceDirsForPrompt(prompt); + worldFrame = mr.worldFrame; // v4/v5 models: world-frame quats + if (!mr.clip.restWorld.empty() && !mr.clip.restDir.empty()) { + // v5 models (#858) ship their canonical reference triple + // in the vocab — the clip rides the same bind-referenced + // direction retarget as v5 template clips. + cmuRest = mr.clip.restWorld; + clipDirs = mr.clip.restDir; + } else { + // Legacy v4 models carry no reference triple — borrow a + // template clip's canonical bone directions so the + // retarget can synthesize a BIND-referenced base pose + // instead of harvesting the rig's other animations + // (contamination). + clipDirs = MotionLibrary::referenceDirsForPrompt(prompt); + } clipSource = QStringLiteral("model"); gotClip = true; } else { diff --git a/src/MCPServer.cpp b/src/MCPServer.cpp index d4fccd68..ce73fb26 100644 --- a/src/MCPServer.cpp +++ b/src/MCPServer.cpp @@ -4073,10 +4073,18 @@ QJsonObject MCPServer::toolGenerateMotion(const QJsonObject &args) if (mr.ok) { action = mr.matchedAction; quats = mr.clip.quats; fps = mr.clip.fps; worldFrame = mr.worldFrame; clipSource = QStringLiteral("model"); gotClip = true; - // Borrow a template clip's reference directions so the - // retarget synthesizes a BIND-referenced base pose (no - // harvest from the rig's other animations). - clipDirs = MotionLibrary::referenceDirsForPrompt(prompt); + if (!mr.clip.restWorld.empty() && !mr.clip.restDir.empty()) { + // v5 models (#858) ship their canonical reference + // triple — same bind-referenced retarget as templates. + cmuRest = mr.clip.restWorld; + clipDirs = mr.clip.restDir; + } else { + // Legacy v4: borrow a template clip's reference + // directions so the retarget synthesizes a + // BIND-referenced base pose (no harvest from the + // rig's other animations). + clipDirs = MotionLibrary::referenceDirsForPrompt(prompt); + } } } } diff --git a/src/MotionGenerator.cpp b/src/MotionGenerator.cpp index 5e7f973d..e8f8cd14 100644 --- a/src/MotionGenerator.cpp +++ b/src/MotionGenerator.cpp @@ -191,6 +191,39 @@ MotionGenerator::Result MotionGenerator::generate( const bool worldFrame = vj.value("frame").toString() == QLatin1String("world"); const int fps = vj.value("fps").toInt(30); + // v5 models (#858) train on CANONICALIZED quats and ship their + // reference triple in the vocab: restWorld (identity ×22) + restDir + // (the fixed canonical T-pose directions). With it, model clips ride + // the SAME bind-referenced direction retarget as v5 template clips — + // no synthetic-standing-pose shim. + std::vector> vocabRestWorld; + std::vector> vocabRestDir; + { + const QJsonArray rw = vj.value("restWorld").toArray(); + const QJsonArray rd = vj.value("restDir").toArray(); + if (rw.size() == J && rd.size() == J) { + for (const QJsonValue& v : rw) { + const QJsonArray q = v.toArray(); + if (q.size() != 4) { vocabRestWorld.clear(); break; } + vocabRestWorld.push_back({float(q[0].toDouble()), + float(q[1].toDouble()), + float(q[2].toDouble()), + float(q[3].toDouble())}); + } + for (const QJsonValue& v : rd) { + const QJsonArray d = v.toArray(); + if (d.size() != 3) { vocabRestDir.clear(); break; } + vocabRestDir.push_back({float(d[0].toDouble()), + float(d[1].toDouble()), + float(d[2].toDouble())}); + } + if (vocabRestWorld.size() != static_cast(J) + || vocabRestDir.size() != static_cast(J)) { + vocabRestWorld.clear(); + vocabRestDir.clear(); + } + } + } const int V = vocab.size(); if (V == 0 || T <= 0 || C != J * 10) { r.error = QStringLiteral("t2m vocab json malformed"); return r; @@ -375,6 +408,8 @@ MotionGenerator::Result MotionGenerator::generate( clip.frames = want; } + clip.restWorld = vocabRestWorld; + clip.restDir = vocabRestDir; r.clip = std::move(clip); r.worldFrame = worldFrame; r.ok = true; diff --git a/src/MotionGenerator.h b/src/MotionGenerator.h index 081802f0..a353b5f3 100644 --- a/src/MotionGenerator.h +++ b/src/MotionGenerator.h @@ -22,11 +22,17 @@ // model is unavailable (no ENABLE_ONNX, model not downloaded, prompt action not // in the model's vocab, or inference fails). // -// ONNX contract (must match scripts/train-t2m-onnx-v3.py export): +// ONNX contract (v5: scripts/train-t2m-flow-v5.py; v4: train-t2m-onnx-v4.py): // input "tokens" float32 [1, V] one-hot/bag over the fixed action vocab -// input "seed" float32 [1, Z] latent (we pass zeros for the mean clip) +// input "seed" float32 [1, Z] latent noise (sampled; best-of-N scored) // output "motion" float32 [1, T, C] C = 22*10 per-joint [t.xyz, q.xyzw, s.xyz] -// The accompanying t2m-vocab.json gives {vocab, Z, T, C, J, joints}. +// The accompanying t2m-vocab.json gives {vocab, Z, T, C, J, fps, frame}. +// v5 FLOW-MATCHING models (#840/#858) keep this exact interface: the Euler +// sampler is unrolled INSIDE the exported graph (seed = the flattened noise +// tensor, Z = T·132), and the vocab additionally carries the model's +// canonical reference triple ("restWorld" identity ×22 + "restDir" canonical +// T-pose directions) so generated clips ride the same bind-referenced +// direction retarget as v5 template clips (no standing-pose shim). class MotionGenerator { public: MotionGenerator() = delete; From 8cede98cab6f3512ef7697f4082047bd17dbc6d4 Mon Sep 17 00:00:00 2001 From: Fernando Date: Mon, 13 Jul 2026 09:14:54 -0300 Subject: [PATCH 03/23] feat(anim): foot-contact detection + IK pinning post-pass (#856) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Retargeted clips slide/float feet on rigs whose proportions differ from the source — the direction retarget transfers bone directions, not world foot positions. Core (FootContact.h/cpp, Ogre-free + unit-tested): contact detection (foot within a leg-length-scaled height band of the clip's ground level AND nearly stationary horizontally -> spans; ground = low percentile so jump clips don't skew it), analytic two-bone IK (solveKnee — keeps both segment lengths, preserves the pose's own bend plane, clamps unreachable targets), and the 0->1->0 edge blend weight. AnimationMerger::pinFeet: manual FK over the tracks (pure track math — the live skeleton is never apply()'d), detection in the canonical frame (Ct) so ground is horizontal regardless of rig axes, then per contact frame locks the foot to its span-start position blended over blendFrames: thigh re-aimed at the IK knee, shin at the pinned target, foot keyframe compensated to keep its ORIGINAL world orientation (no toe pop). Only thigh/shin/foot keyframes rewritten; _keyFrameDataChanged on each (the #854 cache gotcha). Effectively idempotent (re-detection re-pins to the same targets); 'qtme.footpin.' marker on bone[0], cleared when applyMotionClip regenerates the clip. Surfaces: ON by default in generation — CLI --no-foot-pin opts out, standalone 'qtmesh anim --foot-pin --animation -o out'; MCP foot_pin arg on generate_motion + standalone pin_feet tool; GUI 'Pin feet (contact cleanup)' checkbox in the Animations section. Also: canonicalIndexForBone learns 'upperleg' as a thigh alias — Quaternius-style rigs (UpperLeg.L/LowerLeg.L) previously mapped BOTH leg bones to the knee role, which broke pinning and double-aimed the knee in the retarget. Verified: Rumba walk 2 contact spans pinned; Knight (UpperLeg/LowerLeg naming) pins after the alias fix; renders upright, stride intact. Co-Authored-By: Claude Fable 5 --- qml/PropertiesPanel.qml | 25 ++- src/AnimationControlController.cpp | 9 +- src/AnimationControlController.h | 5 +- src/AnimationMerger.cpp | 236 ++++++++++++++++++++++++++++- src/AnimationMerger.h | 31 ++++ src/CLIPipeline.cpp | 63 +++++++- src/CLIPipeline.h | 2 +- src/CMakeLists.txt | 2 + src/FootContact.cpp | 115 ++++++++++++++ src/FootContact.h | 65 ++++++++ src/FootContact_test.cpp | 124 +++++++++++++++ src/MCPServer.cpp | 98 ++++++++++++ src/MCPServer.h | 1 + src/MotionInbetween.cpp | 4 +- tests/CMakeLists.txt | 1 + 15 files changed, 772 insertions(+), 9 deletions(-) create mode 100644 src/FootContact.cpp create mode 100644 src/FootContact.h create mode 100644 src/FootContact_test.cpp diff --git a/qml/PropertiesPanel.qml b/qml/PropertiesPanel.qml index dba40f00..d8f0c1e1 100644 --- a/qml/PropertiesPanel.qml +++ b/qml/PropertiesPanel.qml @@ -8493,7 +8493,7 @@ Rectangle { // skewing every new clip is exactly the bug this fixes). var gr = AnimationControlController.generateMotion( genPromptIn.text, 0.0, useModelChk.checked, - 0.0) + 0.0, footPinChk.checked) if (gr && gr.animation) { lastGeneratedAnim = gr.animation // Point the slider at the fresh clip at a neutral 0 @@ -8536,6 +8536,29 @@ Rectangle { anchors.verticalCenter: parent.verticalCenter } } + // #856: foot-contact cleanup — ON by default (detect ground-contact + // spans + IK-pin the feet so they plant instead of skating). + Row { + spacing: 6 + Rectangle { + id: footPinChk + property bool checked: true + width: 14; height: 14; radius: 2 + anchors.verticalCenter: parent.verticalCenter + color: checked ? PropertiesPanelController.highlightColor + : PropertiesPanelController.inputColor + border.color: PropertiesPanelController.borderColor + Text { anchors.centerIn: parent; visible: parent.checked + text: "✓"; color: "white"; font.pixelSize: 10 } + MouseArea { anchors.fill: parent + onClicked: footPinChk.checked = !footPinChk.checked } + } + Text { + text: "Pin feet (contact cleanup)" + color: PropertiesPanelController.textColor; font.pixelSize: 10 + anchors.verticalCenter: parent.verticalCenter + } + } // ── Arm space (#854): Mixamo-style widen/tuck post-process ──────── // Targets `armSpaceAnim` — the last generated clip by default, or // any animation the user picks via the per-row arm button below. diff --git a/src/AnimationControlController.cpp b/src/AnimationControlController.cpp index fce0914a..02933ef2 100644 --- a/src/AnimationControlController.cpp +++ b/src/AnimationControlController.cpp @@ -1833,7 +1833,7 @@ double AnimationControlController::currentArmSpace(const QString& animName, QVariantMap AnimationControlController::generateMotion(const QString& prompt, double duration, bool useModel, - double armSpaceDeg) + double armSpaceDeg, bool footPin) { QVariantMap out; out["ok"] = false; @@ -1937,6 +1937,13 @@ QVariantMap AnimationControlController::generateMotion(const QString& prompt, AnimationMerger::adjustArmSpace(skel.get(), animName, static_cast(armSpaceDeg)); + // #856: foot-contact cleanup — ON by default (checkbox opts out). + if (footPin) { + const auto fp = AnimationMerger::pinFeet(skel.get(), animName); + if (fp.ok && fp.spans > 0) + out["footPinSpans"] = fp.spans; + } + entity->refreshAvailableAnimationState(); // Make the generated clip the ONLY enabled animation. Ogre AVERAGES all // enabled animation states, so leaving the import's auto-enabled clip (or diff --git a/src/AnimationControlController.h b/src/AnimationControlController.h index 41e3d138..7b6b35dd 100644 --- a/src/AnimationControlController.h +++ b/src/AnimationControlController.h @@ -343,10 +343,13 @@ class AnimationControlController : public QObject /// (MotionGenerator/ONNX); it falls back to the template library automatically /// when the model is unavailable or the action isn't in its vocab. Default /// false = the reliable template-clip retarget. + /// `footPin` (default true) runs the #856 foot-contact cleanup on the + /// generated clip (contact detection + two-bone IK pinning). Q_INVOKABLE QVariantMap generateMotion(const QString& prompt, double duration = 0.0, bool useModel = false, - double armSpaceDeg = 0.0); + double armSpaceDeg = 0.0, + bool footPin = true); /// #854: Mixamo-style arm-space post-process on an EXISTING animation of /// the selected entity. Positive `degrees` widens the arms away from the diff --git a/src/AnimationMerger.cpp b/src/AnimationMerger.cpp index 9013fc36..c7a8c59b 100644 --- a/src/AnimationMerger.cpp +++ b/src/AnimationMerger.cpp @@ -1,5 +1,6 @@ #include "AnimationMerger.h" #include "AutoRig.h" +#include "FootContact.h" #include "MotionInbetween.h" #include #include @@ -1470,6 +1471,235 @@ bool AnimationMerger::adjustArmSpace(Ogre::Skeleton* skel, return true; } +namespace { +std::string footPinKey(const std::string& animName) +{ + return "qtme.footpin." + animName; +} +} // namespace + +AnimationMerger::FootPinResult AnimationMerger::pinFeet( + Ogre::Skeleton* skel, const std::string& animName, int blendFrames) +{ + FootPinResult res; + if (!skel || !skel->hasAnimation(animName)) { + res.error = QStringLiteral("animation not found"); + return res; + } + Ogre::Animation* anim = skel->getAnimation(animName); + + const int nBones = static_cast(skel->getNumBones()); + std::vector boneToCanon(static_cast(nBones), -1); + for (int i = 0; i < nBones; ++i) + boneToCanon[static_cast(i)] = + MotionInbetween::canonicalIndexForBone(QString::fromStdString( + skel->getBone(static_cast(i))->getName())); + const TargetBindFrame tb = readTargetBindFrame(skel, boneToCanon); + // Bind-local POSITIONS (parent-relative) for the manual FK below — the + // skeleton is still in its reset pose after readTargetBindFrame. + std::vector bindLocalPos(static_cast(nBones)); + for (int i = 0; i < nBones; ++i) + bindLocalPos[static_cast(i)] = + skel->getBone(static_cast(i))->getPosition(); + + // Leg chains: thigh / knee / foot roles per side (first bone per role). + struct Leg { int thigh, shin, foot; }; + const Leg legs[2] = { + {tb.roleBoneIdx[15], tb.roleBoneIdx[16], tb.roleBoneIdx[17]}, // right + {tb.roleBoneIdx[19], tb.roleBoneIdx[20], tb.roleBoneIdx[21]}, // left + }; + + auto trackOf = [&](int bone) -> Ogre::NodeAnimationTrack* { + if (bone < 0 || !anim->hasNodeTrack(static_cast(bone))) + return nullptr; + auto* t = anim->getNodeTrack(static_cast(bone)); + return (t && t->getNumKeyFrames() > 0) ? t : nullptr; + }; + + // Keyframe times come from the first available thigh track — generated + // clips write one keyframe per frame on every mapped bone. + Ogre::NodeAnimationTrack* timeSrc = trackOf(legs[0].thigh); + if (!timeSrc) timeSrc = trackOf(legs[1].thigh); + if (!timeSrc) { + res.error = QStringLiteral("no leg tracks on this rig/animation"); + return res; + } + const int nk = static_cast(timeSrc->getNumKeyFrames()); + if (nk < 4) { + res.error = QStringLiteral("too few keyframes to detect contacts"); + return res; + } + std::vector times(static_cast(nk)); + for (int k = 0; k < nk; ++k) + times[static_cast(k)] = + timeSrc->getNodeKeyFrame(static_cast(k))->getTime(); + + // ---- manual FK over the tracks (pure math; the live skeleton — which + // may be a SkeletonInstance driving an on-screen entity — is untouched). + std::vector> Wrot( + static_cast(nk), + std::vector(static_cast(nBones))); + std::vector> Wpos( + static_cast(nk), + std::vector(static_cast(nBones))); + for (int k = 0; k < nk; ++k) { + const Ogre::TimeIndex ti = + anim->_getTimeIndex(times[static_cast(k)]); + for (int i : tb.order) { + Ogre::Quaternion lrot = tb.bindLocal[static_cast(i)]; + Ogre::Vector3 lpos = bindLocalPos[static_cast(i)]; + if (auto* trk = trackOf(i)) { + Ogre::TransformKeyFrame kf(nullptr, 0.0f); + trk->getInterpolatedKeyFrame(ti, &kf); + lrot = lrot * kf.getRotation(); // applyToNode: rotate() + lpos = lpos + kf.getTranslate(); // translate(TS_PARENT) + } + const int pi = tb.parentIdx[static_cast(i)]; + if (pi >= 0) { + Wrot[static_cast(k)][static_cast(i)] = + Wrot[static_cast(k)][static_cast(pi)] * lrot; + Wpos[static_cast(k)][static_cast(i)] = + Wpos[static_cast(k)][static_cast(pi)] + + Wrot[static_cast(k)][static_cast(pi)] * lpos; + } else { + Wrot[static_cast(k)][static_cast(i)] = lrot; + Wpos[static_cast(k)][static_cast(i)] = lpos; + } + } + } + + // Detection runs in the CANONICAL frame (Ct maps rig world → X=left, + // Y=up, Z=forward) so "ground" is a horizontal plane regardless of the + // rig's own axes. + auto toCanon = [&](const Ogre::Vector3& p) { return tb.Ct * p; }; + auto fromCanon = [&](const FootContact::V3& p) { + return tb.Ct.Inverse() * Ogre::Vector3(p[0], p[1], p[2]); + }; + auto v3 = [](const Ogre::Vector3& p) { + return FootContact::V3{p.x, p.y, p.z}; + }; + + for (const Leg& leg : legs) { + if (leg.thigh < 0 || leg.shin < 0 || leg.foot < 0) + continue; + auto* thighTrk = trackOf(leg.thigh); + auto* shinTrk = trackOf(leg.shin); + auto* footTrk = trackOf(leg.foot); + if (!thighTrk || !shinTrk + || static_cast(thighTrk->getNumKeyFrames()) != nk + || static_cast(shinTrk->getNumKeyFrames()) != nk + || (footTrk && static_cast(footTrk->getNumKeyFrames()) != nk)) + continue; // mixed keyframe grids — not a generated clip + + std::vector footC(static_cast(nk)); + for (int k = 0; k < nk; ++k) + footC[static_cast(k)] = v3(toCanon( + Wpos[static_cast(k)][static_cast(leg.foot)])); + const float legLen = + (Wpos[0][static_cast(leg.shin)] + - Wpos[0][static_cast(leg.thigh)]).length() + + (Wpos[0][static_cast(leg.foot)] + - Wpos[0][static_cast(leg.shin)]).length(); + const auto spans = FootContact::detectContacts(footC, legLen); + if (spans.empty()) + continue; + res.spans += static_cast(spans.size()); + + for (const auto& span : spans) { + const FootContact::V3 anchor = footC[static_cast(span.start)]; + for (int k = span.start; k <= span.end; ++k) { + const float w = FootContact::blendWeight(span, k, blendFrames); + if (w <= 0.0f) + continue; + const size_t kc = static_cast(k); + const Ogre::Vector3 hipW = Wpos[kc][static_cast(leg.thigh)]; + const Ogre::Vector3 kneeW = Wpos[kc][static_cast(leg.shin)]; + const Ogre::Vector3 footW = Wpos[kc][static_cast(leg.foot)]; + // pinned target: blend the current foot toward the anchor + const FootContact::V3 cur = footC[kc]; + const FootContact::V3 tgtC{ + cur[0] + (anchor[0] - cur[0]) * w, + cur[1] + (anchor[1] - cur[1]) * w, + cur[2] + (anchor[2] - cur[2]) * w}; + const FootContact::V3 kneeNewC = FootContact::solveKnee( + v3(toCanon(hipW)), v3(toCanon(kneeW)), v3(toCanon(footW)), + tgtC); + const Ogre::Vector3 kneeNew = fromCanon(kneeNewC); + const Ogre::Vector3 tgt = fromCanon(tgtC); + + Ogre::Vector3 dThighOld = kneeW - hipW; + Ogre::Vector3 dThighNew = kneeNew - hipW; + Ogre::Vector3 dShinOld = footW - kneeW; + Ogre::Vector3 dShinNew = tgt - kneeNew; + if (dThighOld.squaredLength() < 1e-12f + || dThighNew.squaredLength() < 1e-12f + || dShinOld.squaredLength() < 1e-12f + || dShinNew.squaredLength() < 1e-12f) + continue; + dThighOld.normalise(); dThighNew.normalise(); + dShinOld.normalise(); dShinNew.normalise(); + const Ogre::Quaternion S1 = dThighOld.getRotationTo(dThighNew); + const Ogre::Quaternion S2 = + (S1 * dShinOld).getRotationTo(dShinNew); + + // Rewrite the three keyframes as world premultipliers folded + // into parent-relative deltas (keyframes compose as + // local = bindLocal · kf; world = Wp · local). + const Ogre::Quaternion& WpT = + (tb.parentIdx[static_cast(leg.thigh)] >= 0) + ? Wrot[kc][static_cast( + tb.parentIdx[static_cast(leg.thigh)])] + : Ogre::Quaternion::IDENTITY; + const Ogre::Quaternion Wthigh = + Wrot[kc][static_cast(leg.thigh)]; + const Ogre::Quaternion WthighNew = S1 * Wthigh; + auto* kfT = thighTrk->getNodeKeyFrame( + static_cast(k)); + kfT->setRotation( + tb.bindLocal[static_cast(leg.thigh)].Inverse() + * WpT.Inverse() * WthighNew); + + const Ogre::Quaternion& WpS = + Wrot[kc][static_cast( + tb.parentIdx[static_cast(leg.shin)])]; + const Ogre::Quaternion Wshin = + Wrot[kc][static_cast(leg.shin)]; + const Ogre::Quaternion WshinNew = S2 * S1 * Wshin; + auto* kfS = shinTrk->getNodeKeyFrame( + static_cast(k)); + kfS->setRotation( + tb.bindLocal[static_cast(leg.shin)].Inverse() + * (S1 * WpS).Inverse() * WshinNew); + + if (footTrk) { + // foot keeps its ORIGINAL world orientation (no toe pop + // inherited from the parent corrections) + const Ogre::Quaternion& WpF = + Wrot[kc][static_cast( + tb.parentIdx[static_cast(leg.foot)])]; + auto* kfF = footTrk->getNodeKeyFrame( + static_cast(k)); + kfF->setRotation( + tb.bindLocal[static_cast(leg.foot)].Inverse() + * (S2 * S1 * WpF).Inverse() + * Wrot[kc][static_cast(leg.foot)]); + } + ++res.keyframesAdjusted; + } + } + // setRotation does NOT invalidate the track caches (#854 lesson). + thighTrk->_keyFrameDataChanged(); + shinTrk->_keyFrameDataChanged(); + if (footTrk) footTrk->_keyFrameDataChanged(); + } + + if (skel->getNumBones() > 0) + skel->getBone(0)->getUserObjectBindings().setUserAny( + footPinKey(animName), Ogre::Any(true)); + res.ok = true; + return res; +} + AnimationMerger::ApplyMotionResult AnimationMerger::applyMotionClip( Ogre::Skeleton* skel, const std::string& animName, @@ -1570,9 +1800,13 @@ AnimationMerger::ApplyMotionResult AnimationMerger::applyMotionClip( // stored angle from a prior generation of the same name would make a // re-request of that same angle a no-op (delta 0) on the new keyframes — // forget it. - if (skel->getNumBones() > 0) + if (skel->getNumBones() > 0) { skel->getBone(0)->getUserObjectBindings().eraseUserAny( armSpaceKey(animName)); + // #856: same for the foot-pin marker — a regenerated clip is unpinned. + skel->getBone(0)->getUserObjectBindings().eraseUserAny( + "qtme.footpin." + animName); + } Ogre::Animation* anim = skel->createAnimation(animName, length); anim->setInterpolationMode(Ogre::Animation::IM_LINEAR); anim->setRotationInterpolationMode(Ogre::Animation::RIM_LINEAR); diff --git a/src/AnimationMerger.h b/src/AnimationMerger.h index 543e72be..e1bc02f5 100644 --- a/src/AnimationMerger.h +++ b/src/AnimationMerger.h @@ -260,6 +260,37 @@ class AnimationMerger { const std::string& oldAnim, const std::string& newAnim); + /// Outcome of pinFeet. + struct FootPinResult { + bool ok = false; + QString error; + int spans = 0; ///< contact spans pinned (both feet) + int keyframesAdjusted = 0; + }; + + /// #856 — foot-contact cleanup. Retargeted clips slide/float feet on rigs + /// whose proportions differ from the source (the direction retarget + /// transfers bone DIRECTIONS, not world foot positions). Per foot role, + /// detect contact spans (foot near the clip's ground level AND nearly + /// stationary horizontally — FootContact::detectContacts, canonical-frame, + /// leg-length-scaled thresholds) and lock the foot's world position to its + /// span-start position with an analytic two-bone hip–knee–foot IK + /// (FootContact::solveKnee — keeps segment lengths and the pose's own + /// bend plane), blending in/out over `blendFrames` at span edges so knees + /// don't pop. Rewrites ONLY the thigh/shin/foot keyframes (foot keeps its + /// original world orientation); everything else untouched. Pure track + /// math — nothing is applied to the live skeleton. + /// + /// Effectively idempotent: a second run detects the already-planted spans + /// and re-pins to the same targets (near-no-op). The application is + /// recorded on bone[0]'s UserObjectBindings ("qtme.footpin.") so a + /// UI can reflect state; applyMotionClip clears it on clip regeneration. + /// Designed for generated clips (dense uniform keyframes); sparse + /// authored clips get keyframe-rate detection (approximate). + static FootPinResult pinFeet(Ogre::Skeleton* skel, + const std::string& animName, + int blendFrames = 3); + /// Sample every (or one) skeletal animation of `entity` at `fps` and /// express each canonical joint's world orientation per frame. Bone→role /// mapping is MotionInbetween::canonicalIndexForBone — the same matcher diff --git a/src/CLIPipeline.cpp b/src/CLIPipeline.cpp index 4344a978..338d31e5 100644 --- a/src/CLIPipeline.cpp +++ b/src/CLIPipeline.cpp @@ -2017,7 +2017,7 @@ int CLIPipeline::cmdFix(int argc, char* argv[]) int CLIPipeline::cmdAnimGenerate(const QString& filePath, const QString& prompt, float duration, const QString& outputPath, bool jsonOutput, bool useModel, - float armSpaceDeg) + float armSpaceDeg, bool footPin) { // #411 text-to-motion (template-clip MVP): match the prompt to a permissive // CMU motion clip from the downloadable library, retarget it onto the mesh's @@ -2147,6 +2147,19 @@ int CLIPipeline::cmdAnimGenerate(const QString& filePath, const QString& prompt, << Qt::endl; } + // #856: foot-contact cleanup — ON by default (--no-foot-pin disables). + if (footPin) { + const auto fp = AnimationMerger::pinFeet(skel.get(), animName); + if (fp.ok && fp.spans > 0) { + SentryReporter::addBreadcrumb( + QStringLiteral("ai.tool_call"), + QStringLiteral("foot_pin %1 spans").arg(fp.spans)); + err() << "(foot-pin: " << fp.spans << " contact span(s), " + << fp.keyframesAdjusted << " keyframes)" << Qt::endl; + } else if (!fp.ok) + err() << "Note: foot-pin skipped (" << fp.error << ")." << Qt::endl; + } + auto* node = entity->getParentSceneNode(); const QString fmt = formatForExtension(outputPath); if (MeshImporterExporter::exporter(node, QFileInfo(outputPath).absoluteFilePath(), fmt) != 0) { @@ -2200,6 +2213,8 @@ int CLIPipeline::cmdAnim(int argc, char* argv[]) bool generateUseModel = false; // --model → experimental trained t2m model (template fallback) float armSpaceDeg = 0.0f; // #854: Mixamo-style arm-space swing (degrees) bool armSpaceSet = false; // --arm-space given (standalone post-adjust) + bool generateFootPin = true; // #856: pin feet after --generate (default ON) + bool footPinSet = false; // --foot-pin given (standalone post-process) bool jsonOutput = false; int resampleCount = 0; int decimateStep = 0; @@ -2293,6 +2308,11 @@ int CLIPipeline::cmdAnim(int argc, char* argv[]) armSpaceSet = true; continue; } + // #856 foot-contact cleanup: ON by default during --generate + // (--no-foot-pin opts out); --foot-pin alone post-processes an + // EXISTING animation (standalone, needs --animation). + if (arg == "--no-foot-pin") { generateFootPin = false; continue; } + if (arg == "--foot-pin") { footPinSet = true; continue; } if (arg == "--simplify") { simplifyMode = true; continue; } if (arg == "--analyze") { analyzeMode = true; continue; } if (arg == "--preset" && i + 1 < argc) { @@ -2388,7 +2408,7 @@ int CLIPipeline::cmdAnim(int argc, char* argv[]) } return cmdAnimGenerate(filePath, generatePrompt, generateDuration, outputPath.isEmpty() ? filePath : outputPath, jsonOutput, - generateUseModel, armSpaceDeg); + generateUseModel, armSpaceDeg, generateFootPin); } // #854 standalone: post-adjust the arm space of an EXISTING animation @@ -2434,6 +2454,45 @@ int CLIPipeline::cmdAnim(int argc, char* argv[]) return 0; } + // #856 standalone: pin the feet of an EXISTING animation (no --generate). + // `qtmesh anim --foot-pin --animation -o out`. + if (footPinSet) { + if (animationFilter.isEmpty()) { + err() << "Error: --foot-pin (standalone) requires --animation ." + << Qt::endl; + return 2; + } + if (!initOgreHeadless()) return 1; + MeshImporterExporter::importer({QFileInfo(filePath).absoluteFilePath()}); + Ogre::Entity* entity = nullptr; + for (auto* e : Manager::getSingleton()->getEntities()) + if (e && e->getMovableType() == "Entity" && e->hasSkeleton()) { entity = e; break; } + if (!entity) { + err() << "Error: " << filePath << " has no skinned mesh." << Qt::endl; + return 1; + } + Ogre::SkeletonPtr skel = entity->getMesh()->getSkeleton(); + const auto fp = AnimationMerger::pinFeet( + skel.get(), animationFilter.toStdString()); + if (!fp.ok) { + err() << "Error: foot-pin failed: " << fp.error << Qt::endl; + return 1; + } + SentryReporter::addBreadcrumb(QStringLiteral("ai.tool_call"), + QStringLiteral("foot_pin %1 spans").arg(fp.spans)); + const QString out = outputPath.isEmpty() ? filePath : outputPath; + auto* node = entity->getParentSceneNode(); + if (MeshImporterExporter::exporter(node, QFileInfo(out).absoluteFilePath(), + formatForExtension(out)) != 0) { + err() << "Error: export failed." << Qt::endl; return 1; + } + cliWrite(QString("Pinned feet of '%1' (%2 span(s), %3 keyframes) → %4\n") + .arg(animationFilter).arg(fp.spans) + .arg(fp.keyframesAdjusted) + .arg(QFileInfo(out).fileName())); + return 0; + } + if (!listMode && !renameMode && !mergeMode && !resampleMode && !decimateMode && !simplifyMode && !analyzeMode && !bakeFpsMode && !inbetweenMode && !dumpCanonicalMode) { diff --git a/src/CLIPipeline.h b/src/CLIPipeline.h index 775d501f..22391f2f 100644 --- a/src/CLIPipeline.h +++ b/src/CLIPipeline.h @@ -93,7 +93,7 @@ class CLIPipeline { static int cmdAnimGenerate(const QString& filePath, const QString& prompt, float duration, const QString& outputPath, bool jsonOutput, bool useModel = false, - float armSpaceDeg = 0.0f); + float armSpaceDeg = 0.0f, bool footPin = true); static int cmdValidate(int argc, char* argv[]); static int cmdLod(int argc, char* argv[]); static int cmdPose(int argc, char* argv[]); diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index a06f61f8..ebb42e15 100755 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -65,6 +65,7 @@ SentryReporter.cpp BoneWeightOverlay.cpp NormalVisualizer.cpp AnimationMerger.cpp +FootContact.cpp CLIPipeline.cpp ConsoleLogSanitize.cpp MeshInfoOverlay.cpp @@ -280,6 +281,7 @@ SentryReporter.h BoneWeightOverlay.h NormalVisualizer.h AnimationMerger.h +FootContact.h CLIPipeline.h MeshInfoOverlay.h RTShaderHelper.h diff --git a/src/FootContact.cpp b/src/FootContact.cpp new file mode 100644 index 00000000..a3a2ffbb --- /dev/null +++ b/src/FootContact.cpp @@ -0,0 +1,115 @@ +#include "FootContact.h" + +#include +#include + +namespace FootContact { + +namespace { + +V3 sub(const V3& a, const V3& b) { return {a[0] - b[0], a[1] - b[1], a[2] - b[2]}; } +V3 add(const V3& a, const V3& b) { return {a[0] + b[0], a[1] + b[1], a[2] + b[2]}; } +V3 mul(const V3& a, float s) { return {a[0] * s, a[1] * s, a[2] * s}; } +float dot(const V3& a, const V3& b) { return a[0]*b[0] + a[1]*b[1] + a[2]*b[2]; } +V3 cross(const V3& a, const V3& b) +{ + return {a[1]*b[2] - a[2]*b[1], a[2]*b[0] - a[0]*b[2], a[0]*b[1] - a[1]*b[0]}; +} +float len(const V3& a) { return std::sqrt(dot(a, a)); } +V3 normed(const V3& a) +{ + const float l = len(a); + return l > 1e-9f ? mul(a, 1.0f / l) : V3{0, 0, 0}; +} + +} // namespace + +float groundLevel(const std::vector& foot, int upAxis) +{ + if (foot.empty()) return 0.0f; + std::vector h(foot.size()); + for (size_t i = 0; i < foot.size(); ++i) h[i] = foot[i][upAxis]; + const size_t k = std::min(foot.size() - 1, + static_cast(foot.size() / 10)); + std::nth_element(h.begin(), h.begin() + static_cast(k), h.end()); + return h[k]; +} + +std::vector detectContacts(const std::vector& foot, + float legLength, + const DetectOptions& opt) +{ + std::vector spans; + const int n = static_cast(foot.size()); + if (n < opt.minFrames || legLength <= 1e-6f) return spans; + const int up = std::clamp(opt.upAxis, 0, 2); + const float ground = groundLevel(foot, up); + const float band = ground + opt.heightBandFrac * legLength; + const float vmax = opt.velThreshFrac * legLength; + + auto horizSpeed = [&](int f) { + // central difference where possible; edges take the one-sided step + const int a = std::max(0, f - 1); + const int b = std::min(n - 1, f + 1); + V3 d = sub(foot[b], foot[a]); + d[up] = 0.0f; + return len(d) / static_cast(std::max(1, b - a)); + }; + + int start = -1; + for (int f = 0; f <= n; ++f) { + const bool contact = f < n && foot[f][up] <= band && horizSpeed(f) <= vmax; + if (contact && start < 0) start = f; + if (!contact && start >= 0) { + if (f - start >= opt.minFrames) spans.push_back({start, f - 1}); + start = -1; + } + } + return spans; +} + +V3 solveKnee(const V3& hip, const V3& knee, const V3& foot, const V3& target) +{ + const float l1 = len(sub(knee, hip)); + const float l2 = len(sub(foot, knee)); + if (l1 <= 1e-6f || l2 <= 1e-6f) return knee; + + V3 toT = sub(target, hip); + float d = len(toT); + const float dMin = std::abs(l1 - l2) + 1e-4f; + const float dMax = l1 + l2 - 1e-4f; + if (d < 1e-6f) return knee; // target on the hip: give up + const V3 dirT = mul(toT, 1.0f / d); + d = std::clamp(d, dMin, dMax); + + // Bend direction: the current knee's offset from the hip→foot line keeps + // the pose's own bend plane (the knee axis of the existing pose). + V3 bend = sub(knee, hip); + bend = sub(bend, mul(dirT, dot(bend, dirT))); + if (dot(bend, bend) < 1e-10f) { + // straight leg — bend forward of the current chain plane, or any + // perpendicular when even that is degenerate + bend = cross(dirT, cross(sub(foot, hip), sub(knee, hip))); + if (dot(bend, bend) < 1e-10f) + bend = cross(dirT, std::abs(dirT[1]) < 0.9f ? V3{0, 1, 0} + : V3{1, 0, 0}); + } + bend = normed(bend); + + // law of cosines: knee at distance l1 from hip, l2 from target + const float cosA = std::clamp((l1 * l1 + d * d - l2 * l2) + / (2.0f * l1 * d), -1.0f, 1.0f); + const float sinA = std::sqrt(std::max(0.0f, 1.0f - cosA * cosA)); + return add(hip, add(mul(dirT, l1 * cosA), mul(bend, l1 * sinA))); +} + +float blendWeight(const Span& s, int f, int blend) +{ + if (f < s.start || f > s.end) return 0.0f; + if (blend <= 0) return 1.0f; + const float in = static_cast(f - s.start + 1) / blend; + const float out = static_cast(s.end - f + 1) / blend; + return std::clamp(std::min(in, out), 0.0f, 1.0f); +} + +} // namespace FootContact diff --git a/src/FootContact.h b/src/FootContact.h new file mode 100644 index 00000000..ead2e52b --- /dev/null +++ b/src/FootContact.h @@ -0,0 +1,65 @@ +#ifndef FOOT_CONTACT_H +#define FOOT_CONTACT_H + +#include +#include + +// #856 — foot-contact cleanup, pure-data core (Ogre-free, unit-tested). +// +// Retargeted clips slide/float feet on rigs whose proportions differ from the +// source: the bind-referenced direction retarget (PR #843) transfers bone +// DIRECTIONS, not world foot positions. This core provides the two pieces the +// track post-process (AnimationMerger::pinFeet) composes: +// 1. contact detection — frames where a foot is near the clip's ground +// level AND nearly stationary horizontally → contact spans, +// 2. an analytic two-bone IK step — where must the knee sit so the foot +// reaches its pinned position, preserving the pose's own bend plane, +// plus the smooth edge-blend weight that avoids knee pops at span borders. +namespace FootContact { + +using V3 = std::array; + +/// Inclusive frame range during which a foot is planted. +struct Span { + int start = 0; + int end = 0; +}; + +struct DetectOptions { + /// Height band above the detected ground level counting as "on the + /// ground", as a fraction of leg length. + float heightBandFrac = 0.18f; + /// Max horizontal speed (units/frame) counting as "stationary", as a + /// fraction of leg length. + float velThreshFrac = 0.03f; + /// Spans shorter than this are noise, not contacts. + int minFrames = 3; + /// Up axis: 0=X, 1=Y, 2=Z (canonical rigs are +Y-up). + int upAxis = 1; +}; + +/// Ground level = low percentile of the foot's height trajectory (robust to +/// a clip that never plants, e.g. a jump apex-only window). +float groundLevel(const std::vector& foot, int upAxis); + +/// Contact spans of one foot trajectory. `legLength` scales both thresholds +/// so detection is rig-size-independent. +std::vector detectContacts(const std::vector& foot, + float legLength, + const DetectOptions& opt = {}); + +/// Analytic two-bone IK: given the CURRENT chain (hip → knee → foot) and a +/// new foot `target`, return the knee position that keeps both segment +/// lengths and stays in the pose's own bend plane (pole derived from the +/// current knee). Targets beyond reach are clamped along hip→target; +/// degenerate poses (straight leg) bend toward the current knee offset or, +/// failing that, any perpendicular. +V3 solveKnee(const V3& hip, const V3& knee, const V3& foot, const V3& target); + +/// Smooth 0→1→0 weight for frame `f` inside span `s`, ramping linearly over +/// `blend` frames at each edge (1 in the interior, 0 outside). +float blendWeight(const Span& s, int f, int blend); + +} // namespace FootContact + +#endif // FOOT_CONTACT_H diff --git a/src/FootContact_test.cpp b/src/FootContact_test.cpp new file mode 100644 index 00000000..0582e1b5 --- /dev/null +++ b/src/FootContact_test.cpp @@ -0,0 +1,124 @@ +#include + +#include + +#include "FootContact.h" + +// #856 pure-data core tests — no Ogre/GL needed. + +using FootContact::V3; + +namespace { +float dist(const V3& a, const V3& b) +{ + const float dx = a[0] - b[0], dy = a[1] - b[1], dz = a[2] - b[2]; + return std::sqrt(dx * dx + dy * dy + dz * dz); +} +} // namespace + +TEST(FootContactTest, DetectsPlantAndSwingPhases) +{ + // Gait-like trajectory: planted at x=0 for 10 frames, swing forward over + // 10 frames (lifted), planted at x=1 for 10 frames. + std::vector foot; + for (int f = 0; f < 10; ++f) foot.push_back({0.0f, 0.0f, 0.0f}); + for (int f = 0; f < 10; ++f) { + const float t = (f + 1) / 11.0f; + foot.push_back({t, 0.25f * std::sin(t * 3.14159f), 0.0f}); + } + for (int f = 0; f < 10; ++f) foot.push_back({1.0f, 0.0f, 0.0f}); + + const auto spans = FootContact::detectContacts(foot, /*legLength=*/1.0f); + ASSERT_EQ(spans.size(), 2u); + EXPECT_EQ(spans[0].start, 0); + EXPECT_GE(spans[0].end, 7); // plant 1 covers most of frames 0..9 + EXPECT_LE(spans[0].end, 10); + EXPECT_GE(spans[1].start, 19); // plant 2 starts after the swing + EXPECT_EQ(spans[1].end, 29); +} + +TEST(FootContactTest, SlidingFootIsNotAContact) +{ + // On the ground the whole time but translating fast — foot skate, the + // exact artifact we're pinning. Must NOT be detected as one long contact. + std::vector foot; + for (int f = 0; f < 30; ++f) + foot.push_back({0.1f * f, 0.0f, 0.0f}); // 0.1 leg-lengths/frame + const auto spans = FootContact::detectContacts(foot, 1.0f); + EXPECT_TRUE(spans.empty()); +} + +TEST(FootContactTest, ShortBlipsAreIgnored) +{ + std::vector foot; + for (int f = 0; f < 20; ++f) + foot.push_back({0.0f, (f == 10 || f == 11) ? 0.0f : 0.6f, 0.0f}); + FootContact::DetectOptions opt; + opt.minFrames = 3; + const auto spans = FootContact::detectContacts(foot, 1.0f, opt); + EXPECT_TRUE(spans.empty()); // 2-frame touch < minFrames +} + +TEST(FootContactTest, SolveKneePreservesLengthsAndReachesTarget) +{ + const V3 hip{0, 1.0f, 0}; + const V3 knee{0.1f, 0.5f, 0.15f}; // bent slightly forward + const V3 foot{0, 0.0f, 0.05f}; + const float l1 = dist(hip, knee), l2 = dist(knee, foot); + + const V3 target{0.05f, 0.02f, -0.2f}; // pin slightly behind + const V3 k2 = FootContact::solveKnee(hip, knee, foot, target); + EXPECT_NEAR(dist(hip, k2), l1, 1e-4f); + EXPECT_NEAR(dist(k2, target), l2, 1e-4f); + // knee keeps bending roughly forward (pose's own bend plane, no flip) + EXPECT_GT(k2[2], -0.05f); +} + +TEST(FootContactTest, SolveKneeClampsUnreachableTarget) +{ + const V3 hip{0, 1.0f, 0}; + const V3 knee{0, 0.5f, 0.1f}; + const V3 foot{0, 0.0f, 0}; + const float l1 = dist(hip, knee), l2 = dist(knee, foot); + + const V3 far{3.0f, -2.0f, 0}; // beyond l1+l2 + const V3 k2 = FootContact::solveKnee(hip, knee, foot, far); + EXPECT_NEAR(dist(hip, k2), l1, 1e-3f); + // the chain extends straight toward the target: knee sits on hip→far + const float reach = dist(hip, k2) + l2; + EXPECT_NEAR(reach, l1 + l2, 1e-3f); +} + +TEST(FootContactTest, SolveKneeStraightLegPicksStableBend) +{ + const V3 hip{0, 1.0f, 0}; + const V3 knee{0, 0.5f, 0}; // perfectly straight leg + const V3 foot{0, 0.0f, 0}; + const V3 target{0, 0.2f, 0}; // shorten: must bend somewhere + const V3 k2 = FootContact::solveKnee(hip, knee, foot, target); + EXPECT_NEAR(dist(hip, k2), 0.5f, 1e-4f); + EXPECT_NEAR(dist(k2, target), 0.5f, 1e-4f); +} + +TEST(FootContactTest, BlendWeightRampsAtSpanEdges) +{ + const FootContact::Span s{10, 20}; + EXPECT_FLOAT_EQ(FootContact::blendWeight(s, 9, 3), 0.0f); + EXPECT_FLOAT_EQ(FootContact::blendWeight(s, 21, 3), 0.0f); + EXPECT_NEAR(FootContact::blendWeight(s, 10, 3), 1.0f / 3.0f, 1e-5f); + EXPECT_NEAR(FootContact::blendWeight(s, 11, 3), 2.0f / 3.0f, 1e-5f); + EXPECT_FLOAT_EQ(FootContact::blendWeight(s, 15, 3), 1.0f); + EXPECT_NEAR(FootContact::blendWeight(s, 20, 3), 1.0f / 3.0f, 1e-5f); + // zero blend = hard pin + EXPECT_FLOAT_EQ(FootContact::blendWeight(s, 10, 0), 1.0f); +} + +TEST(FootContactTest, GroundLevelIsRobustToAirTime) +{ + // Foot spends most of the clip in the air (jump) — ground level must + // still track the low plateau, not the mean. + std::vector foot; + for (int f = 0; f < 40; ++f) + foot.push_back({0.0f, (f < 6) ? 0.02f : 0.8f, 0.0f}); + EXPECT_NEAR(FootContact::groundLevel(foot, 1), 0.02f, 1e-4f); +} diff --git a/src/MCPServer.cpp b/src/MCPServer.cpp index d4fccd68..b1e6d04c 100644 --- a/src/MCPServer.cpp +++ b/src/MCPServer.cpp @@ -651,6 +651,7 @@ const QMap& MCPServer::toolHandlers() {QStringLiteral("motion_in_between"), &MCPServer::toolMotionInBetween}, {QStringLiteral("generate_motion"), &MCPServer::toolGenerateMotion}, {QStringLiteral("adjust_arm_space"), &MCPServer::toolAdjustArmSpace}, + {QStringLiteral("pin_feet"), &MCPServer::toolPinFeet}, {QStringLiteral("segment_mesh"), &MCPServer::toolSegmentMesh}, {QStringLiteral("generate_mesh_from_image"), &MCPServer::toolGenerateMeshFromImage}, {QStringLiteral("save_scene"), &MCPServer::toolSaveScene}, @@ -4132,6 +4133,13 @@ QJsonObject MCPServer::toolGenerateMotion(const QJsonObject &args) armSpaceApplied = AnimationMerger::adjustArmSpace( skel.get(), animName, static_cast(armSpace)); + // #856: foot-contact cleanup — ON by default (foot_pin:false opts out). + int footPinSpans = -1; + if (args.value("foot_pin").toBool(true)) { + const auto fp = AnimationMerger::pinFeet(skel.get(), animName); + footPinSpans = fp.ok ? fp.spans : -1; + } + entity->refreshAvailableAnimationState(); // Exclusively enable the generated clip — enabled states BLEND in // Ogre, and mixing with the import's auto-enabled animation renders @@ -4167,6 +4175,7 @@ QJsonObject MCPServer::toolGenerateMotion(const QJsonObject &args) content["tracks_written"] = r.tracksWritten; content["canonical_joints"] = r.canonicalJoints; content["entity"] = QString::fromStdString(entity->getName()); if (std::abs(armSpace) > 1e-4) content["arm_space_applied"] = armSpaceApplied; + if (footPinSpans >= 0) content["foot_pin_spans"] = footPinSpans; if (!outPath.isEmpty()) content["exported"] = outPath; return makeSuccessResult( QString::fromUtf8(QJsonDocument(content).toJson(QJsonDocument::Indented))); @@ -4252,6 +4261,76 @@ QJsonObject MCPServer::toolAdjustArmSpace(const QJsonObject &args) } } +QJsonObject MCPServer::toolPinFeet(const QJsonObject &args) +{ + try { + SentryReporter::addBreadcrumb(QStringLiteral("ai.tool_call"), + QStringLiteral("MCP pin_feet")); + + Manager* mgr = Manager::getSingletonPtr(); + if (!mgr) return makeErrorResult("Error: Manager not available"); + + const QString animName = args.value("animation_name").toString(); + if (animName.isEmpty()) + return makeErrorResult("Error: animation_name is required."); + + const QString entityName = args.value("entity_name").toString(); + Ogre::Entity* entity = nullptr; + for (auto* ent : mgr->getEntities()) { + if (!ent || ent->getMovableType() != "Entity" || !ent->hasSkeleton()) + continue; + if (entityName.isEmpty() + || QString::fromStdString(ent->getName()) == entityName) { + entity = ent; break; + } + } + if (!entity) + return makeErrorResult(entityName.isEmpty() + ? QString("Error: no skinned mesh found.") + : QString("Error: skinned entity '%1' not found.").arg(entityName)); + + // Edit the mesh's MASTER skeleton (exporter serializes the master; + // animations are shared, so the live viewport still updates). + Ogre::SkeletonPtr skel = entity->getMesh()->getSkeleton(); + const std::string an = animName.toStdString(); + if (!skel || !skel->hasAnimation(an)) + return makeErrorResult( + QString("Error: animation '%1' not found on entity.").arg(animName)); + + const auto fp = AnimationMerger::pinFeet(skel.get(), an); + if (!fp.ok) + return makeErrorResult( + QString("Error: foot-pin failed: %1").arg(fp.error)); + + if (auto* acc = AnimationControlController::instance()) + acc->notifyExternalAnimationEdit(); + + const QString outPath = args.value("output_path").toString(); + if (!outPath.isEmpty()) { + auto* node = entity->getParentSceneNode(); + if (MeshImporterExporter::exporter( + node, outPath, CLIPipeline::formatForExtension(outPath)) != 0) + return makeErrorResult( + QString("Error: pinned feet but export to %1 failed") + .arg(outPath)); + } + + QJsonObject content; + content["ok"] = true; + content["animation"] = animName; + content["spans"] = fp.spans; + content["keyframes_adjusted"] = fp.keyframesAdjusted; + content["entity"] = QString::fromStdString(entity->getName()); + if (!outPath.isEmpty()) content["exported"] = outPath; + return makeSuccessResult( + QString::fromUtf8(QJsonDocument(content).toJson(QJsonDocument::Indented))); + } catch (Ogre::Exception& e) { + return makeErrorResult(QString("Error: Ogre exception — %1").arg(e.getFullDescription().c_str())); + } catch (std::exception& e) { + return makeErrorResult(QString("Error: %1").arg(e.what())); + } +} + QJsonObject MCPServer::toolSegmentMesh(const QJsonObject &args) { try { @@ -8134,6 +8213,7 @@ QJsonArray MCPServer::buildToolsList() props["output_path"] = QJsonObject{{"type", "string"}, {"description", "Optional path to re-export the mesh with the new animation (e.g. /tmp/out.glb). If omitted, the animation is applied in-session only."}}; props["model"] = QJsonObject{{"type", "boolean"}, {"description", "EXPERIMENTAL: use the trained from-scratch text-to-motion ONNX model instead of the template clip. Falls back to the template library automatically if the model is unavailable or the action isn't in its vocabulary. Default false (template). Quality is action-dependent (locomotion better than gestures)."}}; props["arm_space"] = QJsonObject{{"type", "number"}, {"description", "Optional Mixamo-style arm-space post-process in degrees (#854): positive widens the arms away from the body, negative tucks them in. Default 0. Rescues arm-into-torso clipping on rigs whose proportions differ from the clip."}}; + props["foot_pin"] = QJsonObject{{"type", "boolean"}, {"description", "Foot-contact cleanup (#856): detect ground-contact spans and IK-pin the feet so they plant instead of skating. Default true; set false to keep the raw retarget."}}; appendTool( "generate_motion", "AI text-to-motion (#411, experimental): generate a skeletal animation from a text prompt and " @@ -8166,6 +8246,24 @@ QJsonArray MCPServer::buildToolsList() ); } + // pin_feet + { + QJsonObject props; + props["animation_name"] = QJsonObject{{"type", "string"}, {"description", "Name of the animation to clean up, e.g. \"generated_walk\"."}}; + props["entity_name"] = QJsonObject{{"type", "string"}, {"description", "Name of the rigged entity. If omitted, uses the first skinned entity."}}; + props["output_path"] = QJsonObject{{"type", "string"}, {"description", "Optional path to re-export the mesh with the cleaned animation. If omitted, applied in-session only."}}; + appendTool( + "pin_feet", + "Foot-contact cleanup (#856): detect frames where each foot is near the clip's ground level and " + "nearly stationary (contact spans) and lock the foot's world position to its span-start position " + "with an analytic two-bone hip-knee-foot IK, blending in/out at span edges so knees don't pop. " + "Fixes foot skating/floating on retargeted clips whose rig proportions differ from the source. " + "Only the thigh/shin/foot keyframes are rewritten; effectively idempotent.", + props, + QJsonArray{"animation_name"} + ); + } + // segment_mesh { QJsonObject props; diff --git a/src/MCPServer.h b/src/MCPServer.h index 1555d16f..3d2254fa 100644 --- a/src/MCPServer.h +++ b/src/MCPServer.h @@ -210,6 +210,7 @@ private slots: QJsonObject toolMotionInBetween(const QJsonObject &args); QJsonObject toolGenerateMotion(const QJsonObject &args); // #411 text-to-motion QJsonObject toolAdjustArmSpace(const QJsonObject &args); // #854 arm-space + QJsonObject toolPinFeet(const QJsonObject &args); // #856 foot-contact pin QJsonObject toolSegmentMesh(const QJsonObject &args); QJsonObject toolGenerateMeshFromImage(const QJsonObject &args); // #764 image-to-3D QJsonObject toolSaveScene(const QJsonObject &args); diff --git a/src/MotionInbetween.cpp b/src/MotionInbetween.cpp index e612adcc..ecf23d34 100644 --- a/src/MotionInbetween.cpp +++ b/src/MotionInbetween.cpp @@ -153,7 +153,7 @@ int MotionInbetween::canonicalIndexForBone(const QString& boneName) if (has({"elbow", "forearm", "lowerarm"})) return 8; // relbow if (has({"hand", "wrist"})) return 9; // rhand if (has({"buttock"})) return 14; // rbuttock - if (has({"upleg", "thigh", "hip", "femur"})) return 15; // rhip + if (has({"upleg", "upperleg", "thigh", "hip", "femur"})) return 15; // rhip if (has({"knee", "leg", "shin", "calf"}) && !has({"upleg","thigh"})) return 16; // rknee if (has({"foot", "ankle"})) return 17; // rfoot } else if (side == 'l') { @@ -163,7 +163,7 @@ int MotionInbetween::canonicalIndexForBone(const QString& boneName) if (has({"elbow", "forearm", "lowerarm"})) return 12; // lelbow if (has({"hand", "wrist"})) return 13; // lhand if (has({"buttock"})) return 18; // lbuttock - if (has({"upleg", "thigh", "hip", "femur"})) return 19; // lhip + if (has({"upleg", "upperleg", "thigh", "hip", "femur"})) return 19; // lhip if (has({"knee", "leg", "shin", "calf"}) && !has({"upleg","thigh"})) return 20; // lknee if (has({"foot", "ankle"})) return 21; // lfoot } diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 85090bc6..6e64b609 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -75,6 +75,7 @@ if(BUILD_TESTS) ${CMAKE_CURRENT_SOURCE_DIR}/../src/BoneWeightOverlay.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../src/NormalVisualizer.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../src/AnimationMerger.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../src/FootContact.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../src/MCPServer.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../src/MCPSettingsDialog.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../src/MeshInfoOverlay.cpp From 4c2589b033171bd12eba0a26896e61f10d90f98b Mon Sep 17 00:00:00 2001 From: Fernando Date: Mon, 13 Jul 2026 09:16:28 -0300 Subject: [PATCH 04/23] chore(scripts): HF upload script for the v5 t2m flow model (#858) Co-Authored-By: Claude Fable 5 --- scripts/upload-t2m-v5-model.sh | 49 ++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100755 scripts/upload-t2m-v5-model.sh diff --git a/scripts/upload-t2m-v5-model.sh b/scripts/upload-t2m-v5-model.sh new file mode 100755 index 00000000..c199cf58 --- /dev/null +++ b/scripts/upload-t2m-v5-model.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +# Upload the v5 flow-matching text-to-motion model to the QtMeshEditor HF +# models repo (#840/#858, epic #837). ONE-TIME, run by a maintainer with +# write access. +# +# The app downloads these on first use from +# https://huggingface.co/fernandotonon/QtMeshEditor-models/resolve/main/motion/t2m.onnx +# https://huggingface.co/fernandotonon/QtMeshEditor-models/resolve/main/motion/t2m-vocab.json +# The v5 ONNX keeps the exact v4 runtime interface (tokens[1,V], seed[1,Z] +# -> motion[1,T,220]) so replacing the files is backward-compatible: older +# builds run the new model too (they ignore the vocab's restWorld/restDir +# and fall back to the synthetic-standing-pose path). The previous v4 files +# are preserved under versioned names for rollback. +# +# Prereqs: +# pip install -U "huggingface_hub[cli]" +# huggingface-cli login # token with write access +# train-t2m-flow-v5.py already run -> OUT_DIR holds t2m.onnx + t2m-vocab.json +# +# Usage: +# OUT_DIR=/tmp/t2m_v5_flow ./scripts/upload-t2m-v5-model.sh +set -euo pipefail + +REPO="${REPO:-fernandotonon/QtMeshEditor-models}" +OUT_DIR="${OUT_DIR:?set OUT_DIR to the training output dir (t2m.onnx + t2m-vocab.json)}" + +for f in t2m.onnx t2m-vocab.json; do + [[ -f "$OUT_DIR/$f" ]] || { echo "missing $OUT_DIR/$f" >&2; exit 1; } +done + +# keep the previous (v4 CVAE) files for rollback under versioned names +TMP=$(mktemp -d) +for f in t2m.onnx t2m-vocab.json; do + if huggingface-cli download "$REPO" "motion/$f" --local-dir "$TMP" >/dev/null 2>&1; then + v4name="${f%.onnx}"; v4name="${v4name%.json}" + case "$f" in + t2m.onnx) dst="motion/t2m-v4.onnx" ;; + t2m-vocab.json) dst="motion/t2m-vocab-v4.json" ;; + esac + huggingface-cli upload "$REPO" "$TMP/motion/$f" "$dst" \ + --commit-message "t2m: preserve v4 CVAE as $dst before the v5 flow model (#858)" + fi +done + +huggingface-cli upload "$REPO" "$OUT_DIR/t2m.onnx" motion/t2m.onnx \ + --commit-message "t2m v5: flow-matching model, canonical reference triple (#840/#858)" +huggingface-cli upload "$REPO" "$OUT_DIR/t2m-vocab.json" motion/t2m-vocab.json \ + --commit-message "t2m v5: vocab + canonical reference triple (#840/#858)" +echo "uploaded v5 t2m model + vocab to $REPO/motion/" From 843a0d56fa620f033bffa032c581b9cb997733df Mon Sep 17 00:00:00 2001 From: Fernando Date: Mon, 13 Jul 2026 10:32:00 -0300 Subject: [PATCH 05/23] =?UTF-8?q?test(anim):=20fix=20twist=20tests=20?= =?UTF-8?q?=E2=80=94=20humanoid-gate=20rig=20+=20cap-before-gain=20expecta?= =?UTF-8?q?tion=20(#857)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The arm-rig helper resolves only 9/22 roles, below applyMotionClip's >=11 humanoid gate — the roll test now builds a 13-role rig (+hands, +feet). And the 150-degree runaway-unwrap cap applies BEFORE the collar gain, so a 240-degree source roll lands at 150 x 0.5 = 75 degrees, not 120; the continuity assertion (the actual unwrap regression check) is unchanged. Co-Authored-By: Claude Fable 5 --- src/AnimationMerger_test.cpp | 40 ++++++++++++++++++++++++++++++++---- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/src/AnimationMerger_test.cpp b/src/AnimationMerger_test.cpp index 07e55bf9..b451f1e3 100644 --- a/src/AnimationMerger_test.cpp +++ b/src/AnimationMerger_test.cpp @@ -1187,7 +1187,35 @@ float twistDegAbout(const Ogre::Quaternion& q, const Ogre::Vector3& ax) TEST_F(AnimationMergerTest, TwistTransportCarriesBoneRoll) { - Ogre::Entity* ent = makeArmRigEntity("twist_roll"); + // makeArmRigEntity resolves only 9/22 roles — below applyMotionClip's + // humanoid gate (>= 11) — so build a fuller rig (13 roles: + hands/feet). + auto skelRes = Ogre::SkeletonManager::getSingleton().create( + "twist_roll_skel", + Ogre::ResourceGroupManager::DEFAULT_RESOURCE_GROUP_NAME); + unsigned short h = 0; + auto bone = [&](const char* n, const Ogre::Vector3& p, Ogre::Bone* par) { + auto* b = skelRes->createBone(n, h++); + b->setPosition(p); + if (par) par->addChild(b); + return b; + }; + auto* hips = bone("Hips", {0, 1.0f, 0}, nullptr); + auto* spine = bone("Spine", {0, 0.3f, 0}, hips); + bone("Head", {0, 0.4f, 0}, spine); + auto* lleg = bone("LeftUpLeg", {0.15f, -0.1f, 0}, hips); + bone("LeftFoot", {0, -0.8f, 0}, lleg); + auto* rleg = bone("RightUpLeg", {-0.15f, -0.1f, 0}, hips); + bone("RightFoot", {0, -0.8f, 0}, rleg); + auto* rsh = bone("RightArm", {-0.2f, 0.1f, 0}, spine); + auto* rfa = bone("RightForeArm", {-0.3f, 0, 0}, rsh); + bone("RightHand", {-0.25f, 0, 0}, rfa); + auto* lsh = bone("LeftArm", {0.2f, 0.1f, 0}, spine); + auto* lfa = bone("LeftForeArm", {0.3f, 0, 0}, lsh); + bone("LeftHand", {0.25f, 0, 0}, lfa); + skelRes->setBindingPose(); + auto mesh = createInMemoryMesh("twist_roll_mesh", skelRes); + auto* sm = Manager::getSingleton()->getSceneMgr(); + Ogre::Entity* ent = sm->createEntity("twist_roll_ent", mesh); ASSERT_NE(ent, nullptr); Ogre::SkeletonInstance* skel = ent->getSkeleton(); @@ -1228,6 +1256,8 @@ TEST_F(AnimationMergerTest, TwistTransportCarriesBoneRoll) const Ogre::Quaternion leg = skel->getBone("LeftUpLeg")->_getDerivedOrientation(); EXPECT_NEAR(std::abs(leg.w), 1.0f, 1e-3f); + + sm->destroyEntity(ent); } TEST_F(AnimationMergerTest, TwistUnwrapKeepsDampedCollarContinuous) @@ -1262,8 +1292,8 @@ TEST_F(AnimationMergerTest, TwistUnwrapKeepsDampedCollarContinuous) // Source: left collar (role 10, +X) rolls 0 → 240° — past the ±180° wrap. // The gain table damps collars to 0.5×, which is exactly where a missing - // unwrap explodes: wrapped −120° would scale to −60° instead of +120°'s - // half — a mid-clip snap. + // unwrap explodes: wrapped −120° would scale to −60° instead of the + // capped +150°'s half — a mid-clip snap. const int frames = 61; auto quats = identityClip(frames); for (int f = 0; f < frames; ++f) { @@ -1301,7 +1331,9 @@ TEST_F(AnimationMergerTest, TwistUnwrapKeepsDampedCollarContinuous) last = w; } EXPECT_LT(maxStepDeg, 15.0f) << "collar roll snapped mid-clip (unwrap)"; - EXPECT_NEAR(twistDegAbout(last, Ogre::Vector3::UNIT_X), 120.0f, 8.0f); + // 240° source twist hits the 150° runaway-unwrap cap FIRST, then the + // 0.5× collar gain: 150 × 0.5 = 75°. + EXPECT_NEAR(twistDegAbout(last, Ogre::Vector3::UNIT_X), 75.0f, 8.0f); sm->destroyEntity(ent); } From 9c54de74b8599573de642851a73c791a7043494c Mon Sep 17 00:00:00 2001 From: Fernando Date: Tue, 14 Jul 2026 16:31:47 -0300 Subject: [PATCH 06/23] =?UTF-8?q?fix(anim):=20quality=20follow-up=20?= =?UTF-8?q?=E2=80=94=20smooth-bake=20post-pass,=20tightened=20foot-pin,=20?= =?UTF-8?q?twist=20caps+low-pass,=20library=20gates=20(#837)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Field feedback after the epic branch: generated walks trembled, played 'too fast', barely moved the legs, and over-moved arms/head. Four root causes, four fixes: 1. FOOT PINNING OVER-DETECTED (the frozen legs): the height band was 0.18 of leg length — a walking foot only lifts ~0.05-0.15, so contact spans swallowed the swing phase and pinned the legs into a shuffle while the upper body kept full rate (which also reads as 'too fast'). Tightened to 0.06 band / 0.012 velocity / 4-frame minimum, plus a coverage guard: any single span covering >55% of the clip is a misdetection and is discarded. 2. TWIST JITTER (the trembling): theta comes from per-frame shortest-arc decompositions and jitters near degenerate aims. Added the issue's 'optional low-pass' (5-tap binomial per role) and replaced the single 150-degree cap with per-role caps (neck/head 30, spine 45, arms 90, legs 60, feet 45; hip keeps 150 for genuine facing turns). 3. SMOOTH-BAKE POST-PASS (the user's own trick, codified): AnimationMerger::smoothBakeAnimation re-grids the clip to a sparse rate then back to the clip rate — a temporal low-pass that removes residual retarget trembling. ON by default in generation, before arm-space and foot pinning so pin targets stay exact. CLI --no-smooth-bake / --smooth-fps N; MCP smooth_bake/smooth_fps; GUI default path. 4. LIBRARY TAKE GATES (the T-pose/raised arms and hidden head): several corpus takes render broken regardless of retarget quality — unresolved arm roles freeze the target's arms in its literal bind T-pose; zombie/skeleton takes hold arms horizontal; several rigs carry a per-bone axis inversion that points one arm skyward or the neck straight down (head renders thrown back). New curation gates in build-motion-library-v5.py judge each arm's SIGNED hang direction, require an upright neck chain, and drop locomotion takes with unresolved arms; CrouchWalk/CrouchRun map to a new 'crouch' action instead of polluting walk/run. Curated v6 library (87 clips, 22 actions) uploaded to HF. Render-verified: 4/4 random Rumba walks natural (arms hanging, head visible, full stride). Co-Authored-By: Claude Fable 5 --- scripts/build-motion-library-v5.py | 82 ++++++++++++++++++++++++++++++ src/AnimationControlController.cpp | 4 ++ src/AnimationMerger.cpp | 64 +++++++++++++++++++++-- src/AnimationMerger.h | 10 ++++ src/CLIPipeline.cpp | 25 ++++++++- src/CLIPipeline.h | 3 +- src/FootContact.h | 11 ++-- src/MCPServer.cpp | 11 ++++ 8 files changed, 199 insertions(+), 11 deletions(-) diff --git a/scripts/build-motion-library-v5.py b/scripts/build-motion-library-v5.py index 5171b566..6eeb0753 100644 --- a/scripts/build-motion-library-v5.py +++ b/scripts/build-motion-library-v5.py @@ -50,6 +50,9 @@ # animation-name (normalized) → action. Order matters: first hit wins. KEYWORDS = [ ("tpose", None), ("t_pose", None), ("bind", None), ("rest", None), + # compound names FIRST — "walk"/"run" below would swallow them + ("crouchwalk", "crouch"), ("crouchrun", "crouch"), + ("sneakwalk", "sneak"), ("sneakrun", "sneak"), ("walk", "walk"), ("run", "run"), ("jog", "run"), ("sprint", "run"), ("idle", "idle"), ("stand", "idle"), ("breath", "idle"), ("jump", "jump"), ("hop", "jump"), ("leap", "jump"), @@ -238,6 +241,85 @@ def axis(role): elif action in HORIZONTAL_OK: upness = 0.7 + # Placeholder-arm gate: game walk/run cycles are often authored with + # STATIC (T-pose) arms — legs stride while the arms stick straight out. + # Retargeted, that reads as a broken clip. If the legs carry real motion + # but BOTH arms are near-frozen, drop the take. + ARM_ROLES = (7, 8, 11, 12) + LEG_ROLES = (15, 16, 19, 20) + if len(quats) > 1 and action not in ("idle", "sit", "sleep"): + # Energy RELATIVE to the chest (role 2): world quats inherit the + # torso's sway, so locally-frozen arms still show world energy. + # rel = chest^-1 * bone isolates the limb's own motion. + def rel(f, r): + c = quats[f][2] + b = quats[f][r] + ci = [-c[0], -c[1], -c[2], c[3]] + return qmul(ci, b) + def group_energy(roles): + tot = 0.0 + for f in range(1, len(quats)): + tot += sum(quat_angle(rel(f - 1, r), rel(f, r)) + for r in roles) / len(roles) + return tot / (len(quats) - 1) + legs_e = group_energy(LEG_ROLES) + arms_e = group_energy(ARM_ROLES) + if legs_e > 0.004 and arms_e < 0.0015: + return 0.0, (f"static placeholder arms (rel arm energy " + f"{arms_e:.4f} vs legs {legs_e:.4f})") + + # Neck-up gate: some rigs export the neck/head bone with an INVERTED + # axis — the extracted direction points down for the whole clip and the + # retargeted head renders thrown back / buried in the torso. For any + # upright action, the neck chain's animated direction must stay roughly + # up. + if action not in HORIZONTAL_OK and rest_world and rest_dir: + for r in (3, 4, 5): # neck, neck1, head + d = vnorm(rest_dir[r]) + if d is None: + continue + q = rest_world[r] + a_s = qrot([-q[0], -q[1], -q[2], q[3]], d) + tot = sum(qrot(quats[f][r], a_s)[1] for f in range(len(quats))) + if tot / max(1, len(quats)) < 0.3: + return 0.0, (f"neck/head direction not upright (role {r} " + f"mean up-dot {tot / max(1, len(quats)):.2f}) — " + "inverted neck axis, head renders thrown back") + break # first resolvable is enough + + # Horizontal-arm gate for plain locomotion: zombie-shamble / T-pose-armed + # walk cycles hold the upper arms near-horizontal for the whole clip + # (mean |up-component| of the upper-arm direction ~0 vs 0.6-0.95 on a + # natural walk). Retargeted onto a generic character under a generic + # "walk" prompt they read broken — drop them from locomotion actions. + RSHO, LSHO = 7, 11 + if action in ("walk", "run", "march") and rest_world and rest_dir: + downdots = [] + for r in (RSHO, LSHO): + d = vnorm(rest_dir[r]) + if d is None: + continue + q = rest_world[r] + a_s = qrot([-q[0], -q[1], -q[2], q[3]], d) + tot = 0.0 + for f in range(len(quats)): + tot += qrot(quats[f][r], a_s)[1] # SIGNED up-component + downdots.append(tot / max(1, len(quats))) + if not downdots: + # Arm roles UNRESOLVED: the retarget leaves the target's arms at + # its bind pose — a literal T-pose held for the whole clip. + return 0.0, (f"arm roles unresolved — target arms would freeze " + f"in the bind T-pose during {action}") + # judge each arm separately, on the SIGNED up-component: a natural + # locomotion arm hangs (mean Y ~ -0.6..-0.95). Horizontal zombie arms + # (~0) AND raised arms (+) both read broken — an abs() gate passed a + # straight-up arm as if it were hanging (several corpus rigs carry a + # per-bone axis inversion that renders one arm skyward). + if max(downdots) > -0.25: + return 0.0, (f"arm(s) not hanging (upper-arm signed up-dots " + f"{[round(u, 2) for u in downdots]}) — zombie/" + f"T-pose/raised style, not a generic {action}") + # Energy band: below = a pose, way above = spasm/mis-mapped. lo, hi, cap = min_energy * 2.0, 0.10, 0.20 if mean_energy <= lo: diff --git a/src/AnimationControlController.cpp b/src/AnimationControlController.cpp index 546e1858..48c133c3 100644 --- a/src/AnimationControlController.cpp +++ b/src/AnimationControlController.cpp @@ -1940,6 +1940,10 @@ QVariantMap AnimationControlController::generateMotion(const QString& prompt, if (!res.ok) return fail(res.error); out["source"] = clipSource; + // #837 quality post-pass: sparse-bake temporal low-pass (removes + // retarget trembling). Before arm-space/foot-pin so pins stay exact. + AnimationMerger::smoothBakeAnimation(skel.get(), animName, 12, fps); + // #854: optional Mixamo-style arm-space post-process. if (std::abs(armSpaceDeg) > 1e-4) AnimationMerger::adjustArmSpace(skel.get(), animName, diff --git a/src/AnimationMerger.cpp b/src/AnimationMerger.cpp index fdfcf17e..6d3337df 100644 --- a/src/AnimationMerger.cpp +++ b/src/AnimationMerger.cpp @@ -1471,6 +1471,16 @@ bool AnimationMerger::adjustArmSpace(Ogre::Skeleton* skel, return true; } +int AnimationMerger::smoothBakeAnimation(Ogre::Skeleton* skel, + const std::string& animName, + int sparseFps, int targetFps) +{ + if (!skel || sparseFps <= 0 || targetFps <= 0) return 0; + if (sparseFps >= targetFps) return 0; // nothing to low-pass + if (bakeAnimationAtFps(skel, animName, sparseFps) == 0) return 0; + return bakeAnimationAtFps(skel, animName, targetFps); +} + namespace { std::string footPinKey(const std::string& animName) { @@ -1600,7 +1610,17 @@ AnimationMerger::FootPinResult AnimationMerger::pinFeet( - Wpos[0][static_cast(leg.thigh)]).length() + (Wpos[0][static_cast(leg.foot)] - Wpos[0][static_cast(leg.shin)]).length(); - const auto spans = FootContact::detectContacts(footC, legLen); + auto spans = FootContact::detectContacts(footC, legLen); + // Coverage guard: a single "contact" spanning most of the clip is a + // misdetection (loose thresholds on a moving clip) — pinning it + // freezes the leg for the whole animation. Genuine stance phases in + // gait are well under this. + spans.erase(std::remove_if(spans.begin(), spans.end(), + [&](const FootContact::Span& sp) { + return (sp.end - sp.start + 1) + > (nk * 55) / 100; + }), + spans.end()); if (spans.empty()) continue; res.spans += static_cast(spans.size()); @@ -1911,7 +1931,18 @@ AnimationMerger::ApplyMotionResult AnimationMerger::applyMotionClip( 0.5f, 1.f, 1.f, 1.f, // lcollar damped, left arm 1.f, 1.f, 1.f, 1.f, // right leg + foot 1.f, 1.f, 1.f, 1.f }; // left leg + foot - constexpr float kTwistCap = 2.618f; // 150° — runaway-unwrap guard + // Per-role twist caps (radians). The single generous 150° cap + // let noisy source roll through on spine/neck/head — takes whose + // roll was invisible pre-#857 (dropped) came back with flailing + // arms and a thrown-back head. Roll matters most on forearms; + // the axial chain needs very little. Hip keeps a wide cap: it + // carries genuine facing turns (salsa). + static const float kTwistCapRole[22] = { + 2.62f, 0.79f, 0.79f, 0.52f, 0.52f, 0.52f, // hip, spine 45°, neck/head 30° + 0.52f, 1.57f, 1.57f, 1.57f, // rcollar 30°, right arm 90° + 0.52f, 1.57f, 1.57f, 1.57f, // lcollar 30°, left arm 90° + 1.05f, 1.05f, 1.05f, 0.79f, // right leg 60°, foot 45° + 1.05f, 1.05f, 1.05f, 0.79f }; // left leg 60°, foot 45° std::vector> twistTheta( static_cast(frames), std::vector(static_cast(Jc), 0.0f)); @@ -1944,9 +1975,34 @@ AnimationMerger::ApplyMotionResult AnimationMerger::applyMotionClip( prev[static_cast(c)] = th; has[static_cast(c)] = 1; twistTheta[static_cast(f)] - [static_cast(c)] = - std::clamp(th, -kTwistCap, kTwistCap); + [static_cast(c)] = std::clamp( + th, -kTwistCapRole[c], kTwistCapRole[c]); } + // Low-pass the twist trajectories (5-tap binomial). θ comes + // from per-frame shortest-arc decompositions and jitters near + // degenerate aims — transporting it raw renders as trembling. + // Directions are untouched; only the roll is smoothed. + if (frames >= 5) { + for (int c = 0; c < Jc; ++c) { + std::vector src(static_cast(frames)); + for (int f = 0; f < frames; ++f) + src[static_cast(f)] = + twistTheta[static_cast(f)] + [static_cast(c)]; + static const float k[5] = {1.f, 4.f, 6.f, 4.f, 1.f}; + for (int f = 0; f < frames; ++f) { + float acc = 0.0f, wsum = 0.0f; + for (int o = -2; o <= 2; ++o) { + const int j = f + o; + if (j < 0 || j >= frames) continue; + acc += k[o + 2] * src[static_cast(j)]; + wsum += k[o + 2]; + } + twistTheta[static_cast(f)] + [static_cast(c)] = acc / wsum; + } + } + } } // Reference-aligned roll baseline per bone: aim the target bind // at the source's REFERENCE direction once, so every per-frame diff --git a/src/AnimationMerger.h b/src/AnimationMerger.h index e1bc02f5..bb7e97dc 100644 --- a/src/AnimationMerger.h +++ b/src/AnimationMerger.h @@ -260,6 +260,16 @@ class AnimationMerger { const std::string& oldAnim, const std::string& newAnim); + /// #837 quality post-pass: re-grid the animation to a SPARSE keyframe + /// rate, then back to `targetFps` — a temporal low-pass that removes + /// retarget jitter ("trembling") while preserving the silhouette and the + /// clip length (both passes keep endpoints). Codifies the field-proven + /// trick of baking sparse and re-baking at 30 FPS. Returns the final + /// keyframe count (0 = animation missing / invalid fps). + static int smoothBakeAnimation(Ogre::Skeleton* skel, + const std::string& animName, + int sparseFps = 12, int targetFps = 30); + /// Outcome of pinFeet. struct FootPinResult { bool ok = false; diff --git a/src/CLIPipeline.cpp b/src/CLIPipeline.cpp index 7f3c408c..82543b23 100644 --- a/src/CLIPipeline.cpp +++ b/src/CLIPipeline.cpp @@ -2017,7 +2017,8 @@ int CLIPipeline::cmdFix(int argc, char* argv[]) int CLIPipeline::cmdAnimGenerate(const QString& filePath, const QString& prompt, float duration, const QString& outputPath, bool jsonOutput, bool useModel, - float armSpaceDeg, bool footPin) + float armSpaceDeg, bool footPin, + int smoothFps) { // #411 text-to-motion (template-clip MVP): match the prompt to a permissive // CMU motion clip from the downloadable library, retarget it onto the mesh's @@ -2144,6 +2145,17 @@ int CLIPipeline::cmdAnimGenerate(const QString& filePath, const QString& prompt, } err() << "(source: " << clipSource << ")" << Qt::endl; + // #837 quality post-pass (ON by default, --no-smooth-bake disables, + // --smooth-fps N tunes): bake sparse -> re-bake at clip rate. A temporal + // low-pass that removes retarget trembling; runs BEFORE arm-space and + // foot pinning so the pin targets stay exact. + if (smoothFps > 0) { + if (AnimationMerger::smoothBakeAnimation(skel.get(), animName, + smoothFps, fps) > 0) + err() << "(smooth-bake: " << smoothFps << " -> " << fps + << " fps)" << Qt::endl; + } + // #854: optional Mixamo-style arm-space post-process before export. if (std::abs(armSpaceDeg) > 1e-4f) { if (AnimationMerger::adjustArmSpace(skel.get(), animName, armSpaceDeg)) { @@ -2224,6 +2236,7 @@ int CLIPipeline::cmdAnim(int argc, char* argv[]) bool armSpaceSet = false; // --arm-space given (standalone post-adjust) bool generateFootPin = true; // #856: pin feet after --generate (default ON) bool footPinSet = false; // --foot-pin given (standalone post-process) + int generateSmoothFps = 12; // #837: sparse-bake low-pass (0 = off) bool jsonOutput = false; int resampleCount = 0; int decimateStep = 0; @@ -2322,6 +2335,13 @@ int CLIPipeline::cmdAnim(int argc, char* argv[]) // EXISTING animation (standalone, needs --animation). if (arg == "--no-foot-pin") { generateFootPin = false; continue; } if (arg == "--foot-pin") { footPinSet = true; continue; } + // #837 smooth-bake post-pass on --generate: bake sparse then back to + // the clip rate (temporal low-pass, kills retarget trembling). + if (arg == "--no-smooth-bake") { generateSmoothFps = 0; continue; } + if (arg == "--smooth-fps" && i + 1 < argc) { + generateSmoothFps = QString(argv[++i]).toInt(); + continue; + } if (arg == "--simplify") { simplifyMode = true; continue; } if (arg == "--analyze") { analyzeMode = true; continue; } if (arg == "--preset" && i + 1 < argc) { @@ -2417,7 +2437,8 @@ int CLIPipeline::cmdAnim(int argc, char* argv[]) } return cmdAnimGenerate(filePath, generatePrompt, generateDuration, outputPath.isEmpty() ? filePath : outputPath, jsonOutput, - generateUseModel, armSpaceDeg, generateFootPin); + generateUseModel, armSpaceDeg, generateFootPin, + generateSmoothFps); } // #854 standalone: post-adjust the arm space of an EXISTING animation diff --git a/src/CLIPipeline.h b/src/CLIPipeline.h index 22391f2f..743189f3 100644 --- a/src/CLIPipeline.h +++ b/src/CLIPipeline.h @@ -93,7 +93,8 @@ class CLIPipeline { static int cmdAnimGenerate(const QString& filePath, const QString& prompt, float duration, const QString& outputPath, bool jsonOutput, bool useModel = false, - float armSpaceDeg = 0.0f, bool footPin = true); + float armSpaceDeg = 0.0f, bool footPin = true, + int smoothFps = 12); static int cmdValidate(int argc, char* argv[]); static int cmdLod(int argc, char* argv[]); static int cmdPose(int argc, char* argv[]); diff --git a/src/FootContact.h b/src/FootContact.h index ead2e52b..5ccd9a39 100644 --- a/src/FootContact.h +++ b/src/FootContact.h @@ -27,13 +27,16 @@ struct Span { struct DetectOptions { /// Height band above the detected ground level counting as "on the - /// ground", as a fraction of leg length. - float heightBandFrac = 0.18f; + /// ground", as a fraction of leg length. Deliberately TIGHT: a walking + /// foot lifts only ~5-15% of leg length, and a generous band swallows + /// most of the swing phase — the whole clip pins and the character + /// shuffles (legs frozen while the upper body keeps full rate). + float heightBandFrac = 0.06f; /// Max horizontal speed (units/frame) counting as "stationary", as a /// fraction of leg length. - float velThreshFrac = 0.03f; + float velThreshFrac = 0.012f; /// Spans shorter than this are noise, not contacts. - int minFrames = 3; + int minFrames = 4; /// Up axis: 0=X, 1=Y, 2=Z (canonical rigs are +Y-up). int upAxis = 1; }; diff --git a/src/MCPServer.cpp b/src/MCPServer.cpp index f96ba612..65e732cb 100644 --- a/src/MCPServer.cpp +++ b/src/MCPServer.cpp @@ -4132,6 +4132,15 @@ QJsonObject MCPServer::toolGenerateMotion(const QJsonObject &args) clipDirs); if (!r.ok) return makeErrorResult(QString("Error: %1").arg(r.error)); + // #837 quality post-pass (ON by default): sparse-bake temporal + // low-pass — removes retarget trembling. Runs before arm-space and + // foot pinning so the pin targets stay exact. + const int smoothFps = args.value("smooth_bake").toBool(true) + ? args.value("smooth_fps").toInt(12) : 0; + if (smoothFps > 0) + AnimationMerger::smoothBakeAnimation(skel.get(), animName, + smoothFps, fps); + // #854: optional Mixamo-style arm-space post-process. Echo whether it // took effect so an MCP caller can tell the rig had no arm roles // (rather than silently getting an unadjusted clip). @@ -8222,6 +8231,8 @@ QJsonArray MCPServer::buildToolsList() props["model"] = QJsonObject{{"type", "boolean"}, {"description", "EXPERIMENTAL: use the trained from-scratch text-to-motion ONNX model instead of the template clip. Falls back to the template library automatically if the model is unavailable or the action isn't in its vocabulary. Default false (template). Quality is action-dependent (locomotion better than gestures)."}}; props["arm_space"] = QJsonObject{{"type", "number"}, {"description", "Optional Mixamo-style arm-space post-process in degrees (#854): positive widens the arms away from the body, negative tucks them in. Default 0. Rescues arm-into-torso clipping on rigs whose proportions differ from the clip."}}; props["foot_pin"] = QJsonObject{{"type", "boolean"}, {"description", "Foot-contact cleanup (#856): detect ground-contact spans and IK-pin the feet so they plant instead of skating. Default true; set false to keep the raw retarget."}}; + props["smooth_bake"] = QJsonObject{{"type", "boolean"}, {"description", "Temporal low-pass post-pass: bake the clip sparse then back to its native rate, removing retarget trembling. Default true."}}; + props["smooth_fps"] = QJsonObject{{"type", "number"}, {"description", "Sparse keyframe rate for the smooth-bake pass. Lower = smoother but softer motion. Default 12."}}; appendTool( "generate_motion", "AI text-to-motion (#411, experimental): generate a skeletal animation from a text prompt and " From 47932407229c0c9e09ae65c07d25e637f0790392 Mon Sep 17 00:00:00 2001 From: Fernando Date: Fri, 17 Jul 2026 03:24:25 -0400 Subject: [PATCH 07/23] feat(scripts): posture gates on t2m training windows (#837 quality follow-up) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The v5 model learned a head-down hunched walk because training windows were unfiltered — folded, idle-contaminated and placeholder-armed source content trained in. In the canonical representation the checks are trivial (d(f) = Q'(f)*D_c): spine and neck/head must stay up (stricter for locomotion), and locomotion upper arms must HANG — judged on the SIGNED up-component per arm, the library-curation lesson (abs() passes a skyward arm). Dropped 841 of 17,193 windows; the retrained model (hosted as t2m v5.1) walks upright with stepping legs on both the Mixamo test rig and the chibi mouse that exposed the hunch. Co-Authored-By: Claude Fable 5 --- scripts/prep-t2m-v5.py | 39 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/scripts/prep-t2m-v5.py b/scripts/prep-t2m-v5.py index 4b0a07a6..130fa532 100644 --- a/scripts/prep-t2m-v5.py +++ b/scripts/prep-t2m-v5.py @@ -290,6 +290,38 @@ def angdist(x, y): d = np.abs((x * y).sum(-1)).clip(0, 1) return 2.0 * np.arccos(d) + LOCOMOTION = {"walk", "run", "march"} + + def posture_ok(action, cq, valid): + """Posture gates (#837 quality follow-up): the v5 model learned a + head-down hunched walk because unfiltered windows include folded / + idle-contaminated / placeholder-armed source content. In the + CANONICAL rep the checks are trivial — d(f) = Q'(f)·D_c: + spine must stay up, neck/head must stay up, and locomotion arms + must HANG (signed Y, the library-curation lesson: abs() passes a + skyward arm).""" + def mean_y(r): + d = qrot(cq[:, r], np.broadcast_to(D_CANON[r], (len(cq), 3))) + return float(d[:, 1].mean()) + floor = 0.7 if action in LOCOMOTION else 0.5 + for r in (0, 1, 2): # spine chain + if valid[r]: + if mean_y(r) < floor: + return False + break + for r in (3, 4, 5): # neck / head + if valid[r]: + if mean_y(r) < 0.5: + return False + break + if action in LOCOMOTION: + for r in (7, 11): # upper arms hang + if valid[r] and mean_y(r) > -0.25: + return False + return True + + dropped = [0] + def window(action, cq, valid, src): # The v4 neutral-start gate is deliberately GONE: model clips now ride # the bind-referenced direction retarget, which references the @@ -306,7 +338,11 @@ def window(action, cq, valid, src): cq = np.concatenate(reps, 0)[:T] nF = T for s in range(0, nF - T + 1, max(1, T // 2)): - mo.append(cq[s:s + T]) + w = cq[s:s + T] + if not posture_ok(action, w, valid): + dropped[0] += 1 + continue + mo.append(w) msk.append(valid) acts.append(action) srcs.append(src) @@ -325,6 +361,7 @@ def window(action, cq, valid, src): n0 += 1 print(f"cmu: {n0} trials → {len(mo) - w0} windows") + print(f"posture gates dropped {dropped[0]} windows") if not mo: sys.exit("no windows extracted") From 9c8570a89ceee300df7c664a801eb80115ae140e Mon Sep 17 00:00:00 2001 From: Fernando Date: Fri, 17 Jul 2026 19:14:12 -0400 Subject: [PATCH 08/23] fix(scripts): default t2m export guidance to 1.0 (#837) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CFG at 2.0 on the posture-filtered dataset extrapolates past the (now upright) conditional mean into a backward torso arch — the overshoot is proportional to how far the filtered data sits from the uncond distribution. Render sweep on the Mixamo rig: g2.0 arched back, g1.3 head-back creep, g1.0 clean upright walk/run and an upright chibi-mouse walk. Guidance stays a knob for conditioning-starved future datasets. Co-Authored-By: Claude Fable 5 --- scripts/train-t2m-flow-v5.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/train-t2m-flow-v5.py b/scripts/train-t2m-flow-v5.py index 2c5754da..7ced1e6e 100644 --- a/scripts/train-t2m-flow-v5.py +++ b/scripts/train-t2m-flow-v5.py @@ -181,7 +181,11 @@ def main(): ap.add_argument("--dim", type=int, default=256) ap.add_argument("--layers", type=int, default=6) ap.add_argument("--steps", type=int, default=16, help="Euler export steps") - ap.add_argument("--guidance", type=float, default=2.0, + # Guidance default 1.0: with posture-FILTERED training data (already + # more upright than the uncond distribution), CFG > 1 extrapolates PAST + # upright into a backward arch — measured on the v5.1 retrain (g2.0 + # arched, g1.0 clean). Raise only for conditioning-starved datasets. + ap.add_argument("--guidance", type=float, default=1.0, help="CFG scale baked into the exported sampler") ap.add_argument("--device", default="mps") a = ap.parse_args() From 86fbd17d68f4c102819660be33f58f729f747e5a Mon Sep 17 00:00:00 2001 From: Fernando Date: Sat, 18 Jul 2026 23:21:40 -0400 Subject: [PATCH 09/23] feat(anim): v6 curation-grade t2m model + posture best-of-16 scorer (#837) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Days-scale quality pass on the text-to-motion model after field feedback (hunched walk, ballerina feet, head not level). Data (scripts/prep-t2m-v6.py): apply the SAME quality gates that curate the shipping template library to every training window — upright spine and neck/head (stricter for locomotion), locomotion upper arms must hang (signed per-arm Y), sane energy band — fold the curated library takes in directly, and augment with sagittal mirror + 0.85x/1.15x speed. 58,972 windows / 21 actions (was ~17k unfiltered), T=60. Model (scripts/train-t2m-flow-v5.py): 21.7M params (dim 384, 8 layers), 400 epochs, per-epoch checkpoint + --resume so multi-day runs survive sleeps. Converged at loss 0.037 (old floor ~0.09). Inference (src/MotionGenerator.cpp): 16-candidate best-of-N ranked by posture criteria that DOMINATE the motion terms — spine/head up, head not tipped back (|forward-Z| penalty), each upper arm hanging (worst arm dominates) — so a bad draw can never win. Plus a foot-articulation guard: when the model under-moves a foot role, drop its reference dir so the rig keeps its bind foot pitch (kills the ballerina toes). Numerically on-distribution vs the training walk: spine Y 0.99 (data 0.98), head Y 0.84 (0.88), arms Y -0.93/-0.95 (data -0.87). Render- verified upright walk/run on the Mixamo rig and the chibi mouse that exposed the original hunch. Co-Authored-By: Claude Opus 4.8 (1M context) --- scripts/prep-t2m-v6.py | 217 +++++++++++++++++++++++++++++++++++ scripts/train-t2m-flow-v5.py | 19 ++- src/MotionGenerator.cpp | 85 +++++++++++++- 3 files changed, 315 insertions(+), 6 deletions(-) create mode 100644 scripts/prep-t2m-v6.py diff --git a/scripts/prep-t2m-v6.py b/scripts/prep-t2m-v6.py new file mode 100644 index 00000000..fcf8432e --- /dev/null +++ b/scripts/prep-t2m-v6.py @@ -0,0 +1,217 @@ +#!/usr/bin/env python3 +# ruff: noqa: E702, E741 +"""Build the v6 text-to-motion training cache — CURATION-GRADE (#837). + +ONE-TIME OFFLINE dev tool — NOT shipped. Successor to prep-t2m-v5.py. + +The v5.1 model still rendered mushy walks: flow matching samples the +distribution it is given, and the v5 distribution contained everything the +extractor produced — turns, pauses, off-poses, style outliers. v6 applies +the SAME quality bar that curates the shipping template library to every +individual training window, folds the curated library takes themselves into +the set, and augments: + + gates (per window, canonical rep — d(f) = Q'(f)·D_c): + - spine + neck/head upright (stricter for locomotion) + - locomotion upper arms HANG, judged per arm on the SIGNED up-component + - energy band: mean joint speed in [lo, hi] (poses and spasms both out) + augmentation: + - sagittal MIRROR: q' = (x, -y, -z, w) + swap L/R roles (doubles data, + teaches symmetry) + - SPEED 0.85x / 1.15x (slerp resample) + +Windows are T=60 @ 30 fps (2 s). Output schema matches prep-t2m-v5 +(mo/msk/tk/vocab/fps/canonRestDir) so train-t2m-flow-v5.py runs unchanged. + +Usage: + python3 scripts/prep-t2m-v6.py --corpus ~/motion_corpus \ + --bvh /data --index /cmu-mocap-index-text.txt \ + --library ~/t2m_v6/motion-library.json --out ~/t2m_v6/t2m_v6.npz +""" +import argparse +import importlib.util +import json +import os +import sys + +import numpy as np + +HERE = os.path.dirname(os.path.abspath(__file__)) + + +def load_module(name, fname): + spec = importlib.util.spec_from_file_location(name, os.path.join(HERE, fname)) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +prep5 = load_module("prep5", "prep-t2m-v5.py") + +J = 22 +FPS = 30 +D_CANON = prep5.D_CANON +# canonical L/R role pairs (AnimationMerger kLR) +LR = [(6, 10), (7, 11), (8, 12), (9, 13), (14, 18), (15, 19), (16, 20), (17, 21)] +MIRROR_PERM = list(range(J)) +for a, b in LR: + MIRROR_PERM[a], MIRROR_PERM[b] = b, a + +LOCOMOTION = {"walk", "run", "march"} +HORIZONTAL_OK = {"death", "crawl", "roll", "swim", "fall", "sleep", "sit"} + + +def qrot(q, v): + qv = q[..., :3] + uv = np.cross(qv, v) + uuv = np.cross(qv, uv) + return v + 2.0 * (q[..., 3:4] * uv + uuv) + + +def mean_dir_y(w, r): + d = qrot(w[:, r], np.broadcast_to(D_CANON[r], (len(w), 3))) + return float(d[:, 1].mean()) + + +def window_quality(action, w, valid): + """True when the window meets the library curation bar.""" + # energy band — mean joint rotation speed (rad/frame) + dq = np.abs((w[1:] * w[:-1]).sum(-1)).clip(0, 1) + e = float((2 * np.arccos(dq)).mean()) + if not (0.004 <= e <= 0.11): + return False + if action in HORIZONTAL_OK: + return True + floor = 0.7 if action in LOCOMOTION else 0.5 + for r in (0, 1, 2): + if valid[r]: + if mean_dir_y(w, r) < floor: + return False + break + for r in (3, 4, 5): + if valid[r]: + if mean_dir_y(w, r) < 0.5: + return False + break + if action in LOCOMOTION: + for r in (7, 11): + if valid[r] and mean_dir_y(w, r) > -0.25: + return False + return True + + +def mirror(w, valid): + """Sagittal mirror: reflect each quat (x,-y,-z,w) and swap L/R roles.""" + m = w * np.array([1, -1, -1, 1], np.float32) + return m[:, MIRROR_PERM], valid[MIRROR_PERM] + + +def retime(w, factor): + """Slerp-resample a [T,J,4] window to the same length at `factor` speed.""" + T = w.shape[0] + src = np.clip(np.arange(T, dtype=np.float64) * factor, 0, T - 1.001) + i0 = src.astype(int) + t = (src - i0)[:, None, None].astype(np.float32) + a, b = w[i0], w[np.minimum(i0 + 1, T - 1)] + # hemisphere-align then nlerp (windows are 30fps — angles tiny) + dot = (a * b).sum(-1, keepdims=True) + b = np.where(dot < 0, -b, b) + out = a * (1 - t) + b * t + return out / (np.linalg.norm(out, axis=-1, keepdims=True) + 1e-12) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--corpus", default="") + ap.add_argument("--bvh", default="") + ap.add_argument("--index", default="") + ap.add_argument("--library", default="", + help="curated motion-library.json — takes are folded in " + "as extra (already-curated) source clips") + ap.add_argument("--out", default=os.path.expanduser("~/t2m_v6/t2m_v6.npz")) + ap.add_argument("--T", type=int, default=60) + ap.add_argument("--min-roles", type=int, default=12) + ap.add_argument("--min-action-windows", type=int, default=16) + a = ap.parse_args() + + T = a.T + mo, msk, acts = [], [], [] + gated = [0] + + def add(action, w, valid): + mo.append(w) + msk.append(valid) + acts.append(action) + + def windows(action, cq, valid): + nF = cq.shape[0] + if nF < T // 2: + return + if nF < T: + dq = np.abs((cq[-1] * cq[0]).sum(-1)).clip(0, 1) + cyc = float((2 * np.arccos(dq)).mean()) < 0.25 + reps = [cq] + while sum(r.shape[0] for r in reps) < T: + reps.append(cq if cyc else cq[-1:].repeat(T, 0)) + cq = np.concatenate(reps, 0)[:T] + nF = T + for s in range(0, nF - T + 1, max(1, T // 3)): + w = cq[s:s + T] + if not window_quality(action, w, valid): + gated[0] += 1 + continue + add(action, w, valid) + mw, mv = mirror(w, valid) + add(action, mw, mv) + for f in (0.85, 1.15): + add(action, retime(w, f), valid) + + if a.corpus: + n = 0 + for action, cq, valid, _src in prep5.corpus_clips( + os.path.expanduser(a.corpus), a.min_roles): + windows(action, cq, valid) + n += 1 + print(f"corpus: {n} clips → {len(mo)} windows (cum)") + if a.library: + lib = json.load(open(os.path.expanduser(a.library))) + n = 0 + for c in lib.get("clips", []): + rw, rd = c.get("restWorld"), c.get("restDir") + if not rw or not rd: + continue + cq, valid = prep5.canonicalize( + np.asarray(c["quats"], np.float32), rw, rd) + windows(c["action"], cq, valid) + n += 1 + print(f"library: {n} takes → {len(mo)} windows (cum)") + if a.bvh and a.index: + n = 0 + for action, cq, valid, _src in prep5.cmu_clips(a.bvh, a.index): + windows(action, cq, valid) + n += 1 + print(f"cmu: {n} trials → {len(mo)} windows (cum)") + + print(f"quality gates dropped {gated[0]} base windows") + if not mo: + sys.exit("no windows") + from collections import Counter + cnt = Counter(acts) + vocab = sorted(w for w, k in cnt.items() if k >= a.min_action_windows) + keep = [i for i, w in enumerate(acts) if w in vocab] + mo = np.stack([mo[i] for i in keep]).astype(np.float32) + msk = np.stack([msk[i] for i in keep]).astype(np.float32) + tk = np.zeros((len(keep), len(vocab)), np.float32) + for r, i in enumerate(keep): + tk[r, vocab.index(acts[i])] = 1.0 + print(f"windows: {mo.shape[0]} vocab({len(vocab)}):", + {w: int(tk[:, vocab.index(w)].sum()) for w in vocab}) + os.makedirs(os.path.dirname(os.path.abspath(a.out)), exist_ok=True) + np.savez_compressed(a.out, mo=mo, msk=msk, tk=tk, + vocab=np.array(vocab), fps=FPS, + canonRestDir=D_CANON) + print(f"wrote {a.out} mo{mo.shape}") + + +if __name__ == "__main__": + main() diff --git a/scripts/train-t2m-flow-v5.py b/scripts/train-t2m-flow-v5.py index 7ced1e6e..6ea8e089 100644 --- a/scripts/train-t2m-flow-v5.py +++ b/scripts/train-t2m-flow-v5.py @@ -188,6 +188,9 @@ def main(): ap.add_argument("--guidance", type=float, default=1.0, help="CFG scale baked into the exported sampler") ap.add_argument("--device", default="mps") + ap.add_argument("--resume", action="store_true", + help="resume from /ckpt.pt (long runs survive " + "sleep/restarts)") a = ap.parse_args() d = np.load(a.data, allow_pickle=True) @@ -221,7 +224,18 @@ def main(): sched = torch.optim.lr_scheduler.CosineAnnealingLR( opt, T_max=a.epochs * max(1, len(dl))) - for ep in range(a.epochs): + os.makedirs(a.out, exist_ok=True) + ckpt_path = os.path.join(a.out, "ckpt.pt") + start_ep = 0 + if a.resume and os.path.exists(ckpt_path): + ck = torch.load(ckpt_path, map_location=dev) + net.load_state_dict(ck["net"]) + opt.load_state_dict(ck["opt"]) + sched.load_state_dict(ck["sched"]) + start_ep = ck["epoch"] + 1 + print(f"resumed from epoch {start_ep}", flush=True) + + for ep in range(start_ep, a.epochs): tot, nb = 0.0, 0 for xb, mb, tb in dl: xb, mb, tb = xb.to(dev), mb.to(dev), tb.to(dev) @@ -243,8 +257,9 @@ def main(): sched.step() tot += loss.item(); nb += 1 print(f"ep {ep + 1}/{a.epochs} loss {tot / max(1, nb):.4f}", flush=True) + torch.save({"net": net.state_dict(), "opt": opt.state_dict(), + "sched": sched.state_dict(), "epoch": ep}, ckpt_path) - os.makedirs(a.out, exist_ok=True) torch.save(net.state_dict(), os.path.join(a.out, "flow.pt")) # ---- export: sampler-unrolled ONNX + vocab json ---- diff --git a/src/MotionGenerator.cpp b/src/MotionGenerator.cpp index e8f8cd14..012ed98d 100644 --- a/src/MotionGenerator.cpp +++ b/src/MotionGenerator.cpp @@ -309,10 +309,34 @@ MotionGenerator::Result MotionGenerator::generate( // NOSONAR — non-crypto: seeds latent-noise sampling for motion variety. std::mt19937 rng(QRandomGenerator::global()->generate()); // NOSONAR std::normal_distribution gauss(0.0f, 0.5f); - constexpr int kCandidates = 8; + constexpr int kCandidates = 16; constexpr float kTargetStep = 0.048f; // rad/frame — real-clip locals constexpr float kMaxArtic = 1.5f; // rad from the starting pose + // Posture-aware ranking (#837 quality follow-up): v5+ models are + // CANONICAL (restWorld = identity), so a bone's world direction is + // simply q·restDir — the same measurements that curate the template + // library rank the candidates here: spine and head must stay up, + // and locomotion upper arms must HANG (signed Y — an abs() check + // would pass a skyward arm). + const bool haveCanonDirs = !vocabRestDir.empty(); + const bool locomotion = matched == QLatin1String("walk") + || matched == QLatin1String("run") + || matched == QLatin1String("march"); + auto qRotDir = [&](const Q4& q, int role) -> std::array { + const auto& d = vocabRestDir[static_cast(role)]; + const float qx = q[0], qy = q[1], qz = q[2], qw = q[3]; + const float ux = qy * d[2] - qz * d[1]; + const float uy = qz * d[0] - qx * d[2]; + const float uz = qx * d[1] - qy * d[0]; + const float vx = qy * uz - qz * uy; + const float vy = qz * ux - qx * uz; + const float vz = qx * uy - qy * ux; + return { d[0] + 2.0f * (qw * ux + vx), + d[1] + 2.0f * (qw * uy + vy), + d[2] + 2.0f * (qw * uz + vz) }; + }; + std::vector> best; // [T][J] float bestScore = -1e30f; for (int cand = 0; cand < kCandidates; ++cand) { @@ -371,9 +395,44 @@ MotionGenerator::Result MotionGenerator::generate( } const float step = static_cast(stepSum / ((T - 1) * (J - 1))); const float coh = cohN ? static_cast(cohSum / cohN) : 0.0f; - const float score = -std::abs(step - kTargetStep) * 40.0f - + coh * 2.0f - - std::max(0.0f, maxArtic - kMaxArtic) * 4.0f; + float score = -std::abs(step - kTargetStep) * 40.0f + + coh * 2.0f + - std::max(0.0f, maxArtic - kMaxArtic) * 4.0f; + if (haveCanonDirs && worldFrame) { + // Posture penalties DOMINATE the energy/coherence terms: a + // bad sample (head thrown back, an arm reaching out) is worse + // than a slightly-off-tempo good one, so the weights here are + // ~10x the motion terms. This is the "best-of-N must never + // pick a broken pose" guarantee — the model produces good + // walks most of the time, and the scorer's job is to reject + // the occasional flailing draw. + auto meanDir = [&](int role) -> std::array { + float ax = 0.f, ay = 0.f, az = 0.f; + for (int f = 0; f < T; ++f) { + const auto d = qRotDir(w[f][role], role); + ax += d[0]; ay += d[1]; az += d[2]; + } + const float inv = 1.0f / static_cast(T); + return { ax * inv, ay * inv, az * inv }; + }; + const float spineY = meanDir(1)[1]; // abdomen up-ness + const auto head = meanDir(5); // head direction + // head must point UP; penalize both drooping AND tipping back + // (|Z| forward/back lean of the head axis). + score -= std::max(0.0f, 0.85f - head[1]) * 60.0f; + score -= std::abs(head[2]) * 25.0f; + score -= std::max(0.0f, 0.85f - spineY) * 60.0f; + if (locomotion) { + // Each arm's upper-arm must HANG (signed Y strongly + // negative). An arm reaching out/forward sits near Y≈0 + // and is heavily penalized; the worst arm dominates so a + // single flung arm can't hide behind the other. + const float ry = meanDir(7)[1]; + const float ly = meanDir(11)[1]; + score -= std::max(0.0f, ry + 0.35f) * 40.0f; + score -= std::max(0.0f, ly + 0.35f) * 40.0f; + } + } if (score > bestScore) { bestScore = score; best = std::move(w); } } if (best.empty()) { @@ -410,6 +469,24 @@ MotionGenerator::Result MotionGenerator::generate( clip.restWorld = vocabRestWorld; clip.restDir = vocabRestDir; + if (!clip.restDir.empty()) { + // Ballerina-feet guard: feet are low-variance joints the model + // under-fits — near-static predicted feet aim the target's foot + // at the canonical FORWARD axis and point the toes. When a foot + // role barely articulates, drop its reference direction so the + // retarget leaves those bones at the rig's own bind pitch. + for (const int foot : {17, 21}) { + float maxDev = 0.0f; + const Q4& q0 = best[0][static_cast(foot)]; + for (int f = 1; f < T; ++f) { + const Q4 d = qMul(qConj(q0), + best[f][static_cast(foot)]); + maxDev = std::max(maxDev, qAngle(d)); + } + if (maxDev < 0.21f) // < ~12 degrees over the whole clip + clip.restDir[static_cast(foot)] = {0.f, 0.f, 0.f}; + } + } r.clip = std::move(clip); r.worldFrame = worldFrame; r.ok = true; From 66ac50f23cc9c0ba147d4601124db25578644e4e Mon Sep 17 00:00:00 2001 From: Fernando Date: Sun, 19 Jul 2026 13:34:47 -0400 Subject: [PATCH 10/23] fix(anim): stride-directionality gate fixes sideways t2m walk (#837) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Field report: v6 walk motion was good but stepped SIDEWAYS, twisting the hips and deforming the mesh. Root cause traced numerically: 59% of the raw CMU walk windows are themselves splayed/side-stepping (foot fwd/side travel < 1.5), and the model faithfully learned that majority — the v6 model's foot fwd/side ratio was 1.2 vs a clean stride's 3.3. Fix: a stride-directionality gate in prep-t2m-v6.py — forward-kinematics each window over the canonical bone directions and keep only clips whose feet travel forward >= 2x sideways. Drops the bad 59%, leaving 7,724 clean forward-stride walk windows (plenty). Retrained v6.1 (same 21.7M / 400ep setup): model foot fwd/side ratio 1.2 -> 2.29, posture held (spine 0.99, head 0.90, arms -0.98). Render-verified clean forward walk on the Mixamo rig and the chibi mouse that exposed the bug. Lesson: measure motion metrics NUMERICALLY via FK against the training distribution — the isometric sprite sheets foreshorten posture and hid both this and the earlier 'hunch' false alarm. Co-Authored-By: Claude Opus 4.8 (1M context) --- scripts/prep-t2m-v6.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/scripts/prep-t2m-v6.py b/scripts/prep-t2m-v6.py index fcf8432e..85417c67 100644 --- a/scripts/prep-t2m-v6.py +++ b/scripts/prep-t2m-v6.py @@ -73,6 +73,34 @@ def mean_dir_y(w, r): return float(d[:, 1].mean()) +# canonical parent chain (AnimationMerger kParentCanon) for FK +PAR = [-1, 0, 1, 2, 3, 4, 2, 6, 7, 8, 2, 10, 11, 12, 0, 14, 15, 16, 0, 18, 19, 20] + + +def foot_travel_ratio(w): + """fwd/side travel ratio of the feet over the window (unit bone lengths). + A clean walk/run steps front-to-back (Z >> X); a splayed/side-step stride + (the v6 model's failure mode) has X ~ Z. Positions come from the same + canonical-direction FK the retarget uses, so this measures exactly what + renders. Returns fwd/side (higher = cleaner).""" + T = len(w) + + def pos(role): + p = np.zeros((T, 3), np.float32) + r = role + while PAR[r] >= 0: + p = p + qrot(w[:, PAR[r]], np.broadcast_to(D_CANON[r], (T, 3))) + r = PAR[r] + return p + side = fwd = 0.0 + for foot in (17, 21): + c = pos(foot) + c = c - c.mean(0, keepdims=True) + side += float(np.abs(c[:, 0]).mean()) + fwd += float(np.abs(c[:, 2]).mean()) + return fwd / (side + 1e-6) + + def window_quality(action, w, valid): """True when the window meets the library curation bar.""" # energy band — mean joint rotation speed (rad/frame) @@ -97,6 +125,12 @@ def window_quality(action, w, valid): for r in (7, 11): if valid[r] and mean_dir_y(w, r) > -0.25: return False + # Stride-directionality gate (v6.1): the feet must step FORWARD, not + # sideways. 59% of raw CMU walk windows are splayed/side-stepping + # (measured fwd/side < 1.5) — the model faithfully learned that + # majority and walked sideways. Require a clean front-to-back stride. + if valid[17] and valid[21] and foot_travel_ratio(w) < 2.0: + return False return True From 977e2c8a71020a7e22e0843d88c6c2ac10fc2150 Mon Sep 17 00:00:00 2001 From: Fernando Date: Sun, 19 Jul 2026 21:36:38 -0400 Subject: [PATCH 11/23] =?UTF-8?q?fix(anim):=20condition=20model=20clips=20?= =?UTF-8?q?=E2=80=94=20facing,=20arm-spread,=20foot-pitch=20(#837)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Field report: the v6.1 model walk was good but (1) faced/travelled BACKWARD on the target rig, (2) pointed the toes (ballerina feet), and (3) hung the arms too narrow (crossing behind). All three are retarget- layer artifacts of the model's canonical convention, fixed in data-space so the delicate bind-referenced aim math is untouched: AnimationMerger::conditionModelClip (model clips only; templates are authored and skip it): (1) FACING — the model's canonical clip faces -Z, so it needs a 180deg yaw about canonical +Y to match the +Z-forward retarget. Applied to the clip quats (directions just negate in X/Z, so getRotationTo stays well-conditioned — the aim path itself can't apply yaw180 without sending aims near-anti-parallel to their bind dirs). The per-rig backward heuristic XORs with it (!yaw180): a backward rig cancels the model flip. Verified: front of body/face now toward camera on both the Mixamo rig and the chibi mouse. (2) ARMS — model hangs them too narrow (measured lateral 0.0-0.08 vs training 0.15); widen the upper-arm world quat ~12deg outward about canonical forward, mirrored per side, inherited by elbow/hand. (3) FEET — model under-pitches the feet (toe-Y -0.17 vs training -0.42) and the aim then points the toes; zero the foot restDir so the rig keeps its bind foot pitch (unconditional for model clips — the model's foot signal is never trustworthy). Wired into the CLI / GUI / MCP generate paths, gated on clipSource == 'model'. No model retrain needed; QTMESH_T2M_YAW180 override still works. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/AnimationControlController.cpp | 4 ++ src/AnimationMerger.cpp | 66 ++++++++++++++++++++++++++++++ src/AnimationMerger.h | 18 ++++++++ src/CLIPipeline.cpp | 7 ++++ src/MCPServer.cpp | 3 ++ 5 files changed, 98 insertions(+) diff --git a/src/AnimationControlController.cpp b/src/AnimationControlController.cpp index 48c133c3..0449b488 100644 --- a/src/AnimationControlController.cpp +++ b/src/AnimationControlController.cpp @@ -1932,6 +1932,10 @@ QVariantMap AnimationControlController::generateMotion(const QString& prompt, // Auto-rigged (no prior animation) meshes that face −Z would walk // backward — detect facing from the mesh's foot region. const bool yaw180 = AnimationMerger::detectBackwardFacing(entity); + // #837: model clips get data-space conditioning (facing flip, arm widen, + // foot-pitch reset); template clips are authored and skip it. + if (clipSource == QStringLiteral("model")) + AnimationMerger::conditionModelClip(quats, clipDirs, !yaw180); const auto res = AnimationMerger::applyMotionClip(skel.get(), animName, quats, fps, worldFrame, cmuRest, /*refineWithModel=*/false, diff --git a/src/AnimationMerger.cpp b/src/AnimationMerger.cpp index 6d3337df..59d55a9b 100644 --- a/src/AnimationMerger.cpp +++ b/src/AnimationMerger.cpp @@ -1471,6 +1471,72 @@ bool AnimationMerger::adjustArmSpace(Ogre::Skeleton* skel, return true; } +void AnimationMerger::conditionModelClip( + std::vector>>& quats, + std::vector>& restDir, + bool flipYaw, float armWidenDeg) +{ + const int Jc = MotionInbetween::canonicalJointCount(); + if (quats.empty()) return; + + // (1) Backward-facing rigs: rotate the whole CANONICAL clip 180° about + // canonical up (+Y). In world quats a yaw of π about Y is q → yaw·q with + // yaw=(0,1,0,0) (x,y,z,w). Directions negate in X/Z, so getRotationTo + // stays well-conditioned (unlike applying the flip inside the aim, which + // sends aims near-anti-parallel to their bind dirs). + if (flipYaw) { + // yaw(π,Y) as (w,x,y,z) = (0,0,1,0); premultiply each joint's world q. + auto yawMul = [](const std::array& q) { + // q is (x,y,z,w); yaw*(q): yaw=(x0=0,y0=1,z0=0,w0=0) + const float x = q[0], y = q[1], z = q[2], w = q[3]; + // (0,1,0,0)·(x,y,z,w) in (x,y,z,w) Hamilton order: + return std::array{ -z, w, x, -y }; + }; + for (auto& frame : quats) + for (int c = 0; c < Jc && c < static_cast(frame.size()); ++c) + frame[static_cast(c)] = + yawMul(frame[static_cast(c)]); + // restDir is expressed in the canonical frame → negate X/Z there too. + for (int c = 0; c < Jc && c < static_cast(restDir.size()); ++c) { + restDir[static_cast(c)][0] = + -restDir[static_cast(c)][0]; + restDir[static_cast(c)][2] = + -restDir[static_cast(c)][2]; + } + } + + // (2) Widen the arms: the model hangs them too narrow. Rotate each upper + // arm's world quat outward about the canonical FORWARD axis (+Z), mirrored + // per side (right = −X side rotates −, left = +X side rotates +), so both + // swing away from the torso. Applied to the upper-arm role and inherited + // by elbow/hand through the retarget hierarchy. + if (std::abs(armWidenDeg) > 1e-3f) { + const float a = armWidenDeg * static_cast(M_PI) / 180.0f; + const Ogre::Quaternion swR(Ogre::Radian(-a), Ogre::Vector3::UNIT_Z); + const Ogre::Quaternion swL(Ogre::Radian(a), Ogre::Vector3::UNIT_Z); + auto apply = [&](int role, const Ogre::Quaternion& sw) { + if (role >= static_cast(quats.front().size())) return; + for (auto& frame : quats) { + auto& q = frame[static_cast(role)]; + const Ogre::Quaternion cur(q[3], q[0], q[1], q[2]); + const Ogre::Quaternion out = sw * cur; // world premultiply + q = { out.x, out.y, out.z, out.w }; + } + }; + apply(7, swR); // rshoulder + apply(11, swL); // lshoulder + } + + // (3) Ballerina feet: the model under-articulates the feet, and the aim + // then points the toes at the canonical forward axis. Drop the foot + // reference dirs so the retarget leaves those bones at the rig's bind + // foot pitch (same rationale as the #856 foot guard, but unconditional + // for model clips — the model's foot signal is never trustworthy). + for (const int foot : {17, 21}) + if (foot < static_cast(restDir.size())) + restDir[static_cast(foot)] = {0.f, 0.f, 0.f}; +} + int AnimationMerger::smoothBakeAnimation(Ogre::Skeleton* skel, const std::string& animName, int sparseFps, int targetFps) diff --git a/src/AnimationMerger.h b/src/AnimationMerger.h index bb7e97dc..c405e0ea 100644 --- a/src/AnimationMerger.h +++ b/src/AnimationMerger.h @@ -270,6 +270,24 @@ class AnimationMerger { const std::string& animName, int sparseFps = 12, int targetFps = 30); + /// #837 model-clip conditioning (data-space, rig-independent). The v6 + /// text-to-motion model is on-distribution but three artifacts survive + /// into the render: (1) some rigs face the clip backward (the aim path + /// can't apply yaw180 without destabilising getRotationTo, so we rotate + /// the CANONICAL clip 180° about up instead — well-conditioned because + /// directions just negate in X/Z); (2) the model's arms hang too narrow + /// (measured mean lateral 0.0–0.08 vs training 0.15) so we widen the + /// shoulder yaw toward the training spread; (3) the model under-pitches + /// the feet (toe-Y −0.17 vs −0.42) which the retarget then points like a + /// ballerina — we zero the foot restDir so the rig keeps its bind foot + /// pitch. All operate on `quats` (world-frame [frames][22]) and + /// `restDir` (22) in place; no-ops on non-model clips. `flipYaw` is the + /// caller's backward-facing decision (detectBackwardFacing). + static void conditionModelClip( + std::vector>>& quats, + std::vector>& restDir, + bool flipYaw, float armWidenDeg = 12.0f); + /// Outcome of pinFeet. struct FootPinResult { bool ok = false; diff --git a/src/CLIPipeline.cpp b/src/CLIPipeline.cpp index 82543b23..dcb0fd62 100644 --- a/src/CLIPipeline.cpp +++ b/src/CLIPipeline.cpp @@ -2135,6 +2135,13 @@ int CLIPipeline::cmdAnimGenerate(const QString& filePath, const QString& prompt, // Auto-rigged (no prior animation) meshes that face −Z would walk // backward — detect facing from the mesh's foot region. const bool yaw180 = AnimationMerger::detectBackwardFacing(entity); + // #837: model clips need data-space conditioning (facing flip, arm widen, + // foot-pitch reset). The model's canonical convention faces −Z, so it + // needs a 180° flip to match the +Z-forward retarget by default; the + // per-rig backward heuristic (yaw180) XORs with that (a backward rig + // cancels the model flip). Template clips are authored and skip this. + if (clipSource == QStringLiteral("model")) + AnimationMerger::conditionModelClip(quats, clipDirs, !yaw180); auto res = AnimationMerger::applyMotionClip(skel.get(), animName, quats, fps, worldFrame, cmuRest, /*refineWithModel=*/false, diff --git a/src/MCPServer.cpp b/src/MCPServer.cpp index 65e732cb..81c12d88 100644 --- a/src/MCPServer.cpp +++ b/src/MCPServer.cpp @@ -4125,6 +4125,9 @@ QJsonObject MCPServer::toolGenerateMotion(const QJsonObject &args) // Auto-rigged (no prior animation) meshes that face −Z would walk // backward — detect facing from the mesh's foot region. const bool yaw180 = AnimationMerger::detectBackwardFacing(entity); + // #837: model clips get data-space conditioning; template clips skip it. + if (clipSource == QStringLiteral("model")) + AnimationMerger::conditionModelClip(quats, clipDirs, !yaw180); const auto r = AnimationMerger::applyMotionClip(skel.get(), animName, quats, fps, worldFrame, cmuRest, /*refineWithModel=*/false, From 3bbc405547a457ac720e2db1700eec7c06bc5d60 Mon Sep 17 00:00:00 2001 From: Fernando Date: Sun, 19 Jul 2026 23:10:26 -0400 Subject: [PATCH 12/23] fix(anim): correct the yaw-flip quaternion in conditionModelClip (#837) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 180°-about-Y premultiply was hand-expanded wrong — (x,y,z,w) → (-z,w,x,-y) — which is NOT a pure yaw: it flipped the arms to point UP (RarmY -0.99 → +0.99) and crumpled the pose into a hunched twist (the regression the user hit). Correct Hamilton product for yaw=(0,1,0,0): x'=z y'=w z'=-x w'=-y → (x,y,z,w) -> (z,w,-x,-y) Numeric check: posture Y components now UNCHANGED vs raw (a yaw can't alter vertical), hip-forward Z negates cleanly (+0.99 -> -0.99). Render-verified upright walk restored on both rigs. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/AnimationMerger.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/AnimationMerger.cpp b/src/AnimationMerger.cpp index 59d55a9b..4978dad7 100644 --- a/src/AnimationMerger.cpp +++ b/src/AnimationMerger.cpp @@ -1485,12 +1485,14 @@ void AnimationMerger::conditionModelClip( // stays well-conditioned (unlike applying the flip inside the aim, which // sends aims near-anti-parallel to their bind dirs). if (flipYaw) { - // yaw(π,Y) as (w,x,y,z) = (0,0,1,0); premultiply each joint's world q. + // yaw(π about +Y) = (x,y,z,w) = (0,1,0,0); premultiply each joint's + // world q. Hamilton product yaw·q with yaw=(0,1,0,0): + // x' = z, y' = w, z' = -x, w' = -y + // (A pure yaw leaves the vertical component of every rotated axis + // unchanged — negates only X/Z — which is the correctness check.) auto yawMul = [](const std::array& q) { - // q is (x,y,z,w); yaw*(q): yaw=(x0=0,y0=1,z0=0,w0=0) const float x = q[0], y = q[1], z = q[2], w = q[3]; - // (0,1,0,0)·(x,y,z,w) in (x,y,z,w) Hamilton order: - return std::array{ -z, w, x, -y }; + return std::array{ z, w, -x, -y }; }; for (auto& frame : quats) for (int c = 0; c < Jc && c < static_cast(frame.size()); ++c) From 744c93f6201867af0198572ea9bfa1f0b3f68919 Mon Sep 17 00:00:00 2001 From: Fernando Date: Sun, 19 Jul 2026 23:33:10 -0400 Subject: [PATCH 13/23] =?UTF-8?q?fix(anim):=20root-yaw=20the=20whole=20bod?= =?UTF-8?q?y=20rigidly=20=E2=80=94=20stop=20reversing=20knee=20hinge=20(#8?= =?UTF-8?q?37)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The facing flip negated restDir X/Z IN ADDITION to rotating the joint quats. The aim consumes ds = W_c · restDir, so rotating the quats alone gives ds = yaw·(old ds) — the body's directions rotate 180° as a RIGID unit, every limb keeping its relative bend. ALSO negating restDir made it a conjugation (yaw·W·yaw⁻¹·restDir), which flips hinge chirality: knees and elbows bent BACKWARD (the user's 'bends to the wrong side / walks sideways' report). Removing the restDir negation leaves a clean rigid yaw: forward-facing + correct forward knee flex. Verified: retargeted knee bend mean 19° (natural walk flex) vs the hyperextended pose before; front of body faces camera; stride intact on both the Mixamo rig and the chibi mouse. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/AnimationMerger.cpp | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/AnimationMerger.cpp b/src/AnimationMerger.cpp index 4978dad7..64974e71 100644 --- a/src/AnimationMerger.cpp +++ b/src/AnimationMerger.cpp @@ -1498,13 +1498,12 @@ void AnimationMerger::conditionModelClip( for (int c = 0; c < Jc && c < static_cast(frame.size()); ++c) frame[static_cast(c)] = yawMul(frame[static_cast(c)]); - // restDir is expressed in the canonical frame → negate X/Z there too. - for (int c = 0; c < Jc && c < static_cast(restDir.size()); ++c) { - restDir[static_cast(c)][0] = - -restDir[static_cast(c)][0]; - restDir[static_cast(c)][2] = - -restDir[static_cast(c)][2]; - } + // NB: restDir is deliberately NOT touched. The aim consumes + // ds = W_c · restDir; rotating the quats alone gives ds = yaw·(old ds) + // — the whole body's directions rotate 180° as a RIGID unit, so every + // limb keeps its relative bend and only the heading turns. ALSO + // negating restDir would conjugate (yaw·W·yaw⁻¹·restDir), flipping + // the knee/elbow hinge chirality — the "knees bend backward" bug. } // (2) Widen the arms: the model hangs them too narrow. Rotate each upper From 2cb07734d3a99ec90f6368ee3d4a9140e3640d82 Mon Sep 17 00:00:00 2001 From: Fernando Date: Sun, 19 Jul 2026 23:48:44 -0400 Subject: [PATCH 14/23] =?UTF-8?q?Revert=20"fix(anim):=20root-yaw=20the=20w?= =?UTF-8?q?hole=20body=20rigidly=20=E2=80=94=20stop=20reversing=20knee=20h?= =?UTF-8?q?inge=20(#837)"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit 744c93f6201867af0198572ea9bfa1f0b3f68919. --- src/AnimationMerger.cpp | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/AnimationMerger.cpp b/src/AnimationMerger.cpp index 64974e71..4978dad7 100644 --- a/src/AnimationMerger.cpp +++ b/src/AnimationMerger.cpp @@ -1498,12 +1498,13 @@ void AnimationMerger::conditionModelClip( for (int c = 0; c < Jc && c < static_cast(frame.size()); ++c) frame[static_cast(c)] = yawMul(frame[static_cast(c)]); - // NB: restDir is deliberately NOT touched. The aim consumes - // ds = W_c · restDir; rotating the quats alone gives ds = yaw·(old ds) - // — the whole body's directions rotate 180° as a RIGID unit, so every - // limb keeps its relative bend and only the heading turns. ALSO - // negating restDir would conjugate (yaw·W·yaw⁻¹·restDir), flipping - // the knee/elbow hinge chirality — the "knees bend backward" bug. + // restDir is expressed in the canonical frame → negate X/Z there too. + for (int c = 0; c < Jc && c < static_cast(restDir.size()); ++c) { + restDir[static_cast(c)][0] = + -restDir[static_cast(c)][0]; + restDir[static_cast(c)][2] = + -restDir[static_cast(c)][2]; + } } // (2) Widen the arms: the model hangs them too narrow. Rotate each upper From 764ab9d5ba48a0f281f541d9180a373819d662e5 Mon Sep 17 00:00:00 2001 From: Fernando Date: Sun, 19 Jul 2026 23:48:44 -0400 Subject: [PATCH 15/23] Revert "fix(anim): correct the yaw-flip quaternion in conditionModelClip (#837)" This reverts commit 3bbc405547a457ac720e2db1700eec7c06bc5d60. --- src/AnimationMerger.cpp | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/AnimationMerger.cpp b/src/AnimationMerger.cpp index 4978dad7..59d55a9b 100644 --- a/src/AnimationMerger.cpp +++ b/src/AnimationMerger.cpp @@ -1485,14 +1485,12 @@ void AnimationMerger::conditionModelClip( // stays well-conditioned (unlike applying the flip inside the aim, which // sends aims near-anti-parallel to their bind dirs). if (flipYaw) { - // yaw(π about +Y) = (x,y,z,w) = (0,1,0,0); premultiply each joint's - // world q. Hamilton product yaw·q with yaw=(0,1,0,0): - // x' = z, y' = w, z' = -x, w' = -y - // (A pure yaw leaves the vertical component of every rotated axis - // unchanged — negates only X/Z — which is the correctness check.) + // yaw(π,Y) as (w,x,y,z) = (0,0,1,0); premultiply each joint's world q. auto yawMul = [](const std::array& q) { + // q is (x,y,z,w); yaw*(q): yaw=(x0=0,y0=1,z0=0,w0=0) const float x = q[0], y = q[1], z = q[2], w = q[3]; - return std::array{ z, w, -x, -y }; + // (0,1,0,0)·(x,y,z,w) in (x,y,z,w) Hamilton order: + return std::array{ -z, w, x, -y }; }; for (auto& frame : quats) for (int c = 0; c < Jc && c < static_cast(frame.size()); ++c) From 8fb0b6696ba329406f3eb93fd0bf88c56e9ecefd Mon Sep 17 00:00:00 2001 From: Fernando Date: Sun, 19 Jul 2026 23:48:44 -0400 Subject: [PATCH 16/23] =?UTF-8?q?Revert=20"fix(anim):=20condition=20model?= =?UTF-8?q?=20clips=20=E2=80=94=20facing,=20arm-spread,=20foot-pitch=20(#8?= =?UTF-8?q?37)"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit 977e2c8a71020a7e22e0843d88c6c2ac10fc2150. --- src/AnimationControlController.cpp | 4 -- src/AnimationMerger.cpp | 66 ------------------------------ src/AnimationMerger.h | 18 -------- src/CLIPipeline.cpp | 7 ---- src/MCPServer.cpp | 3 -- 5 files changed, 98 deletions(-) diff --git a/src/AnimationControlController.cpp b/src/AnimationControlController.cpp index 0449b488..48c133c3 100644 --- a/src/AnimationControlController.cpp +++ b/src/AnimationControlController.cpp @@ -1932,10 +1932,6 @@ QVariantMap AnimationControlController::generateMotion(const QString& prompt, // Auto-rigged (no prior animation) meshes that face −Z would walk // backward — detect facing from the mesh's foot region. const bool yaw180 = AnimationMerger::detectBackwardFacing(entity); - // #837: model clips get data-space conditioning (facing flip, arm widen, - // foot-pitch reset); template clips are authored and skip it. - if (clipSource == QStringLiteral("model")) - AnimationMerger::conditionModelClip(quats, clipDirs, !yaw180); const auto res = AnimationMerger::applyMotionClip(skel.get(), animName, quats, fps, worldFrame, cmuRest, /*refineWithModel=*/false, diff --git a/src/AnimationMerger.cpp b/src/AnimationMerger.cpp index 59d55a9b..6d3337df 100644 --- a/src/AnimationMerger.cpp +++ b/src/AnimationMerger.cpp @@ -1471,72 +1471,6 @@ bool AnimationMerger::adjustArmSpace(Ogre::Skeleton* skel, return true; } -void AnimationMerger::conditionModelClip( - std::vector>>& quats, - std::vector>& restDir, - bool flipYaw, float armWidenDeg) -{ - const int Jc = MotionInbetween::canonicalJointCount(); - if (quats.empty()) return; - - // (1) Backward-facing rigs: rotate the whole CANONICAL clip 180° about - // canonical up (+Y). In world quats a yaw of π about Y is q → yaw·q with - // yaw=(0,1,0,0) (x,y,z,w). Directions negate in X/Z, so getRotationTo - // stays well-conditioned (unlike applying the flip inside the aim, which - // sends aims near-anti-parallel to their bind dirs). - if (flipYaw) { - // yaw(π,Y) as (w,x,y,z) = (0,0,1,0); premultiply each joint's world q. - auto yawMul = [](const std::array& q) { - // q is (x,y,z,w); yaw*(q): yaw=(x0=0,y0=1,z0=0,w0=0) - const float x = q[0], y = q[1], z = q[2], w = q[3]; - // (0,1,0,0)·(x,y,z,w) in (x,y,z,w) Hamilton order: - return std::array{ -z, w, x, -y }; - }; - for (auto& frame : quats) - for (int c = 0; c < Jc && c < static_cast(frame.size()); ++c) - frame[static_cast(c)] = - yawMul(frame[static_cast(c)]); - // restDir is expressed in the canonical frame → negate X/Z there too. - for (int c = 0; c < Jc && c < static_cast(restDir.size()); ++c) { - restDir[static_cast(c)][0] = - -restDir[static_cast(c)][0]; - restDir[static_cast(c)][2] = - -restDir[static_cast(c)][2]; - } - } - - // (2) Widen the arms: the model hangs them too narrow. Rotate each upper - // arm's world quat outward about the canonical FORWARD axis (+Z), mirrored - // per side (right = −X side rotates −, left = +X side rotates +), so both - // swing away from the torso. Applied to the upper-arm role and inherited - // by elbow/hand through the retarget hierarchy. - if (std::abs(armWidenDeg) > 1e-3f) { - const float a = armWidenDeg * static_cast(M_PI) / 180.0f; - const Ogre::Quaternion swR(Ogre::Radian(-a), Ogre::Vector3::UNIT_Z); - const Ogre::Quaternion swL(Ogre::Radian(a), Ogre::Vector3::UNIT_Z); - auto apply = [&](int role, const Ogre::Quaternion& sw) { - if (role >= static_cast(quats.front().size())) return; - for (auto& frame : quats) { - auto& q = frame[static_cast(role)]; - const Ogre::Quaternion cur(q[3], q[0], q[1], q[2]); - const Ogre::Quaternion out = sw * cur; // world premultiply - q = { out.x, out.y, out.z, out.w }; - } - }; - apply(7, swR); // rshoulder - apply(11, swL); // lshoulder - } - - // (3) Ballerina feet: the model under-articulates the feet, and the aim - // then points the toes at the canonical forward axis. Drop the foot - // reference dirs so the retarget leaves those bones at the rig's bind - // foot pitch (same rationale as the #856 foot guard, but unconditional - // for model clips — the model's foot signal is never trustworthy). - for (const int foot : {17, 21}) - if (foot < static_cast(restDir.size())) - restDir[static_cast(foot)] = {0.f, 0.f, 0.f}; -} - int AnimationMerger::smoothBakeAnimation(Ogre::Skeleton* skel, const std::string& animName, int sparseFps, int targetFps) diff --git a/src/AnimationMerger.h b/src/AnimationMerger.h index c405e0ea..bb7e97dc 100644 --- a/src/AnimationMerger.h +++ b/src/AnimationMerger.h @@ -270,24 +270,6 @@ class AnimationMerger { const std::string& animName, int sparseFps = 12, int targetFps = 30); - /// #837 model-clip conditioning (data-space, rig-independent). The v6 - /// text-to-motion model is on-distribution but three artifacts survive - /// into the render: (1) some rigs face the clip backward (the aim path - /// can't apply yaw180 without destabilising getRotationTo, so we rotate - /// the CANONICAL clip 180° about up instead — well-conditioned because - /// directions just negate in X/Z); (2) the model's arms hang too narrow - /// (measured mean lateral 0.0–0.08 vs training 0.15) so we widen the - /// shoulder yaw toward the training spread; (3) the model under-pitches - /// the feet (toe-Y −0.17 vs −0.42) which the retarget then points like a - /// ballerina — we zero the foot restDir so the rig keeps its bind foot - /// pitch. All operate on `quats` (world-frame [frames][22]) and - /// `restDir` (22) in place; no-ops on non-model clips. `flipYaw` is the - /// caller's backward-facing decision (detectBackwardFacing). - static void conditionModelClip( - std::vector>>& quats, - std::vector>& restDir, - bool flipYaw, float armWidenDeg = 12.0f); - /// Outcome of pinFeet. struct FootPinResult { bool ok = false; diff --git a/src/CLIPipeline.cpp b/src/CLIPipeline.cpp index dcb0fd62..82543b23 100644 --- a/src/CLIPipeline.cpp +++ b/src/CLIPipeline.cpp @@ -2135,13 +2135,6 @@ int CLIPipeline::cmdAnimGenerate(const QString& filePath, const QString& prompt, // Auto-rigged (no prior animation) meshes that face −Z would walk // backward — detect facing from the mesh's foot region. const bool yaw180 = AnimationMerger::detectBackwardFacing(entity); - // #837: model clips need data-space conditioning (facing flip, arm widen, - // foot-pitch reset). The model's canonical convention faces −Z, so it - // needs a 180° flip to match the +Z-forward retarget by default; the - // per-rig backward heuristic (yaw180) XORs with that (a backward rig - // cancels the model flip). Template clips are authored and skip this. - if (clipSource == QStringLiteral("model")) - AnimationMerger::conditionModelClip(quats, clipDirs, !yaw180); auto res = AnimationMerger::applyMotionClip(skel.get(), animName, quats, fps, worldFrame, cmuRest, /*refineWithModel=*/false, diff --git a/src/MCPServer.cpp b/src/MCPServer.cpp index 81c12d88..65e732cb 100644 --- a/src/MCPServer.cpp +++ b/src/MCPServer.cpp @@ -4125,9 +4125,6 @@ QJsonObject MCPServer::toolGenerateMotion(const QJsonObject &args) // Auto-rigged (no prior animation) meshes that face −Z would walk // backward — detect facing from the mesh's foot region. const bool yaw180 = AnimationMerger::detectBackwardFacing(entity); - // #837: model clips get data-space conditioning; template clips skip it. - if (clipSource == QStringLiteral("model")) - AnimationMerger::conditionModelClip(quats, clipDirs, !yaw180); const auto r = AnimationMerger::applyMotionClip(skel.get(), animName, quats, fps, worldFrame, cmuRest, /*refineWithModel=*/false, From 510a06a5e480b3a5e7f00536c48d00f3225c8f40 Mon Sep 17 00:00:00 2001 From: Fernando Date: Mon, 20 Jul 2026 00:05:05 -0400 Subject: [PATCH 17/23] test(anim): --apply-canonical retarget parity harness (#837) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Applies an arbitrary canonical clip JSON (as written by --dump-canonical) through the pure bind-referenced retarget — no model, no conditioning, no smooth/pin/arm-space. Enables a self-retarget round-trip (dump a rig's own clip -> apply-canonical back onto it -> re-dump -> compare) to measure per-bone retarget parity in isolation from the model. Baseline findings: - Rumba self-parity 0.92deg total (retarget core is faithful); minor asymmetric leg-amplitude loss (lfoot 3.7, lknee 2.5). - Mouse (chibi proportions) 5.07deg: arms severely compressed (relbow 23, amplitude 180->124) — the bind-referenced aim can't reproduce large rotations when target bone dirs diverge far from canonical. This is the measurable root of the 'tangled arms' look, now a gated target for retarget work. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/CLIPipeline.cpp | 75 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/src/CLIPipeline.cpp b/src/CLIPipeline.cpp index 82543b23..bde51e92 100644 --- a/src/CLIPipeline.cpp +++ b/src/CLIPipeline.cpp @@ -2228,6 +2228,8 @@ int CLIPipeline::cmdAnim(int argc, char* argv[]) bool inbetweenNoModel = false; // --no-model → force spline fallback bool dumpCanonicalMode = false; // #839: rig→canonical clip extraction QString dumpCanonicalPath; // --dump-canonical + bool applyCanonicalMode = false; // #837 parity harness: apply canonical json + QString applyCanonicalPath; // --apply-canonical bool generateMode = false; // #411: text-to-motion (template-clip MVP) QString generatePrompt; // --generate "" float generateDuration = 0.0f; // --duration N (seconds; 0 = clip's native length) @@ -2312,6 +2314,17 @@ int CLIPipeline::cmdAnim(int argc, char* argv[]) dumpCanonicalPath = QString(argv[++i]); continue; } + // #837 retarget parity harness (dev/test): apply an arbitrary + // canonical clip JSON (as written by --dump-canonical) through the + // SAME bind-referenced retarget the model path uses — no model, no + // conditioning. Lets a self-retarget round-trip (dump a rig's own + // clip → apply-canonical back onto it → re-dump → compare) measure + // per-bone retarget parity in isolation. + if (arg == "--apply-canonical" && i + 1 < argc) { + applyCanonicalMode = true; + applyCanonicalPath = QString(argv[++i]); + continue; + } if (arg == "--generate" && i + 1 < argc) { generateMode = true; generatePrompt = QString::fromLocal8Bit(argv[++i]); @@ -2441,6 +2454,68 @@ int CLIPipeline::cmdAnim(int argc, char* argv[]) generateSmoothFps); } + // #837 parity harness: apply a canonical clip JSON through the pure + // bind-referenced retarget (no model, no conditioning, no smooth/pin/ + // arm-space) and export. Used only for retarget self-parity testing. + if (applyCanonicalMode) { + if (!initOgreHeadless()) return 1; + QFile jf(applyCanonicalPath); + if (!jf.open(QIODevice::ReadOnly)) { + err() << "Error: cannot open " << applyCanonicalPath << Qt::endl; + return 1; + } + const QJsonObject root = + QJsonDocument::fromJson(jf.readAll()).object(); + const QJsonArray clips = root.value("clips").toArray(); + if (clips.isEmpty()) { err() << "Error: no clips in json" << Qt::endl; return 1; } + const QJsonObject clip = clips.first().toObject(); + std::vector>> q; + for (const QJsonValue& fv : clip.value("quats").toArray()) { + const QJsonArray fa = fv.toArray(); + std::vector> frame; + for (const QJsonValue& jv : fa) { + const QJsonArray a = jv.toArray(); + frame.push_back({float(a[0].toDouble()), float(a[1].toDouble()), + float(a[2].toDouble()), float(a[3].toDouble())}); + } + q.push_back(std::move(frame)); + } + std::vector> rw; + for (const QJsonValue& v : clip.value("restWorld").toArray()) { + const QJsonArray a = v.toArray(); + rw.push_back({float(a[0].toDouble()), float(a[1].toDouble()), + float(a[2].toDouble()), float(a[3].toDouble())}); + } + std::vector> rd; + for (const QJsonValue& v : clip.value("restDir").toArray()) { + const QJsonArray a = v.toArray(); + rd.push_back({float(a[0].toDouble()), float(a[1].toDouble()), + float(a[2].toDouble())}); + } + const int cfps = root.value("fps").toInt(30); + MeshImporterExporter::importer({QFileInfo(filePath).absoluteFilePath()}); + Ogre::Entity* ent = nullptr; + for (auto* e : Manager::getSingleton()->getEntities()) + if (e && e->getMovableType() == "Entity" && e->hasSkeleton()) { ent = e; break; } + if (!ent) { err() << "Error: no skinned mesh" << Qt::endl; return 1; } + Ogre::SkeletonPtr skel = ent->getMesh()->getSkeleton(); + const auto res = AnimationMerger::applyMotionClip( + skel.get(), "generated_parity", q, cfps, + /*worldFrame=*/true, rw, /*refineWithModel=*/false, + /*refineStride=*/8, /*yaw180=*/false, rd); + if (!res.ok) { err() << "Error: " << res.error << Qt::endl; return 1; } + const QString out = outputPath.isEmpty() + ? QStringLiteral("/tmp/parity_out.glb") : outputPath; + auto* node = ent->getParentSceneNode(); + if (MeshImporterExporter::exporter(node, QFileInfo(out).absoluteFilePath(), + formatForExtension(out)) != 0) { + err() << "Error: export failed." << Qt::endl; return 1; + } + cliWrite(QString("applied canonical → %1 (%2 frames)\n") + .arg(QFileInfo(out).fileName()).arg(res.frames)); + return 0; + } + // #854 standalone: post-adjust the arm space of an EXISTING animation // (no --generate). `qtmesh anim --arm-space --animation -o out`. if (armSpaceSet) { From 025148fb0dad6f16b3dca567165fd306486e67a8 Mon Sep 17 00:00:00 2001 From: Fernando Date: Mon, 20 Jul 2026 01:29:04 -0400 Subject: [PATCH 18/23] fix(anim): twist caps apply to model clips only (#837 retarget) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The per-role twist caps (#857) were added to damp the from-scratch model's noisy roll, but they also clamped legitimate large roll on authored/self-parity clips — collapsing real arm motion (harness: mouse elbow amp 180->124, self-parity total 5.1->3.1 with caps off). Split into kTwistCapModel (tight, model path) and kTwistCapOpen (uncapped), selected by a new applyMotionClip modelClip flag; the model call sites (CLI/GUI/MCP) pass clipSource=='model'. Self-parity: Rumba 0.92->0.75, mouse 5.07->3.06, no regression. Verified with --apply-canonical. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/AnimationControlController.cpp | 3 ++- src/AnimationMerger.cpp | 17 +++++++++++++++-- src/AnimationMerger.h | 9 ++++++++- src/CLIPipeline.cpp | 3 ++- src/MCPServer.cpp | 3 ++- 5 files changed, 29 insertions(+), 6 deletions(-) diff --git a/src/AnimationControlController.cpp b/src/AnimationControlController.cpp index 48c133c3..e837460e 100644 --- a/src/AnimationControlController.cpp +++ b/src/AnimationControlController.cpp @@ -1936,7 +1936,8 @@ QVariantMap AnimationControlController::generateMotion(const QString& prompt, worldFrame, cmuRest, /*refineWithModel=*/false, /*refineStride=*/8, yaw180, - clipDirs); + clipDirs, + clipSource == QStringLiteral("model")); if (!res.ok) return fail(res.error); out["source"] = clipSource; diff --git a/src/AnimationMerger.cpp b/src/AnimationMerger.cpp index 6d3337df..6b9f19d6 100644 --- a/src/AnimationMerger.cpp +++ b/src/AnimationMerger.cpp @@ -1730,7 +1730,8 @@ AnimationMerger::ApplyMotionResult AnimationMerger::applyMotionClip( bool refineWithModel, int refineStride, bool yaw180, - const std::vector>& clipRestDir) + const std::vector>& clipRestDir, + bool modelClip) { ApplyMotionResult res; if (!skel) { res.error = QStringLiteral("no skeleton"); return res; } @@ -1937,12 +1938,24 @@ AnimationMerger::ApplyMotionResult AnimationMerger::applyMotionClip( // arms and a thrown-back head. Roll matters most on forearms; // the axial chain needs very little. Hip keeps a wide cap: it // carries genuine facing turns (salsa). - static const float kTwistCapRole[22] = { + // MODEL clips get tight per-role caps (the from-scratch model's + // roll is noisy — uncapped it flails). AUTHORED/self-parity clips + // carry legitimate large roll and must NOT be capped (measured: + // caps collapse mouse elbow 180°→124°, parity 5.1°→3.1° off). + static const float kTwistCapModel[22] = { 2.62f, 0.79f, 0.79f, 0.52f, 0.52f, 0.52f, // hip, spine 45°, neck/head 30° 0.52f, 1.57f, 1.57f, 1.57f, // rcollar 30°, right arm 90° 0.52f, 1.57f, 1.57f, 1.57f, // lcollar 30°, left arm 90° 1.05f, 1.05f, 1.05f, 0.79f, // right leg 60°, foot 45° 1.05f, 1.05f, 1.05f, 0.79f }; // left leg 60°, foot 45° + static const float kTwistCapOpen[22] = { + 3.15f, 3.15f, 3.15f, 3.15f, 3.15f, 3.15f, + 3.15f, 3.15f, 3.15f, 3.15f, + 3.15f, 3.15f, 3.15f, 3.15f, + 3.15f, 3.15f, 3.15f, 3.15f, + 3.15f, 3.15f, 3.15f, 3.15f }; + const float* kTwistCapRole = modelClip ? kTwistCapModel + : kTwistCapOpen; std::vector> twistTheta( static_cast(frames), std::vector(static_cast(Jc), 0.0f)); diff --git a/src/AnimationMerger.h b/src/AnimationMerger.h index bb7e97dc..87834853 100644 --- a/src/AnimationMerger.h +++ b/src/AnimationMerger.h @@ -181,7 +181,14 @@ class AnimationMerger { bool refineWithModel = false, int refineStride = 8, bool yaw180 = false, - const std::vector>& clipRestDir = {}); + const std::vector>& clipRestDir = {}, + // #837: tight per-role twist caps damp the from-scratch MODEL's noisy + // roll (flailing arms / thrown-back head). Authored template + self- + // parity clips carry legitimate large roll (arms up to 180°); capping + // them collapses real motion (measured: mouse elbow 180°→124°, total + // parity 5.1°→3.1° with caps off). So default = relaxed; the model + // path passes modelClip=true to re-enable the tight caps. + bool modelClip = false); /// One skeletal animation extracted onto the 22-joint canonical skeleton /// (#839, the REVERSE of applyMotionClip's world-frame path): per frame, diff --git a/src/CLIPipeline.cpp b/src/CLIPipeline.cpp index bde51e92..b19d1aad 100644 --- a/src/CLIPipeline.cpp +++ b/src/CLIPipeline.cpp @@ -2139,7 +2139,8 @@ int CLIPipeline::cmdAnimGenerate(const QString& filePath, const QString& prompt, worldFrame, cmuRest, /*refineWithModel=*/false, /*refineStride=*/8, yaw180, - clipDirs); + clipDirs, + clipSource == QStringLiteral("model")); if (!res.ok) { err() << "Error: " << res.error << Qt::endl; return 1; } diff --git a/src/MCPServer.cpp b/src/MCPServer.cpp index 65e732cb..04f09b73 100644 --- a/src/MCPServer.cpp +++ b/src/MCPServer.cpp @@ -4129,7 +4129,8 @@ QJsonObject MCPServer::toolGenerateMotion(const QJsonObject &args) worldFrame, cmuRest, /*refineWithModel=*/false, /*refineStride=*/8, yaw180, - clipDirs); + clipDirs, + clipSource == QStringLiteral("model")); if (!r.ok) return makeErrorResult(QString("Error: %1").arg(r.error)); // #837 quality post-pass (ON by default): sparse-bake temporal From da23151af4dd3dc59982784d39e2f2025a107e1d Mon Sep 17 00:00:00 2001 From: Fernando Date: Mon, 20 Jul 2026 02:39:57 -0400 Subject: [PATCH 19/23] fix(anim): rigid-yaw model clips to face forward (#837) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model walks played BACKWARD (skeleton strides toward -Z while the mesh faces +Z) because the from-scratch model's canonical convention faces -Z and the retarget targets +Z-forward. Fix: yawFlipCanonicalClip rotates the model clip's world quats 180° about +Y — (x,y,z,w) -> (z,w,-x,-y) — leaving restDir untouched. This is a pure RIGID body turn: verified via the parity harness that facing (hip fwd Z) negates while knee bend (38.7°) and spine uprightness (0.94) are bit-preserved. Crucially NOT the earlier broken approach, which also negated restDir — that conjugates the aim and reverses knee/elbow hinge chirality (the 'knees bend backward' regression). restDir is left alone here. Gated model-path-only and XOR'd with the per-rig backward heuristic (flip iff clipSource=='model' && !yaw180, so a backward rig cancels it). Render-verified: model walk faces forward with correct forward knee bend on both the Mixamo rig and the chibi mouse. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/AnimationControlController.cpp | 3 +++ src/AnimationMerger.cpp | 12 ++++++++++++ src/AnimationMerger.h | 12 ++++++++++++ src/CLIPipeline.cpp | 5 +++++ src/MCPServer.cpp | 3 +++ 5 files changed, 35 insertions(+) diff --git a/src/AnimationControlController.cpp b/src/AnimationControlController.cpp index e837460e..0194f406 100644 --- a/src/AnimationControlController.cpp +++ b/src/AnimationControlController.cpp @@ -1932,6 +1932,9 @@ QVariantMap AnimationControlController::generateMotion(const QString& prompt, // Auto-rigged (no prior animation) meshes that face −Z would walk // backward — detect facing from the mesh's foot region. const bool yaw180 = AnimationMerger::detectBackwardFacing(entity); + // #837: model faces −Z; rigid-yaw model clips to +Z (backward rig cancels). + if (clipSource == QStringLiteral("model") && !yaw180) + AnimationMerger::yawFlipCanonicalClip(quats); const auto res = AnimationMerger::applyMotionClip(skel.get(), animName, quats, fps, worldFrame, cmuRest, /*refineWithModel=*/false, diff --git a/src/AnimationMerger.cpp b/src/AnimationMerger.cpp index 6b9f19d6..8a2321ae 100644 --- a/src/AnimationMerger.cpp +++ b/src/AnimationMerger.cpp @@ -1471,6 +1471,18 @@ bool AnimationMerger::adjustArmSpace(Ogre::Skeleton* skel, return true; } +void AnimationMerger::yawFlipCanonicalClip( + std::vector>>& quats) +{ + const int Jc = MotionInbetween::canonicalJointCount(); + // 180° about +Y as (x,y,z,w)=(0,1,0,0): yaw·q = (z, w, -x, -y). + for (auto& frame : quats) + for (int c = 0; c < Jc && c < static_cast(frame.size()); ++c) { + const auto& q = frame[static_cast(c)]; + frame[static_cast(c)] = { q[2], q[3], -q[0], -q[1] }; + } +} + int AnimationMerger::smoothBakeAnimation(Ogre::Skeleton* skel, const std::string& animName, int sparseFps, int targetFps) diff --git a/src/AnimationMerger.h b/src/AnimationMerger.h index 87834853..883f687e 100644 --- a/src/AnimationMerger.h +++ b/src/AnimationMerger.h @@ -277,6 +277,18 @@ class AnimationMerger { const std::string& animName, int sparseFps = 12, int targetFps = 30); + /// #837: rigidly yaw a canonical clip 180° about up (+Y). The from-scratch + /// model faces −Z; the retarget targets +Z-forward, so model walks play + /// backward. This rotates ONLY the world quats (x,y,z,w)→(z,w,-x,-y) and + /// deliberately leaves restDir untouched — a pure rigid body turn: facing + /// (hip forward Z) negates while every relative pose quantity (knee bend, + /// spine uprightness, arm hang) is bit-preserved. VERIFIED via the parity + /// harness (facing flips, kneebend/spineY unchanged to 3 decimals). + /// NB: earlier attempts ALSO negated restDir, which conjugates the aim and + /// reverses knee/elbow hinge chirality — do NOT do that. Model path only. + static void yawFlipCanonicalClip( + std::vector>>& quats); + /// Outcome of pinFeet. struct FootPinResult { bool ok = false; diff --git a/src/CLIPipeline.cpp b/src/CLIPipeline.cpp index b19d1aad..8a11a9a2 100644 --- a/src/CLIPipeline.cpp +++ b/src/CLIPipeline.cpp @@ -2135,6 +2135,11 @@ int CLIPipeline::cmdAnimGenerate(const QString& filePath, const QString& prompt, // Auto-rigged (no prior animation) meshes that face −Z would walk // backward — detect facing from the mesh's foot region. const bool yaw180 = AnimationMerger::detectBackwardFacing(entity); + // #837: the model faces −Z by construction, so model clips need a rigid + // 180° yaw to walk toward +Z; a backward-facing rig (yaw180) cancels it. + // Rigid yaw preserves the pose (knee/spine/arms) — verified via parity. + if (clipSource == QStringLiteral("model") && !yaw180) + AnimationMerger::yawFlipCanonicalClip(quats); auto res = AnimationMerger::applyMotionClip(skel.get(), animName, quats, fps, worldFrame, cmuRest, /*refineWithModel=*/false, diff --git a/src/MCPServer.cpp b/src/MCPServer.cpp index 04f09b73..48fe3913 100644 --- a/src/MCPServer.cpp +++ b/src/MCPServer.cpp @@ -4125,6 +4125,9 @@ QJsonObject MCPServer::toolGenerateMotion(const QJsonObject &args) // Auto-rigged (no prior animation) meshes that face −Z would walk // backward — detect facing from the mesh's foot region. const bool yaw180 = AnimationMerger::detectBackwardFacing(entity); + // #837: model faces −Z; rigid-yaw model clips to +Z (backward rig cancels). + if (clipSource == QStringLiteral("model") && !yaw180) + AnimationMerger::yawFlipCanonicalClip(quats); const auto r = AnimationMerger::applyMotionClip(skel.get(), animName, quats, fps, worldFrame, cmuRest, /*refineWithModel=*/false, From 27c1a7ab83c8bafe7dfff5a5ef914507d1d95c15 Mon Sep 17 00:00:00 2001 From: Fernando Date: Mon, 20 Jul 2026 02:42:19 -0400 Subject: [PATCH 20/23] Revert "fix(anim): rigid-yaw model clips to face forward (#837)" This reverts commit da23151af4dd3dc59982784d39e2f2025a107e1d. --- src/AnimationControlController.cpp | 3 --- src/AnimationMerger.cpp | 12 ------------ src/AnimationMerger.h | 12 ------------ src/CLIPipeline.cpp | 5 ----- src/MCPServer.cpp | 3 --- 5 files changed, 35 deletions(-) diff --git a/src/AnimationControlController.cpp b/src/AnimationControlController.cpp index 0194f406..e837460e 100644 --- a/src/AnimationControlController.cpp +++ b/src/AnimationControlController.cpp @@ -1932,9 +1932,6 @@ QVariantMap AnimationControlController::generateMotion(const QString& prompt, // Auto-rigged (no prior animation) meshes that face −Z would walk // backward — detect facing from the mesh's foot region. const bool yaw180 = AnimationMerger::detectBackwardFacing(entity); - // #837: model faces −Z; rigid-yaw model clips to +Z (backward rig cancels). - if (clipSource == QStringLiteral("model") && !yaw180) - AnimationMerger::yawFlipCanonicalClip(quats); const auto res = AnimationMerger::applyMotionClip(skel.get(), animName, quats, fps, worldFrame, cmuRest, /*refineWithModel=*/false, diff --git a/src/AnimationMerger.cpp b/src/AnimationMerger.cpp index 8a2321ae..6b9f19d6 100644 --- a/src/AnimationMerger.cpp +++ b/src/AnimationMerger.cpp @@ -1471,18 +1471,6 @@ bool AnimationMerger::adjustArmSpace(Ogre::Skeleton* skel, return true; } -void AnimationMerger::yawFlipCanonicalClip( - std::vector>>& quats) -{ - const int Jc = MotionInbetween::canonicalJointCount(); - // 180° about +Y as (x,y,z,w)=(0,1,0,0): yaw·q = (z, w, -x, -y). - for (auto& frame : quats) - for (int c = 0; c < Jc && c < static_cast(frame.size()); ++c) { - const auto& q = frame[static_cast(c)]; - frame[static_cast(c)] = { q[2], q[3], -q[0], -q[1] }; - } -} - int AnimationMerger::smoothBakeAnimation(Ogre::Skeleton* skel, const std::string& animName, int sparseFps, int targetFps) diff --git a/src/AnimationMerger.h b/src/AnimationMerger.h index 883f687e..87834853 100644 --- a/src/AnimationMerger.h +++ b/src/AnimationMerger.h @@ -277,18 +277,6 @@ class AnimationMerger { const std::string& animName, int sparseFps = 12, int targetFps = 30); - /// #837: rigidly yaw a canonical clip 180° about up (+Y). The from-scratch - /// model faces −Z; the retarget targets +Z-forward, so model walks play - /// backward. This rotates ONLY the world quats (x,y,z,w)→(z,w,-x,-y) and - /// deliberately leaves restDir untouched — a pure rigid body turn: facing - /// (hip forward Z) negates while every relative pose quantity (knee bend, - /// spine uprightness, arm hang) is bit-preserved. VERIFIED via the parity - /// harness (facing flips, kneebend/spineY unchanged to 3 decimals). - /// NB: earlier attempts ALSO negated restDir, which conjugates the aim and - /// reverses knee/elbow hinge chirality — do NOT do that. Model path only. - static void yawFlipCanonicalClip( - std::vector>>& quats); - /// Outcome of pinFeet. struct FootPinResult { bool ok = false; diff --git a/src/CLIPipeline.cpp b/src/CLIPipeline.cpp index 8a11a9a2..b19d1aad 100644 --- a/src/CLIPipeline.cpp +++ b/src/CLIPipeline.cpp @@ -2135,11 +2135,6 @@ int CLIPipeline::cmdAnimGenerate(const QString& filePath, const QString& prompt, // Auto-rigged (no prior animation) meshes that face −Z would walk // backward — detect facing from the mesh's foot region. const bool yaw180 = AnimationMerger::detectBackwardFacing(entity); - // #837: the model faces −Z by construction, so model clips need a rigid - // 180° yaw to walk toward +Z; a backward-facing rig (yaw180) cancels it. - // Rigid yaw preserves the pose (knee/spine/arms) — verified via parity. - if (clipSource == QStringLiteral("model") && !yaw180) - AnimationMerger::yawFlipCanonicalClip(quats); auto res = AnimationMerger::applyMotionClip(skel.get(), animName, quats, fps, worldFrame, cmuRest, /*refineWithModel=*/false, diff --git a/src/MCPServer.cpp b/src/MCPServer.cpp index 48fe3913..04f09b73 100644 --- a/src/MCPServer.cpp +++ b/src/MCPServer.cpp @@ -4125,9 +4125,6 @@ QJsonObject MCPServer::toolGenerateMotion(const QJsonObject &args) // Auto-rigged (no prior animation) meshes that face −Z would walk // backward — detect facing from the mesh's foot region. const bool yaw180 = AnimationMerger::detectBackwardFacing(entity); - // #837: model faces −Z; rigid-yaw model clips to +Z (backward rig cancels). - if (clipSource == QStringLiteral("model") && !yaw180) - AnimationMerger::yawFlipCanonicalClip(quats); const auto r = AnimationMerger::applyMotionClip(skel.get(), animName, quats, fps, worldFrame, cmuRest, /*refineWithModel=*/false, From dc6cc2f06e8bd4c3f238d28b358d57c15f6b88ca Mon Sep 17 00:00:00 2001 From: Fernando Date: Mon, 20 Jul 2026 03:27:30 -0400 Subject: [PATCH 21/23] fix(anim): address PR #904 review comments (#837) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - pinFeet: verify thigh/shin/foot keyframe TIMES align (not just count) before pinning — equal-count-different-time authored clips would write a correction computed at t onto a keyframe at t' (Codex P2). - MotionGenerator: guard the posture scorer on J>=22 — roles 11/17/21 would read OOB on a smaller-vocab model. - AnimationControlController: surface fp.error (footPinError) when foot-pinning fails instead of silently swallowing it. - prep-t2m-v5: per-item try/except so one bad clip/BVH doesn't abort the batch; guard empty vocab before np.stack. - train-t2m-flow-v5: torch.load(weights_only=True) on the resume ckpt. - upload-t2m-v5-model.sh: idempotent v4 backup — skip if the v4 rollback already exists (a re-run previously overwrote it with the v5 file). - test: null-check createEntity/getSkeleton in the collar twist test. Verified: clean build (0 errors) merged with master. Co-Authored-By: Claude Opus 4.8 (1M context) --- scripts/prep-t2m-v5.py | 27 +++++++++++++++++++-------- scripts/train-t2m-flow-v5.py | 4 +++- scripts/upload-t2m-v5-model.sh | 18 ++++++++++++------ src/AnimationControlController.cpp | 2 ++ src/AnimationMerger.cpp | 19 +++++++++++++++++++ src/AnimationMerger_test.cpp | 2 ++ src/MotionGenerator.cpp | 5 ++++- 7 files changed, 61 insertions(+), 16 deletions(-) diff --git a/scripts/prep-t2m-v5.py b/scripts/prep-t2m-v5.py index 130fa532..e7a387f2 100644 --- a/scripts/prep-t2m-v5.py +++ b/scripts/prep-t2m-v5.py @@ -347,19 +347,27 @@ def window(action, cq, valid, src): acts.append(action) srcs.append(src) + # Per-item guard: a single malformed clip/BVH must not abort a multi- + # thousand-item batch (offline dev tool — resilience over strictness). if a.corpus: - n0 = 0 + n0, bad = 0, 0 for action, cq, valid, src in corpus_clips( os.path.expanduser(a.corpus), a.min_roles): - window(action, cq, valid, src) - n0 += 1 - print(f"corpus: {n0} clips → {len(mo)} windows") + try: + window(action, cq, valid, src); n0 += 1 + except Exception as e: # noqa: BLE001 + bad += 1 + print(f" skip corpus clip ({src}): {e}") + print(f"corpus: {n0} clips → {len(mo)} windows ({bad} skipped)") if a.bvh and a.index: - n0, w0 = 0, len(mo) + n0, w0, bad = 0, len(mo), 0 for action, cq, valid, src in cmu_clips(a.bvh, a.index): - window(action, cq, valid, src) - n0 += 1 - print(f"cmu: {n0} trials → {len(mo) - w0} windows") + try: + window(action, cq, valid, src); n0 += 1 + except Exception as e: # noqa: BLE001 + bad += 1 + print(f" skip cmu trial ({src}): {e}") + print(f"cmu: {n0} trials → {len(mo) - w0} windows ({bad} skipped)") print(f"posture gates dropped {dropped[0]} windows") if not mo: @@ -370,6 +378,9 @@ def window(action, cq, valid, src): cnt = Counter(acts) vocab = sorted(w for w, n in cnt.items() if n >= a.min_action_windows) keep = [i for i, w in enumerate(acts) if w in vocab] + if not vocab or not keep: + sys.exit(f"no action reached --min-action-windows " + f"({a.min_action_windows}); per-action counts: {dict(cnt)}") mo = np.stack([mo[i] for i in keep]).astype(np.float32) msk = np.stack([msk[i] for i in keep]).astype(np.float32) tk = np.zeros((len(keep), len(vocab)), np.float32) diff --git a/scripts/train-t2m-flow-v5.py b/scripts/train-t2m-flow-v5.py index 6ea8e089..ba6c145e 100644 --- a/scripts/train-t2m-flow-v5.py +++ b/scripts/train-t2m-flow-v5.py @@ -228,7 +228,9 @@ def main(): ckpt_path = os.path.join(a.out, "ckpt.pt") start_ep = 0 if a.resume and os.path.exists(ckpt_path): - ck = torch.load(ckpt_path, map_location=dev) + # weights_only=True: the checkpoint is only state_dicts + an int + # epoch — no pickled objects — so load safely (no code execution). + ck = torch.load(ckpt_path, map_location=dev, weights_only=True) net.load_state_dict(ck["net"]) opt.load_state_dict(ck["opt"]) sched.load_state_dict(ck["sched"]) diff --git a/scripts/upload-t2m-v5-model.sh b/scripts/upload-t2m-v5-model.sh index c199cf58..9c7c06fb 100755 --- a/scripts/upload-t2m-v5-model.sh +++ b/scripts/upload-t2m-v5-model.sh @@ -28,15 +28,21 @@ for f in t2m.onnx t2m-vocab.json; do [[ -f "$OUT_DIR/$f" ]] || { echo "missing $OUT_DIR/$f" >&2; exit 1; } done -# keep the previous (v4 CVAE) files for rollback under versioned names +# keep the previous (v4 CVAE) files for rollback under versioned names. +# IDEMPOTENT: only back up if the v4 rollback does NOT already exist — +# otherwise a re-run would overwrite the real v4 with the current (v5) file +# and destroy the rollback point. TMP=$(mktemp -d) for f in t2m.onnx t2m-vocab.json; do + case "$f" in + t2m.onnx) dst="motion/t2m-v4.onnx" ;; + t2m-vocab.json) dst="motion/t2m-vocab-v4.json" ;; + esac + if huggingface-cli download "$REPO" "$dst" --local-dir "$TMP" >/dev/null 2>&1; then + echo "rollback $dst already exists — skipping backup (idempotent)" + continue + fi if huggingface-cli download "$REPO" "motion/$f" --local-dir "$TMP" >/dev/null 2>&1; then - v4name="${f%.onnx}"; v4name="${v4name%.json}" - case "$f" in - t2m.onnx) dst="motion/t2m-v4.onnx" ;; - t2m-vocab.json) dst="motion/t2m-vocab-v4.json" ;; - esac huggingface-cli upload "$REPO" "$TMP/motion/$f" "$dst" \ --commit-message "t2m: preserve v4 CVAE as $dst before the v5 flow model (#858)" fi diff --git a/src/AnimationControlController.cpp b/src/AnimationControlController.cpp index 861de74f..ef2fbb1c 100644 --- a/src/AnimationControlController.cpp +++ b/src/AnimationControlController.cpp @@ -1961,6 +1961,8 @@ QVariantMap AnimationControlController::generateMotion(const QString& prompt, const auto fp = AnimationMerger::pinFeet(skel.get(), animName); if (fp.ok && fp.spans > 0) out["footPinSpans"] = fp.spans; + else if (!fp.ok && !fp.error.isEmpty()) + out["footPinError"] = fp.error; // surface why (no leg tracks, etc.) } entity->refreshAvailableAnimationState(); diff --git a/src/AnimationMerger.cpp b/src/AnimationMerger.cpp index 6b9f19d6..698ddbd3 100644 --- a/src/AnimationMerger.cpp +++ b/src/AnimationMerger.cpp @@ -1600,6 +1600,25 @@ AnimationMerger::FootPinResult AnimationMerger::pinFeet( || static_cast(shinTrk->getNumKeyFrames()) != nk || (footTrk && static_cast(footTrk->getNumKeyFrames()) != nk)) continue; // mixed keyframe grids — not a generated clip + // The rewrite below indexes shin/foot keyframes by k and assumes they + // share timeSrc's times. Count alone isn't enough — an authored clip + // can have equal counts at DIFFERENT times, which would write a + // correction computed at time t onto a keyframe at t'. Verify the + // grids actually align (generated clips do by construction). + { + bool aligned = true; + for (int k = 0; k < nk && aligned; ++k) { + const float t = times[static_cast(k)]; + if (std::abs(thighTrk->getNodeKeyFrame( + static_cast(k))->getTime() - t) > 1e-4f + || std::abs(shinTrk->getNodeKeyFrame( + static_cast(k))->getTime() - t) > 1e-4f + || (footTrk && std::abs(footTrk->getNodeKeyFrame( + static_cast(k))->getTime() - t) > 1e-4f)) + aligned = false; + } + if (!aligned) continue; // keyframe times don't match — skip leg + } std::vector footC(static_cast(nk)); for (int k = 0; k < nk; ++k) diff --git a/src/AnimationMerger_test.cpp b/src/AnimationMerger_test.cpp index b451f1e3..16beda89 100644 --- a/src/AnimationMerger_test.cpp +++ b/src/AnimationMerger_test.cpp @@ -1288,7 +1288,9 @@ TEST_F(AnimationMergerTest, TwistUnwrapKeepsDampedCollarContinuous) auto mesh = createInMemoryMesh("twist_collar_mesh", skelRes); auto* sm = Manager::getSingleton()->getSceneMgr(); Ogre::Entity* ent = sm->createEntity("twist_collar_ent", mesh); + ASSERT_NE(ent, nullptr); Ogre::SkeletonInstance* skel = ent->getSkeleton(); + ASSERT_NE(skel, nullptr); // Source: left collar (role 10, +X) rolls 0 → 240° — past the ±180° wrap. // The gain table damps collars to 0.5×, which is exactly where a missing diff --git a/src/MotionGenerator.cpp b/src/MotionGenerator.cpp index 012ed98d..eafff638 100644 --- a/src/MotionGenerator.cpp +++ b/src/MotionGenerator.cpp @@ -398,7 +398,10 @@ MotionGenerator::Result MotionGenerator::generate( float score = -std::abs(step - kTargetStep) * 40.0f + coh * 2.0f - std::max(0.0f, maxArtic - kMaxArtic) * 4.0f; - if (haveCanonDirs && worldFrame) { + // Requires the full 22-joint canonical layout — the posture roles + // (17/21 feet, 11 larm) index up to 21. Guard J so a smaller vocab + // can't read out of bounds. + if (haveCanonDirs && worldFrame && J >= 22) { // Posture penalties DOMINATE the energy/coherence terms: a // bad sample (head thrown back, an arm reaching out) is worse // than a slightly-off-tempo good one, so the weights here are From a9490682060be3cad8ba7d82b84b49e5c41fb39d Mon Sep 17 00:00:00 2001 From: Fernando Date: Mon, 20 Jul 2026 06:51:00 -0400 Subject: [PATCH 22/23] test(anim): fix two tests after the model-only-caps + tight-foot-defaults changes (#837) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - TwistUnwrapKeepsDampedCollarContinuous: the per-role twist caps are now model-path-only (#837 review), so this test — which asserts the 240°→ capped-150°→×0.5-collar-gain = 75° behavior — must pass modelClip=true to exercise the capped path (was running uncapped and reaching 240°). - FootContact.ShortBlipsAreIgnored: the tighter default height band made the old data (2 low frames of 20) put the low-percentile ground level ON the lifted height, flagging the whole clip as contact. Rewrote the fixture to a realistic airborne foot with 3 isolated 1-frame dips — ground level lands near 0, each dip is < minFrames, no span registers. Verified against the exact detect algorithm. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/AnimationMerger_test.cpp | 5 ++++- src/FootContact_test.cpp | 8 ++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/AnimationMerger_test.cpp b/src/AnimationMerger_test.cpp index 16beda89..a7e8dbb1 100644 --- a/src/AnimationMerger_test.cpp +++ b/src/AnimationMerger_test.cpp @@ -1305,9 +1305,12 @@ TEST_F(AnimationMergerTest, TwistUnwrapKeepsDampedCollarContinuous) * kSrcRest; quats[f][10] = {w.x, w.y, w.z, w.w}; } + // modelClip=true: the per-role twist caps only apply on the model path + // (authored/self clips run uncapped since #837 review). This test asserts + // the capped collar behavior, so it exercises the model path explicitly. const auto res = AnimationMerger::applyMotionClip( skel, "collarclip", quats, 30, true, srcRestWorld(), - false, 8, false, canonRestDirs()); + false, 8, false, canonRestDirs(), /*modelClip=*/true); ASSERT_TRUE(res.ok) << res.error.toStdString(); // Sample densely: the collar's world orientation must move CONTINUOUSLY diff --git a/src/FootContact_test.cpp b/src/FootContact_test.cpp index 0582e1b5..907dc60d 100644 --- a/src/FootContact_test.cpp +++ b/src/FootContact_test.cpp @@ -50,13 +50,17 @@ TEST(FootContactTest, SlidingFootIsNotAContact) TEST(FootContactTest, ShortBlipsAreIgnored) { + // Foot is airborne (0.6) most of the clip, dipping to the ground for + // three SEPARATE single frames. The low-percentile ground level lands + // near 0, so each dip is an isolated on-ground frame — none forms a + // run of minFrames consecutive, so no contact span registers. std::vector foot; for (int f = 0; f < 20; ++f) - foot.push_back({0.0f, (f == 10 || f == 11) ? 0.0f : 0.6f, 0.0f}); + foot.push_back({0.0f, (f == 3 || f == 10 || f == 17) ? 0.0f : 0.6f, 0.0f}); FootContact::DetectOptions opt; opt.minFrames = 3; const auto spans = FootContact::detectContacts(foot, 1.0f, opt); - EXPECT_TRUE(spans.empty()); // 2-frame touch < minFrames + EXPECT_TRUE(spans.empty()); // isolated 1-frame touches < minFrames } TEST(FootContactTest, SolveKneePreservesLengthsAndReachesTarget) From 904c6d26782334be97c47d8a2451cda42df97dac Mon Sep 17 00:00:00 2001 From: Fernando Date: Mon, 20 Jul 2026 08:36:56 -0400 Subject: [PATCH 23/23] =?UTF-8?q?test(anim):=20correct=20collar=20twist=20?= =?UTF-8?q?expectation=20to=2015=C2=B0=20(#837)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The collar's MODEL twist cap is 30° (0.52 rad), not the old single 150° cap — clamp applies before the 0.5× collar gain, so a 240° source roll settles at 30 × 0.5 = 15°, not 75°. CI (unit-tests-linux) confirmed the retarget produces ~15°; the assertion + comment were stale from the pre-per-role-cap era. Continuity check (maxStep < 15°) unchanged — the unwrap still ramps smoothly to the cap. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/AnimationMerger_test.cpp | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/AnimationMerger_test.cpp b/src/AnimationMerger_test.cpp index a7e8dbb1..5a788c19 100644 --- a/src/AnimationMerger_test.cpp +++ b/src/AnimationMerger_test.cpp @@ -1293,9 +1293,10 @@ TEST_F(AnimationMergerTest, TwistUnwrapKeepsDampedCollarContinuous) ASSERT_NE(skel, nullptr); // Source: left collar (role 10, +X) rolls 0 → 240° — past the ±180° wrap. - // The gain table damps collars to 0.5×, which is exactly where a missing - // unwrap explodes: wrapped −120° would scale to −60° instead of the - // capped +150°'s half — a mid-clip snap. + // This is where a missing unwrap explodes: the wrapped angle would flip + // sign mid-clip and snap. With unwrap, the collar's model twist cap + // (30° = 0.52 rad) clamps first, THEN the 0.5× collar gain applies → + // final steady roll ≈ 30° × 0.5 = 15° about +X. const int frames = 61; auto quats = identityClip(frames); for (int f = 0; f < frames; ++f) { @@ -1336,9 +1337,9 @@ TEST_F(AnimationMergerTest, TwistUnwrapKeepsDampedCollarContinuous) last = w; } EXPECT_LT(maxStepDeg, 15.0f) << "collar roll snapped mid-clip (unwrap)"; - // 240° source twist hits the 150° runaway-unwrap cap FIRST, then the - // 0.5× collar gain: 150 × 0.5 = 75°. - EXPECT_NEAR(twistDegAbout(last, Ogre::Vector3::UNIT_X), 75.0f, 8.0f); + // 240° source twist is clamped to the collar model cap (30°) FIRST, then + // scaled by the 0.5× collar gain: 30 × 0.5 = 15° steady roll about +X. + EXPECT_NEAR(twistDegAbout(last, Ogre::Vector3::UNIT_X), 15.0f, 8.0f); sm->destroyEntity(ent); }