From 2923f607bd7c4c5fba8a3cd597ec9ddbf0f6c8a5 Mon Sep 17 00:00:00 2001 From: Fernando Date: Fri, 10 Jul 2026 21:31:03 -0400 Subject: [PATCH 01/10] feat(anim): canonical clip extraction + motion-library v5 builder (#839) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit t2m-v2 Slice B — turns the harvested motion corpus (#838) into template library clips: - AnimationMerger::extractCanonicalClips: samples every skeletal animation at 30fps onto the 22-joint canonical skeleton as WORLD-frame quats (the v3 library convention), using the SAME bone-role matcher the retarget consumes. Scraped rigs live in arbitrary file frames (Blender FBX armatures are commonly Z-up), so each rig's frame is derived from its own BIND geometry (up=hip→head, left=rhip→lhip) and every quat conjugated into the canonical Y-up/+Z-forward frame — no per-format guessing. - CLI: `qtmesh anim --dump-canonical out.json [--animation N]`. - scripts/build-motion-library-v5.py (offline dev tool): walks the corpus, dumps each validated asset, labels actions from normalized animation names (keyword table + verbatim single-word fallback that widens the prompt vocabulary), selects the highest-energy window snapped to a calm start frame, drops static poses, dedupes sibling characters semantically (same asset+animation+length), and emits the EXISTING qtmesh-motion-library-v3 schema — the shipped app consumes it unchanged — plus the corpus ATTRIBUTION.md. Validated end-to-end: 66 clips / 15 actions (4.3 MB) from the current corpus vs 47/15 in v4 — with new actions (roll, crawl, pickup, shoot, swim…); retargeted walk/roll render upright on the Mixamo test rig with quality matching the CMU baseline (the roll is the Knight's combat roll, tuck→tumble→recover). Unit tests: non-humanoid rigs return empty; a minimal Mixamo-named fixture asserts frame counts, 22-slot poses, identity in unresolved roles, and the ~90° world-frame head delta. Part of #837; implements #839 with the corpus published at https://huggingface.co/datasets/fernandotonon/QtMeshEditor-motion-corpus Co-Authored-By: Claude Fable 5 --- CLAUDE.md | 1 + scripts/build-motion-library-v5.py | 280 +++++++++++++++++++++++++++++ src/AnimationMerger.cpp | 126 +++++++++++++ src/AnimationMerger.h | 23 +++ src/AnimationMerger_test.cpp | 70 ++++++++ src/CLIPipeline.cpp | 77 +++++++- 6 files changed, 576 insertions(+), 1 deletion(-) create mode 100644 scripts/build-motion-library-v5.py diff --git a/CLAUDE.md b/CLAUDE.md index e826b4bbf..864450b5e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -54,6 +54,7 @@ qtmesh anim model.fbx --bake-fps 60 --animation "Run" -o out.fbx # bake one ani qtmesh anim model.fbx --in-between --gap-frames 30 -o filled.fbx # AI in-betweening: fill the clip with 30 predicted keyframes (RMIB ONNX; smooth spline fallback) (#409) qtmesh anim model.fbx --in-between --gap-frames 12 --start-time 0.5 --end-time 1.5 --animation "Jump" -o out.fbx # fill a specific window of one animation qtmesh anim model.fbx --in-between --gap-frames 12 --no-model -o out.fbx # force the deterministic spline fallback (skip the ML model) +qtmesh anim model.fbx --dump-canonical clips.json # #839: extract every skeletal animation onto the 22-joint canonical skeleton (world-frame quats, bind-geometry-derived axis conjugation) — feeds scripts/build-motion-library-v5.py qtmesh anim rigged.fbx --generate "walking confidently" -o out.glb # text-to-motion (#411, experimental): match a permissive CMU clip → retarget onto the rig. Actions: walk/run/jump/dance/march/kick/punch/wave/climb/idle (+ synonyms). Library downloads on first use; needs a humanoid rig qtmesh anim rigged.fbx --generate "jump" --duration 2 -o out.glb # retime the template to N seconds qtmesh pose model.fbx --animation "Walk" --time 0.5 -o posed.stl # export single frame diff --git a/scripts/build-motion-library-v5.py b/scripts/build-motion-library-v5.py new file mode 100644 index 000000000..e0804ab28 --- /dev/null +++ b/scripts/build-motion-library-v5.py @@ -0,0 +1,280 @@ +#!/usr/bin/env python3 +"""Build motion-library v5 from the harvested motion corpus (#839). + +ONE-TIME, OFFLINE developer tool — NOT shipped; the app never runs Python. + +Slice B of the text-to-motion v2 epic (#837): turns the license-filtered +corpus assembled by scrape-motion-corpus.py (#838) into the template clip +library the app downloads — replacing the 47-clip CMU-only v4 with hundreds +of real animation clips across a much wider action vocabulary. + +Pipeline per corpus asset: + 1. `qtmesh anim --dump-canonical tmp.json` — the editor's own + loader + the SAME bone-role matcher the retarget uses maps the rig onto + the 22-joint canonical skeleton and samples every skeletal animation at + 30 fps as WORLD-frame quats (the v3 library convention). + 2. Action labelling: normalized animation name matched against a keyword + table (walk/run/attack/death/...); un-tabled single-word names are kept + verbatim — MotionLibrary::matchPrompt does substring matching, so every + new action widens the usable vocabulary. + 3. Active-window selection (max --max-frames): the window with the highest + rotation energy, start snapped to the calmest nearby frame (the + retarget deltas against clip frame 0 — a mid-swing start reads as a + lurch). Static clips (bind/T-poses) are dropped. + 4. Dedup: sibling characters in one pack share armature actions — clips + with identical (action, frames, sampled-quat fingerprint) collapse. + +OUTPUT: motion-library.json in the EXISTING "qtmesh-motion-library-v3" +schema (frame:"world") — the shipped app consumes it unchanged — plus a copy +of the corpus ATTRIBUTION.md, which MUST ship wherever the library does. + +USAGE + python3 scripts/build-motion-library-v5.py --corpus ~/motion_corpus \ + --out motion-library.json [--qtmesh build_local/bin/qtmesh] +""" + +import argparse +import hashlib +import json +import math +import os +import re +import shutil +import subprocess +import sys +import tempfile + +CANON_COUNT = 22 +FPS = 30 +MODEL_EXTS = (".glb", ".gltf", ".fbx", ".dae") + +# animation-name (normalized) → action. Order matters: first hit wins. +KEYWORDS = [ + ("tpose", None), ("t_pose", None), ("bind", None), ("rest", None), + ("walk", "walk"), ("run", "run"), ("jog", "run"), ("sprint", "run"), + ("idle", "idle"), ("stand", "idle"), ("breath", "idle"), + ("jump", "jump"), ("hop", "jump"), ("leap", "jump"), + ("dance", "dance"), + ("die", "death"), ("death", "death"), ("dead", "death"), ("dying", "death"), + ("attack", "attack"), ("slash", "attack"), ("stab", "attack"), + ("swing", "attack"), ("bite", "attack"), + ("punch", "punch"), ("kick", "kick"), + ("shoot", "shoot"), ("fire", "shoot"), ("aim", "shoot"), + ("cast", "cast"), ("spell", "cast"), ("magic", "cast"), + ("wave", "wave"), ("hello", "wave"), ("greet", "wave"), + ("sit", "sit"), ("crouch", "crouch"), ("sneak", "sneak"), + ("crawl", "crawl"), ("climb", "climb"), ("swim", "swim"), + ("fly", "fly"), ("fall", "fall"), + ("hit", "hit"), ("damage", "hit"), ("hurt", "hit"), ("impact", "hit"), + ("block", "block"), ("dodge", "dodge"), ("roll", "roll"), + ("throw", "throw"), ("pick", "pickup"), ("interact", "interact"), + ("victory", "cheer"), ("cheer", "cheer"), ("win", "cheer"), + ("yes", "nod"), ("no", "shake"), + ("eat", "eat"), ("drink", "eat"), ("sleep", "sleep"), + ("open", "interact"), ("push", "push"), ("pull", "pull"), +] + + +def norm_anim_name(name): + n = name.lower() + n = re.sub(r"^.*\|", "", n) # "Armature|Walk" → "walk" + n = re.sub(r"(armature|action|anim|mixamo\.com|takes?)", " ", n) + n = re.sub(r"[^a-z]+", " ", n) + return " ".join(n.split()) + + +def action_for(anim_name, tags): + n = norm_anim_name(anim_name) + for kw, action in KEYWORDS: + if kw in n.replace(" ", ""): + return action # None = deliberate skip + # single clean word → keep verbatim (widens the prompt vocabulary) + words = n.split() + if len(words) == 1 and 3 <= len(words[0]) <= 16: + return words[0] + for t in tags or []: + for kw, action in KEYWORDS: + if action and kw in str(t).lower(): + return action + return None + + +def quat_angle(a, b): + d = abs(sum(x * y for x, y in zip(a, b))) + return 2.0 * math.acos(max(-1.0, min(1.0, d))) + + +def frame_energy(quats): + """Mean joint rotation speed between consecutive frames (rad/frame).""" + e = [0.0] + for f in range(1, len(quats)): + a = sum(quat_angle(quats[f - 1][j], quats[f][j]) + for j in range(CANON_COUNT)) / CANON_COUNT + e.append(a) + return e + + +def select_window(quats, max_frames): + """Highest-energy window, start snapped to the calmest nearby frame.""" + T = len(quats) + if T <= max_frames: + return 0, T + e = frame_energy(quats) + best_s, best_sum = 0, -1.0 + window = sum(e[:max_frames]) + best_sum, best_s = window, 0 + for s in range(1, T - max_frames + 1): + window += e[s + max_frames - 1] - e[s - 1] + if window > best_sum: + best_sum, best_s = window, s + # snap the start to the calmest frame in the preceding half-second + lo = max(0, best_s - FPS // 2) + calm = min(range(lo, best_s + 1), key=lambda i: e[i]) if best_s > lo \ + else best_s + return calm, min(T, calm + max_frames) + + +def fingerprint(action, quats): + h = hashlib.sha1(action.encode()) + for f in (0, len(quats) // 2, len(quats) - 1): + for j in range(0, CANON_COUNT, 3): + h.update(("%.3f%.3f%.3f%.3f" % tuple(quats[f][j])).encode()) + h.update(str(len(quats)).encode()) + return h.hexdigest() + + +def find_qtmesh(explicit): + if explicit: + return explicit + here = os.path.dirname(os.path.abspath(__file__)) + for c in (os.path.join(here, "..", "build_local", "bin", "qtmesh"), + shutil.which("qtmesh")): + if c and os.path.exists(c): + return c + sys.exit("qtmesh not found — pass --qtmesh") + + +def manifest_lookup(manifest, dirname): + for a in manifest.get("assets", []): + slug = re.sub(r"[^A-Za-z0-9._-]+", "_", a.get("title", "")).strip("_") + if dirname in (slug[:80],) or dirname in slug: + return a + return None + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("--corpus", required=True) + ap.add_argument("--out", default="motion-library.json") + ap.add_argument("--qtmesh", default="") + ap.add_argument("--max-frames", type=int, default=120) # 4 s @ 30 + ap.add_argument("--min-frames", type=int, default=15) # 0.5 s + ap.add_argument("--min-roles", type=int, default=12) + ap.add_argument("--min-energy", type=float, default=0.004, + help="mean rad/frame below which a clip is a pose") + ap.add_argument("--max-per-action", type=int, default=12) + args = ap.parse_args() + + qtmesh = find_qtmesh(args.qtmesh) + corpus = os.path.expanduser(args.corpus) + manifest = {} + mpath = os.path.join(corpus, "manifest.json") + if os.path.exists(mpath): + manifest = json.load(open(mpath)) + + joints = None + clips, seen = [], set() + counts = {} + raw = os.path.join(corpus, "raw") + for source in sorted(os.listdir(raw)): + sdir = os.path.join(raw, source) + if not os.path.isdir(sdir): + continue + for asset in sorted(os.listdir(sdir)): + adir = os.path.join(sdir, asset) + if not os.path.isdir(adir): + continue + prov = manifest_lookup(manifest, asset) or {} + tags = prov.get("tags", []) + title = prov.get("title", asset) + for root, _d, files in os.walk(adir): + for fn in sorted(files): + if not fn.lower().endswith(MODEL_EXTS): + continue + fpath = os.path.join(root, fn) + with tempfile.NamedTemporaryFile( + suffix=".json", delete=False) as tf: + tmp = tf.name + try: + r = subprocess.run( + [qtmesh, "anim", fpath, "--dump-canonical", tmp], + capture_output=True, text=True, timeout=600) + if r.returncode != 0 or not os.path.getsize(tmp): + continue + dump = json.load(open(tmp)) + except Exception: + continue + finally: + try: + os.remove(tmp) + except OSError: + pass + joints = joints or dump.get("joints") + for c in dump.get("clips", []): + if c.get("resolvedRoles", 0) < args.min_roles: + continue + q = c.get("quats", []) + if len(q) < args.min_frames: + continue + action = action_for(c.get("animation", ""), tags) + if not action: + continue + s, epos = select_window(q, args.max_frames) + w = q[s:epos] + e = frame_energy(w) + if sum(e) / max(1, len(e)) < args.min_energy: + continue # a pose, not a motion + # Sibling characters in one pack share armature + # actions but their rest bones differ subtly — the + # quat fingerprint alone misses them, so dedupe + # semantically too: same asset + animation + length + # IS the same motion. + sem = (title, norm_anim_name(c.get("animation", "")), + len(w)) + fp = fingerprint(action, w) + if fp in seen or sem in seen: + continue # duplicate take + if counts.get(action, 0) >= args.max_per_action: + continue + seen.add(fp); seen.add(sem) + counts[action] = counts.get(action, 0) + 1 + clips.append({ + "action": action, + "source": f"{title} — {c.get('animation')}", + "frames": len(w), + "quats": w, + }) + print(f" + {action:<10} {title[:38]:<40}" + f" {c.get('animation')} ({len(w)}f)") + + if not clips: + sys.exit("no clips extracted — is the corpus downloaded/validated?") + + lib = {"schema": "qtmesh-motion-library-v3", "joints": joints, + "fps": FPS, "frame": "world", "clips": clips} + with open(args.out, "w") as f: + json.dump(lib, f) + att = os.path.join(corpus, "ATTRIBUTION.md") + if os.path.exists(att): + shutil.copy(att, os.path.join( + os.path.dirname(os.path.abspath(args.out)) or ".", + "ATTRIBUTION.md")) + print(f"\nwrote {args.out}: {len(clips)} clips, " + f"{len(counts)} actions, " + f"{os.path.getsize(args.out) / 1e6:.1f} MB") + for a in sorted(counts): + print(f" {a}: {counts[a]}") + + +if __name__ == "__main__": + main() diff --git a/src/AnimationMerger.cpp b/src/AnimationMerger.cpp index afc0b11b5..df5086b3b 100644 --- a/src/AnimationMerger.cpp +++ b/src/AnimationMerger.cpp @@ -911,6 +911,132 @@ AnimationMerger::InbetweenResult AnimationMerger::inbetweenAnimation( return res; } +std::vector +AnimationMerger::extractCanonicalClips(Ogre::Entity* entity, int fps, + const QString& onlyAnimation) +{ + std::vector out; + if (!entity || !entity->hasSkeleton() || fps <= 0) + return out; + Ogre::SkeletonInstance* skel = entity->getSkeleton(); + Ogre::AnimationStateSet* states = entity->getAllAnimationStates(); + if (!skel || !states) + return out; + + const int J = MotionInbetween::canonicalJointCount(); + + // Bone → canonical role, first match wins per role (the same matcher the + // retarget uses, so round-trips are consistent by construction). + std::vector roleBone(static_cast(J), nullptr); + int resolved = 0; + for (auto* bone : skel->getBones()) { + const int role = MotionInbetween::canonicalIndexForBone( + QString::fromStdString(bone->getName())); + if (role >= 0 && role < J && !roleBone[static_cast(role)]) { + roleBone[static_cast(role)] = bone; + ++resolved; + } + } + if (resolved == 0) + return out; + + // Remember enabled states so sampling leaves the entity as found. + std::vector> prev; + for (auto& it : states->getAnimationStates()) + prev.emplace_back(it.second, it.second->getEnabled()); + for (auto& p : prev) p.first->setEnabled(false); + + // ── source-frame → canonical-frame conjugation ───────────────────── + // Scraped rigs live in arbitrary file frames (Blender FBX armatures are + // commonly Z-up), while the motion library's world convention is Y-up, + // +Z-facing, +X-left. Guessing per format is fragile — derive the + // source frame from the rig's own BIND geometry instead: up = hip→head, + // left = right-hip→left-hip (shoulders as fallback), forward = left×up. + // Every sampled world quat is then conjugated: q' = C · q · C⁻¹. + Ogre::Quaternion C = Ogre::Quaternion::IDENTITY; + { + skel->reset(true); + skel->_updateTransforms(); + auto posOf = [&](int role) -> const Ogre::Bone* { + return (role >= 0 && role < J) + ? roleBone[static_cast(role)] : nullptr; + }; + const Ogre::Bone* hip = posOf(0); // "hip" + const Ogre::Bone* head = posOf(5); // "head" + if (!head) head = posOf(3); // "neck" fallback + const Ogre::Bone* lSide = posOf(19); // "lhip" + const Ogre::Bone* rSide = posOf(15); // "rhip" + if (!lSide || !rSide) { lSide = posOf(11); rSide = posOf(7); } + if (hip && head && lSide && rSide) { + Ogre::Vector3 up = head->_getDerivedPosition() + - hip->_getDerivedPosition(); + Ogre::Vector3 left = lSide->_getDerivedPosition() + - rSide->_getDerivedPosition(); + if (up.squaredLength() > 1e-12f + && left.squaredLength() > 1e-12f) { + up.normalise(); + left = left - up * left.dotProduct(up); // orthogonalise + if (left.squaredLength() > 1e-12f) { + left.normalise(); + const Ogre::Vector3 fwd = left.crossProduct(up); + // Rotation taking the SOURCE basis (left, up, fwd) onto + // the canonical axes (+X, +Y, +Z). + Ogre::Matrix3 src; + src.SetColumn(0, left); + src.SetColumn(1, up); + src.SetColumn(2, fwd); + C = Ogre::Quaternion(src).Inverse(); + C.normalise(); + } + } + } + } + + for (auto& it : states->getAnimationStates()) { + Ogre::AnimationState* st = it.second; + const QString name = QString::fromStdString(st->getAnimationName()); + if (!onlyAnimation.isEmpty() + && name.compare(onlyAnimation, Qt::CaseInsensitive) != 0) + continue; + const float length = st->getLength(); + if (length <= 0.0f) + continue; + + CanonicalClip clip; + clip.animation = name; + clip.resolvedRoles = resolved; + const int frames = + std::max(2, static_cast(std::lround(length * fps)) + 1); + clip.quats.reserve(static_cast(frames)); + + st->setEnabled(true); + for (int f = 0; f < frames; ++f) { + st->setTimePosition(std::min(length, + static_cast(f) / static_cast(fps))); + skel->setAnimationState(*states); + skel->_updateTransforms(); + std::vector> pose( + static_cast(J), {0.f, 0.f, 0.f, 1.f}); + for (int j = 0; j < J; ++j) { + Ogre::Bone* b = roleBone[static_cast(j)]; + if (!b) continue; + const Ogre::Quaternion w = + C * b->_getDerivedOrientation() * C.Inverse(); + pose[static_cast(j)] = { + static_cast(w.x), static_cast(w.y), + static_cast(w.z), static_cast(w.w)}; + } + clip.quats.push_back(std::move(pose)); + } + st->setEnabled(false); + clip.frames = static_cast(clip.quats.size()); + out.push_back(std::move(clip)); + } + + for (auto& p : prev) p.first->setEnabled(p.second); + return out; +} + bool AnimationMerger::detectBackwardFacing(Ogre::Entity* entity) { // Escape hatch for exotic meshes where the foot-region heuristic guesses diff --git a/src/AnimationMerger.h b/src/AnimationMerger.h index c3ad8e48d..d4b8bbc52 100644 --- a/src/AnimationMerger.h +++ b/src/AnimationMerger.h @@ -182,6 +182,29 @@ class AnimationMerger { int refineStride = 8, bool yaw180 = false); + /// One skeletal animation extracted onto the 22-joint canonical skeleton + /// (#839, the REVERSE of applyMotionClip's world-frame path): per frame, + /// per canonical role, the source bone's WORLD orientation — exactly the + /// "frame":"world" convention the v3 motion library stores, so extracted + /// clips ride the existing retarget unchanged (delta vs clip frame 0). + struct CanonicalClip { + QString animation; ///< source animation name + int frames = 0; ///< sampled frame count at `fps` + int resolvedRoles = 0; ///< canonical roles matched on this rig (≤22) + /// frames × 22 × [x,y,z,w]; unresolved roles hold identity. + std::vector>> quats; + }; + + /// Sample every (or one) skeletal animation of `entity` at `fps` and + /// express each canonical joint's world orientation per frame. Bone→role + /// mapping is MotionInbetween::canonicalIndexForBone — the same matcher + /// the retarget uses, so extraction and application are consistent by + /// construction. Animations whose rig resolves 0 roles are skipped. + static std::vector extractCanonicalClips( + Ogre::Entity* entity, + int fps = 30, + const QString& onlyAnimation = {}); + /// True when the entity's mesh appears to FACE −Z (the retarget and the /// CMU clips assume +Z): detected from the foot region — toe mass extends /// forward of the ankle joints. Rigs WITH a harvested standing pose don't diff --git a/src/AnimationMerger_test.cpp b/src/AnimationMerger_test.cpp index 952f4a43c..f9abfd958 100644 --- a/src/AnimationMerger_test.cpp +++ b/src/AnimationMerger_test.cpp @@ -10,6 +10,7 @@ #include #include #include +#include "MotionInbetween.h" class AnimationMergerTest : public ::testing::Test { protected: @@ -893,3 +894,72 @@ TEST(AnimationMergerStandaloneTest, NullSkeletonCompatibility) Ogre::SkeletonPtr null; EXPECT_FALSE(AnimationMerger::areSkeletonsCompatible(null, null)); } + +// ── #839: rig→canonical clip extraction ───────────────────────────────────── + +TEST_F(AnimationMergerTest, ExtractCanonicalClipsRejectsNonHumanoid) +{ + // Root/Child bone names resolve no canonical roles → empty result. + Ogre::Entity* ent = createAnimatedTestEntity("extract_nonhuman"); + ASSERT_NE(ent, nullptr); + EXPECT_TRUE(AnimationMerger::extractCanonicalClips(ent).empty()); +} + +TEST_F(AnimationMergerTest, ExtractCanonicalClipsSamplesWorldFrame) +{ + // Minimal humanoid: Mixamo-style names resolve hip/head/lhip/rhip roles. + auto skel = Ogre::SkeletonManager::getSingleton().create( + "extract_skel", Ogre::ResourceGroupManager::DEFAULT_RESOURCE_GROUP_NAME); + auto* hips = skel->createBone("Hips", 0); + hips->setPosition(Ogre::Vector3(0, 1, 0)); + auto* head = skel->createBone("Head", 1); + head->setPosition(Ogre::Vector3(0, 0.7f, 0)); + hips->addChild(head); + auto* lleg = skel->createBone("LeftUpLeg", 2); + lleg->setPosition(Ogre::Vector3(0.15f, -0.1f, 0)); + hips->addChild(lleg); + auto* rleg = skel->createBone("RightUpLeg", 3); + rleg->setPosition(Ogre::Vector3(-0.15f, -0.1f, 0)); + hips->addChild(rleg); + skel->setBindingPose(); + + auto* anim = skel->createAnimation("Spin", 1.0f); + auto* track = anim->createNodeTrack(1, head); + track->createNodeKeyFrame(0.0f); + auto* k1 = static_cast( + track->createNodeKeyFrame(1.0f)); + k1->setRotation(Ogre::Quaternion(Ogre::Degree(90), + Ogre::Vector3::UNIT_Y)); + + auto mesh = createInMemoryMesh("extract_mesh", skel); + auto* sceneMgr = Manager::getSingleton()->getSceneMgr(); + Ogre::Entity* ent = sceneMgr->createEntity("extract_ent", mesh); + + const auto clips = AnimationMerger::extractCanonicalClips(ent, 30); + ASSERT_EQ(clips.size(), 1u); + const auto& c = clips.front(); + EXPECT_EQ(c.animation, QStringLiteral("Spin")); + EXPECT_EQ(c.resolvedRoles, 4); + EXPECT_EQ(c.frames, 31); // 1s @ 30fps inclusive + ASSERT_EQ(static_cast(c.quats.size()), c.frames); + for (const auto& pose : c.quats) + ASSERT_EQ(pose.size(), + static_cast(MotionInbetween::canonicalJointCount())); + + // Unresolved roles (e.g. chest, index 2) stay identity in every frame. + const auto& idq = c.quats.front()[2]; + EXPECT_FLOAT_EQ(idq[0], 0.f); EXPECT_FLOAT_EQ(idq[3], 1.f); + + // The animated head (role 5) actually rotates ~90° about the up axis + // between first and last frame — world-frame delta, conjugation-safe. + const auto& h0 = c.quats.front()[5]; + const auto& h1 = c.quats.back()[5]; + const Ogre::Quaternion q0(h0[3], h0[0], h0[1], h0[2]); + const Ogre::Quaternion q1(h1[3], h1[0], h1[1], h1[2]); + const Ogre::Quaternion d = q1 * q0.Inverse(); + const double ang = 2.0 * std::acos(std::min(1.0, + static_cast(std::abs(d.w)))) * 180.0 / M_PI; + EXPECT_NEAR(ang, 90.0, 5.0); + + sceneMgr->destroyEntity(ent); +} diff --git a/src/CLIPipeline.cpp b/src/CLIPipeline.cpp index 8f4adb7bf..edda89de5 100644 --- a/src/CLIPipeline.cpp +++ b/src/CLIPipeline.cpp @@ -2135,6 +2135,8 @@ int CLIPipeline::cmdAnim(int argc, char* argv[]) bool bakeFpsMode = false; bool inbetweenMode = false; // #409: AI in-betweening bool inbetweenNoModel = false; // --no-model → force spline fallback + bool dumpCanonicalMode = false; // #839: rig→canonical clip extraction + QString dumpCanonicalPath; // --dump-canonical bool generateMode = false; // #411: text-to-motion (template-clip MVP) QString generatePrompt; // --generate "" float generateDuration = 0.0f; // --duration N (seconds; 0 = clip's native length) @@ -2208,6 +2210,11 @@ int CLIPipeline::cmdAnim(int argc, char* argv[]) continue; } if (arg == "--no-model") { inbetweenNoModel = true; continue; } + if (arg == "--dump-canonical" && i + 1 < argc) { + dumpCanonicalMode = true; + dumpCanonicalPath = QString(argv[++i]); + continue; + } if (arg == "--generate" && i + 1 < argc) { generateMode = true; generatePrompt = QString::fromLocal8Bit(argv[++i]); @@ -2279,7 +2286,8 @@ int CLIPipeline::cmdAnim(int argc, char* argv[]) } if (!listMode && !renameMode && !mergeMode && !resampleMode && !decimateMode - && !simplifyMode && !analyzeMode && !bakeFpsMode && !inbetweenMode) { + && !simplifyMode && !analyzeMode && !bakeFpsMode && !inbetweenMode + && !dumpCanonicalMode) { err() << "Error: Specify --list, --rename, --merge, --resample, --decimate-step, --simplify, --bake-fps, --in-between, --generate, or --analyze." << Qt::endl; err() << "Usage: qtmesh anim --list [--json]" << Qt::endl; err() << " qtmesh anim --analyze [--json]" << Qt::endl; @@ -2292,6 +2300,7 @@ int CLIPipeline::cmdAnim(int argc, char* argv[]) err() << " (--tolerance T sets translation+scale tolerance in world units)" << Qt::endl; err() << " qtmesh anim --analyze [--json] [--preset ...] [--tolerance T] [--rotation-tolerance-deg D]" << Qt::endl; err() << " qtmesh anim --in-between --gap-frames N [--start-time S] [--end-time S] [--no-model] [-o ] [--animation ]" << Qt::endl; + err() << " qtmesh anim --dump-canonical [--animation ] (#839: 22-joint world-frame clip extraction)" << Qt::endl; return 2; } @@ -2360,6 +2369,72 @@ int CLIPipeline::cmdAnim(int argc, char* argv[]) const bool isAnimOnlyInput = (entity == nullptr); + if (dumpCanonicalMode) { + // #839 (t2m-v2 Slice B): extract every skeletal animation onto the + // 22-joint canonical skeleton as WORLD-frame quats — the same + // "frame":"world" convention the v3 motion library stores, produced + // by the same bone-role matcher the retarget consumes. + if (isAnimOnlyInput) { + err() << "Error: --dump-canonical needs a mesh with its rig " + "(animation-only input has no bind pose)." << Qt::endl; + return 1; + } + SentryReporter::addBreadcrumb("ai.tool_call", + QString("anim dump-canonical: %1").arg(fi.fileName())); + const auto clips = AnimationMerger::extractCanonicalClips( + entity, 30, animationFilter); + if (clips.empty()) { + err() << "Error: no canonical clips extracted (rig resolves no " + "canonical roles, or no matching animation)." << Qt::endl; + return 1; + } + QJsonObject root; + root["schema"] = "qtmesh-canonical-clips-v1"; + root["frame"] = "world"; + root["fps"] = 30; + root["source"] = fi.fileName(); + QJsonArray joints; + for (int j = 0; j < MotionInbetween::canonicalJointCount(); ++j) + joints.append(MotionInbetween::canonicalJointName(j)); + root["joints"] = joints; + QJsonArray clipArr; + for (const auto& c : clips) { + QJsonObject co; + co["animation"] = c.animation; + co["frames"] = c.frames; + co["resolvedRoles"] = c.resolvedRoles; + QJsonArray frames; + for (const auto& pose : c.quats) { + QJsonArray row; + for (const auto& q : pose) { + QJsonArray quat; + // [x,y,z,w] — the motion-library component order. + for (float v : q) + quat.append(static_cast( + std::round(v * 100000.0f) / 100000.0f)); + row.append(quat); + } + frames.append(row); + } + co["quats"] = frames; + clipArr.append(co); + } + root["clips"] = clipArr; + QFile out(dumpCanonicalPath); + if (!out.open(QIODevice::WriteOnly | QIODevice::Truncate)) { + err() << "Error: cannot write " << dumpCanonicalPath << Qt::endl; + return 1; + } + out.write(QJsonDocument(root).toJson(QJsonDocument::Compact)); + out.close(); + cliWrite(QString("Dumped %1 canonical clip(s) (%2/%3 roles) to %4\n") + .arg(clips.size()) + .arg(clips.front().resolvedRoles) + .arg(MotionInbetween::canonicalJointCount()) + .arg(dumpCanonicalPath)); + return 0; + } + if (listMode) { if (skel->getNumAnimations() == 0) { cliWrite(jsonOutput ? "[]\n" : "No animations found.\n"); From 188687fe72fbbc429c66b82550ca3ba5e3c3e56f Mon Sep 17 00:00:00 2001 From: Fernando Date: Fri, 10 Jul 2026 22:09:57 -0400 Subject: [PATCH 02/10] fix(anim): sample canonical clips via direct Animation::apply MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The state-set path (Skeleton::setAnimationState via the entity's AnimationStateSet) applied nothing for hand-built skeletons on the CI fixture — the head delta sampled 0°. Apply each Animation directly to the skeleton instance per frame instead: deterministic for both imported and hand-built rigs, no enabled-state bookkeeping, and the bind pose is restored after sampling. Output verified bit-identical to the previous path on real corpus files. Co-Authored-By: Claude Fable 5 --- src/AnimationMerger.cpp | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/src/AnimationMerger.cpp b/src/AnimationMerger.cpp index df5086b3b..b8dfad063 100644 --- a/src/AnimationMerger.cpp +++ b/src/AnimationMerger.cpp @@ -919,8 +919,7 @@ AnimationMerger::extractCanonicalClips(Ogre::Entity* entity, int fps, if (!entity || !entity->hasSkeleton() || fps <= 0) return out; Ogre::SkeletonInstance* skel = entity->getSkeleton(); - Ogre::AnimationStateSet* states = entity->getAllAnimationStates(); - if (!skel || !states) + if (!skel) return out; const int J = MotionInbetween::canonicalJointCount(); @@ -940,12 +939,6 @@ AnimationMerger::extractCanonicalClips(Ogre::Entity* entity, int fps, if (resolved == 0) return out; - // Remember enabled states so sampling leaves the entity as found. - std::vector> prev; - for (auto& it : states->getAnimationStates()) - prev.emplace_back(it.second, it.second->getEnabled()); - for (auto& p : prev) p.first->setEnabled(false); - // ── source-frame → canonical-frame conjugation ───────────────────── // Scraped rigs live in arbitrary file frames (Blender FBX armatures are // commonly Z-up), while the motion library's world convention is Y-up, @@ -992,13 +985,18 @@ AnimationMerger::extractCanonicalClips(Ogre::Entity* entity, int fps, } } - for (auto& it : states->getAnimationStates()) { - Ogre::AnimationState* st = it.second; - const QString name = QString::fromStdString(st->getAnimationName()); + // Sample by applying each Animation DIRECTLY to the skeleton instance — + // deterministic regardless of the entity's animation-state bookkeeping + // (state-set application proved instance-dependent for hand-built + // skeletons), and it leaves the entity's enabled states untouched. + for (unsigned short a = 0; a < skel->getNumAnimations(); ++a) { + Ogre::Animation* anim = skel->getAnimation(a); + if (!anim) continue; + const QString name = QString::fromStdString(anim->getName()); if (!onlyAnimation.isEmpty() && name.compare(onlyAnimation, Qt::CaseInsensitive) != 0) continue; - const float length = st->getLength(); + const float length = anim->getLength(); if (length <= 0.0f) continue; @@ -1009,11 +1007,10 @@ AnimationMerger::extractCanonicalClips(Ogre::Entity* entity, int fps, std::max(2, static_cast(std::lround(length * fps)) + 1); clip.quats.reserve(static_cast(frames)); - st->setEnabled(true); for (int f = 0; f < frames; ++f) { - st->setTimePosition(std::min(length, + skel->reset(true); + anim->apply(skel, std::min(length, static_cast(f) / static_cast(fps))); - skel->setAnimationState(*states); skel->_updateTransforms(); std::vector> pose( static_cast(J), {0.f, 0.f, 0.f, 1.f}); @@ -1028,12 +1025,13 @@ AnimationMerger::extractCanonicalClips(Ogre::Entity* entity, int fps, } clip.quats.push_back(std::move(pose)); } - st->setEnabled(false); clip.frames = static_cast(clip.quats.size()); out.push_back(std::move(clip)); } - for (auto& p : prev) p.first->setEnabled(p.second); + // Restore the bind pose so the on-screen entity isn't left mid-clip. + skel->reset(true); + skel->_updateTransforms(); return out; } From 331fe02ae503fdbe65788dfdd8ede59a2cc9a5ad Mon Sep 17 00:00:00 2001 From: Fernando Date: Sat, 11 Jul 2026 01:28:32 -0400 Subject: [PATCH 03/10] fix(anim): harvest the standing pose at the calmest frame, not frame 0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Authored animations often OPEN on a stylized pose (a dance intro, a wind-up), and composing every generated clip onto frame 0 baked that style into the rig's neutral — generations looked "based on the previous animation's first frame" instead of a relaxed stance. The standing-pose harvest now samples the reference animation's rotation energy (pure track math, nothing applied to the live skeleton) and harvests at the calmest frame — the closest thing the rig has to a relaxed standing pose. Generated clips remain excluded from the harvest as before. Co-Authored-By: Claude Fable 5 --- src/AnimationMerger.cpp | 43 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 41 insertions(+), 2 deletions(-) diff --git a/src/AnimationMerger.cpp b/src/AnimationMerger.cpp index b8dfad063..d588c0583 100644 --- a/src/AnimationMerger.cpp +++ b/src/AnimationMerger.cpp @@ -1204,10 +1204,49 @@ AnimationMerger::ApplyMotionResult AnimationMerger::applyMotionClip( && a->getName().rfind("generated_", 0) != 0) { ref = a; break; } } if (ref) { - for (const auto& [h, trk] : ref->_getNodeTrackList()) { + // Harvest at the reference animation's CALMEST frame, not + // blindly frame 0 — authored clips often OPEN on a stylized + // pose (a dance intro, a wind-up), and composing every + // generated clip onto that bakes the style into the neutral + // ("generations look based on the previous animation's first + // frame"). The calmest frame is the closest thing the rig has + // to a relaxed standing pose. Pure track math — nothing is + // applied to the live skeleton. + const float len = ref->getLength(); + const int samples = std::clamp( + static_cast(std::lround(len * 30.0f)) + 1, 2, 301); + const auto& tracks = ref->_getNodeTrackList(); + std::vector energy(static_cast(samples), 0.0); + for (const auto& [h, trk] : tracks) { + if (!trk || trk->getNumKeyFrames() == 0) continue; + Ogre::Quaternion prevQ; + for (int f = 0; f < samples; ++f) { + const float t = len * static_cast(f) + / static_cast(samples - 1); + Ogre::TransformKeyFrame kf(nullptr, 0.0f); + trk->getInterpolatedKeyFrame(ref->_getTimeIndex(t), &kf); + const Ogre::Quaternion q = kf.getRotation(); + if (f > 0) { + const double d = std::min(1.0, std::abs( + static_cast(q.Dot(prevQ)))); + energy[static_cast(f)] += 2.0 * std::acos(d); + } + prevQ = q; + } + } + // energy[f] = motion between samples f-1 and f; energy[0] is 0 + // by construction, so start the argmin at frame 1. + int calm = 1; + for (int f = 2; f < samples; ++f) + if (energy[static_cast(f)] + < energy[static_cast(calm)]) + calm = f; + const float tCalm = len * static_cast(calm) + / static_cast(samples - 1); + for (const auto& [h, trk] : tracks) { if (!trk || trk->getNumKeyFrames() == 0) continue; Ogre::TransformKeyFrame f0(nullptr, 0.0f); - trk->getInterpolatedKeyFrame(ref->_getTimeIndex(0.0f), &f0); + trk->getInterpolatedKeyFrame(ref->_getTimeIndex(tCalm), &f0); standPose[h] = { f0.getRotation(), f0.getTranslate(), f0.getScale(), true }; } } From 32d6b86b0edd66fb918f312d4e6e9f807280a62c Mon Sep 17 00:00:00 2001 From: Fernando Date: Sat, 11 Jul 2026 01:50:49 -0400 Subject: [PATCH 04/10] feat(anim): restore pelvic sway in the retarget (yaw-locked swing-twist root) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The root was fully locked to the standing pose (CMU/scraped clips bake whole-body facing into the hip), which zeroed the hip's natural walk sway — measured 14.5° on a Mixamo reference walk vs 0° retargeted, the single largest fidelity gap in the reference-parity harness. Split the root's world-frame delta about canonical +Y (swing–twist): the twist (facing) stays locked, the swing (pitch/roll pelvic sway) applies pre-multiplied in world axes. Reference-parity numbers (self-retarget of a Mixamo walk through dump-canonical → library → generate on the same rig): mean per-joint delta error 2.25° → 1.84°, hip 7.84° → 2.62° (amplitude 0 → 10.1° vs 14.5° ref — the residual is the removed yaw, by design), torso errors also improved; legs remain ≤0.15°. Pose-shape IoU on isometric sheets (same render pipeline): 0.787 → 0.816 (self-ceiling 1.0, residual is sub-frame phase quantization). Cross-rig (reference walk onto a dance-rigged character): 2.32° with no contamination from the dance — verifying the calm-frame standing-pose harvest end to end. Co-Authored-By: Claude Fable 5 --- src/AnimationMerger.cpp | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/src/AnimationMerger.cpp b/src/AnimationMerger.cpp index d588c0583..cdf599068 100644 --- a/src/AnimationMerger.cpp +++ b/src/AnimationMerger.cpp @@ -1385,7 +1385,30 @@ AnimationMerger::ApplyMotionResult AnimationMerger::applyMotionClip( artic = Ogre::Quaternion::Slerp(1.0f / static_cast(dup), Ogre::Quaternion::IDENTITY, artic, /*shortestPath=*/true); - const Ogre::Quaternion local = (c == 0) ? bind : (bind * artic); + Ogre::Quaternion local; + if (c == 0) { + // Root: CMU/scraped clips bake whole-body FACING into the hip, + // so the yaw must stay locked to the standing pose — but the + // pitch/roll component is the pelvic sway that makes walks + // read as alive (measured 14.5° on a reference walk, 0° when + // fully locked). Swing–twist split about canonical +Y: drop + // the twist (facing), pre-multiply the swing in world axes. + local = bind; + if (worldFrame) { + const Ogre::Quaternion d = cmuLocalDelta[0][f]; + Ogre::Quaternion twist(d.w, 0.0f, d.y, 0.0f); + const float n = std::sqrt(twist.w * twist.w + + twist.y * twist.y); + if (n > 1e-6f) { + twist.w /= n; twist.y /= n; + Ogre::Quaternion swing = d * twist.Inverse(); + swing.normalise(); + local = swing * bind; + } + } + } else { + local = bind * artic; + } Ogre::TransformKeyFrame* kf = track->createNodeKeyFrame(f * dt); kf->setRotation(local); kf->setTranslate(standPos); From 2fc3047bba2ed7b55e407b39ffe3a70fd252febb Mon Sep 17 00:00:00 2001 From: Fernando Date: Sat, 11 Jul 2026 02:09:15 -0400 Subject: [PATCH 05/10] feat(scripts): motion retarget evaluation harness (#837 Slice D) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit joint-deltas (convention-safe per-joint delta-angle trajectories) and sheets (bbox/scale-normalized silhouette IoU with cyclic frame alignment) comparisons between a reference animation and a retargeted one. Documents the local-eval-only posture for Mixamo references and the recorded 2026-07-11 baselines (1.84° mean joint error, IoU 0.816). Co-Authored-By: Claude Fable 5 --- scripts/compare-motion-clips.py | 116 ++++++++++++++++++++++++++++++++ 1 file changed, 116 insertions(+) create mode 100755 scripts/compare-motion-clips.py diff --git a/scripts/compare-motion-clips.py b/scripts/compare-motion-clips.py new file mode 100755 index 000000000..688465884 --- /dev/null +++ b/scripts/compare-motion-clips.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +"""Motion retarget evaluation harness (#837 Slice D). + +OFFLINE dev tool — NOT shipped. Two convention-safe comparisons between a +REFERENCE animation and a RETARGETED/generated one: + +1. joint-deltas: per-canonical-joint delta-ANGLE trajectories (each clip's + frames vs its own frame 0) — absolute world quats are NOT comparable + across rigs (bone-axis conventions differ; the same lesson as skin-weight + L1), but delta angles are. Reports per-joint amplitude + mean |Δ| error. +2. sheets: pose-shape IoU between two isometric sprite sheets — each cell's + silhouette is bbox-cropped and scale-normalized so framing/render-style + differences don't drown the pose signal; per direction the best cyclic + frame alignment is used. + +Reference workflow (LOCAL evaluation only — e.g. Mixamo downloads may not +enter the corpus/training data per Adobe ToS and THIRD_PARTY_AI_MODELS.md): + qtmesh anim ref.fbx --dump-canonical ref.json + qtmesh anim target.fbx --generate walk -o gen.glb # eval library + qtmesh anim gen.glb --dump-canonical gen.json --animation generated_walk + python3 scripts/compare-motion-clips.py joint-deltas ref.json gen.json + qtmesh isometric ref.fbx --animation mixamo.com -o ref_%02d.png --frames 6 + qtmesh isometric gen.glb --animation generated_walk -o gen_%02d.png --frames 6 + python3 scripts/compare-motion-clips.py sheets "ref_%02d.png" "gen_%02d.png" + +Baselines recorded 2026-07-11 (Mixamo Walk self-retarget, see PR #843): +mean joint error 1.84°, legs ≤0.15°, IoU 0.816 (pipeline self-ceiling 1.0). +""" +import json +import math +import sys + +JN = ["hip", "abdomen", "chest", "neck", "neck1", "head", + "rcollar", "rshoulder", "relbow", "rhand", + "lcollar", "lshoulder", "lelbow", "lhand", + "rbuttock", "rhip", "rknee", "rfoot", + "lbuttock", "lhip", "lknee", "lfoot"] + + +def q_ang(a, b): + return 2 * math.degrees(math.acos(min(1.0, abs(sum(x * y for x, y in zip(a, b)))))) + + +def clip_deltas(path): + c = json.load(open(path))["clips"][0] + q = c["quats"] + return [[q_ang(q[f][j], q[0][j]) for f in range(len(q))] + for j in range(len(JN))] + + +def cmd_joint_deltas(ref_path, gen_path): + A, B = clip_deltas(ref_path), clip_deltas(gen_path) + n = min(len(A[0]), len(B[0])) + print(f"{'joint':<10} {'refAmp':>7} {'genAmp':>7} {'meanErr':>8}") + errs = [] + for j in range(len(JN)): + err = sum(abs(A[j][f] - B[j][f]) for f in range(n)) / n + errs.append(err) + print(f"{JN[j]:<10} {max(A[j][:n]):7.1f} {max(B[j][:n]):7.1f} {err:8.2f}") + print(f"\nTOTAL mean |Δangle| err: {sum(errs) / len(errs):.2f} deg") + + +def cmd_sheets(path_a, path_b, rows=8, grid=64): + from PIL import Image + + def cells(path): + im = Image.open(path).convert("L") + w, h = im.size + ch = h // rows + cols = max(1, round(w / ch)) + cw = w // cols + px = im.load() + out = [] + for r in range(rows): + row = [] + for c in range(cols): + pts = [(x - c * cw, y - r * ch) + for y in range(r * ch, (r + 1) * ch) + for x in range(c * cw, (c + 1) * cw) if px[x, y] > 45] + if not pts: + row.append(frozenset()) + continue + xs = [p[0] for p in pts]; ys = [p[1] for p in pts] + x0, y0 = min(xs), min(ys) + s = min((grid - 1) / max(1, max(xs) - x0), + (grid - 1) / max(1, max(ys) - y0)) + row.append(frozenset((round((x - x0) * s), round((y - y0) * s)) + for x, y in pts)) + out.append(row) + return out + + def iou(a, b): + if not a and not b: + return 1.0 + if not a or not b: + return 0.0 + return len(a & b) / len(a | b) + + A, B = cells(path_a), cells(path_b) + cols = len(A[0]) + scores = [] + for r in range(rows): + best = max(sum(iou(A[r][c], B[r][(c + s) % cols]) + for c in range(cols)) / cols for s in range(cols)) + scores.append(best) + print("per-direction IoU:", " ".join(f"{v:.3f}" for v in scores)) + print(f"MEAN IoU: {sum(scores) / len(scores):.4f}") + + +if __name__ == "__main__": + if len(sys.argv) == 4 and sys.argv[1] == "joint-deltas": + cmd_joint_deltas(sys.argv[2], sys.argv[3]) + elif len(sys.argv) == 4 and sys.argv[1] == "sheets": + cmd_sheets(sys.argv[2], sys.argv[3]) + else: + sys.exit(__doc__) From 09937f7bdcb53c71b5d2aa7cba9def6d4b393f2a Mon Sep 17 00:00:00 2001 From: Fernando Date: Sat, 11 Jul 2026 02:10:35 -0400 Subject: [PATCH 06/10] chore(scripts): record asset dir in the corpus manifest for exact provenance joins Co-Authored-By: Claude Fable 5 --- scripts/build-motion-library-v5.py | 64 ++++++++++++++++++++---------- scripts/scrape-motion-corpus.py | 4 +- 2 files changed, 46 insertions(+), 22 deletions(-) diff --git a/scripts/build-motion-library-v5.py b/scripts/build-motion-library-v5.py index e0804ab28..b56cb0fe4 100644 --- a/scripts/build-motion-library-v5.py +++ b/scripts/build-motion-library-v5.py @@ -42,7 +42,6 @@ import shutil import subprocess import sys -import tempfile CANON_COUNT = 22 FPS = 30 @@ -75,12 +74,20 @@ ] +STOPWORDS = {"armature", "action", "anim", "animation", "animations", + "mixamo", "com", "take", "takes", "fbx", "rig", "rigged", + "character", "model", "mesh", "skeleton", "base", "layer", + "scene", "root", "main", "default", "final", "new", "test"} +# junk that survives normalization but is not an action +BAD_ACTIONS = {"ation", "bot", "jad", "loose", "pose", "still", "static"} + + def norm_anim_name(name): n = name.lower() n = re.sub(r"^.*\|", "", n) # "Armature|Walk" → "walk" - n = re.sub(r"(armature|action|anim|mixamo\.com|takes?)", " ", n) - n = re.sub(r"[^a-z]+", " ", n) - return " ".join(n.split()) + n = re.sub(r"[^a-z]+", " ", n) # squash first, THEN drop + words = [w for w in n.split() if w not in STOPWORDS] + return " ".join(words) def action_for(anim_name, tags): @@ -90,7 +97,9 @@ def action_for(anim_name, tags): return action # None = deliberate skip # single clean word → keep verbatim (widens the prompt vocabulary) words = n.split() - if len(words) == 1 and 3 <= len(words[0]) <= 16: + if len(words) == 1 and 3 <= len(words[0]) <= 16 \ + and words[0] not in BAD_ACTIONS \ + and not any(sw in words[0] for sw in STOPWORDS if len(sw) > 3): return words[0] for t in tags or []: for kw, action in KEYWORDS: @@ -155,6 +164,12 @@ def find_qtmesh(explicit): def manifest_lookup(manifest, dirname): + # Exact join on the recorded on-disk dir (newer manifests), slug-based + # fuzzy fallback for corpora scraped before `dir` was recorded. + for a in manifest.get("assets", []): + d = a.get("dir", "") + if d and (d.endswith("/" + dirname) or d == dirname): + return a for a in manifest.get("assets", []): slug = re.sub(r"[^A-Za-z0-9._-]+", "_", a.get("title", "")).strip("_") if dirname in (slug[:80],) or dirname in slug: @@ -202,23 +217,30 @@ def main(): if not fn.lower().endswith(MODEL_EXTS): continue fpath = os.path.join(root, fn) - with tempfile.NamedTemporaryFile( - suffix=".json", delete=False) as tf: - tmp = tf.name - try: - r = subprocess.run( - [qtmesh, "anim", fpath, "--dump-canonical", tmp], - capture_output=True, text=True, timeout=600) - if r.returncode != 0 or not os.path.getsize(tmp): - continue - dump = json.load(open(tmp)) - except Exception: - continue - finally: + # Sidecar cache: extraction dominates rebuild time, so + # dumps persist next to the model file and are reused + # when newer than it (delete *.canonical.json to force). + cache = fpath + ".canonical.json" + dump = None + if os.path.exists(cache) \ + and os.path.getmtime(cache) >= os.path.getmtime(fpath): try: - os.remove(tmp) - except OSError: - pass + dump = json.load(open(cache)) + except Exception: + dump = None + if dump is None: + try: + r = subprocess.run( + [qtmesh, "anim", fpath, + "--dump-canonical", cache], + capture_output=True, text=True, timeout=600) + if r.returncode != 0 \ + or not os.path.exists(cache) \ + or not os.path.getsize(cache): + continue + dump = json.load(open(cache)) + except Exception: + continue joints = joints or dump.get("joints") for c in dump.get("clips", []): if c.get("resolvedRoles", 0) < args.min_roles: diff --git a/scripts/scrape-motion-corpus.py b/scripts/scrape-motion-corpus.py index 5f59fbd60..cf3f1a09b 100644 --- a/scripts/scrape-motion-corpus.py +++ b/scripts/scrape-motion-corpus.py @@ -392,7 +392,9 @@ def scrape_packs(out_raw, max_per_pack=8): def write_outputs(out_dir, assets): for a in assets: - a.pop("_dir", None) + d = a.pop("_dir", None) + if d: + a["dir"] = os.path.relpath(d, out_dir) # Incremental: merge with any manifest already in the corpus dir so # successive runs (different sources/queries) accumulate; an asset seen # again (same source_url) is replaced by the fresh record. From ecc387fb2948df9ca8d37bed2476f1f10ca0ef70 Mon Sep 17 00:00:00 2001 From: Fernando Date: Sat, 11 Jul 2026 02:57:09 -0400 Subject: [PATCH 07/10] feat(anim): bind-referenced direction-matching retarget (#839) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Generated animations now reference the TARGET RIG'S BIND POSE instead of a standing pose harvested from another animation — the root cause of the "first frame of the previous animation contaminates the generated one" report. How it works (clips extracted with --dump-canonical carry two new per-clip blocks): - restWorld: the SOURCE rig's bind world orientation per canonical joint (conjugated into the canonical frame like the clip quats). - restDir: the source's canonical-topology bind bone DIRECTIONS. At apply time each source bone's constant local direction axis a_s = Ws_bind^-1 * ds_bind is rotated by the clip frame, expressed in the target's raw frame via the target's own bind-geometry conjugation Ct, and the target bone is aimed at exactly that world direction: Wt(f) = arc(dt_bind -> ds(f)) * Wt_bind, propagated hierarchy-ordered. Bind-to-bind delta transplant was rejected: Assimp reset poses are not trustworthy T-poses across importers, but each rig is self-consistent. Key correctness detail: Ogre skeleton keyframes are DELTAS applied multiplicatively onto the binding pose (NodeAnimationTrack::applyToNode rotates the reset bone), so the keyframe writes bindLocal^-1 * local — absolute locals double-compose on rigs with non-identity bind locals (the cartoon fox rig that exposed all of this). Results (eval harness, scripts/compare-motion-clips.py): - Mixamo Walk self-retarget: 2.13 deg mean joint delta, IoU 0.836 (prior baseline 0.816). - Cross-rig fox walk: pose IoU vs reference 0.44 (all standing-path attempts: ~0.25); arms hang naturally, posed bbox w/h 0.57 vs 0.58 ground truth. - v4/CMU clips (no restWorld) are untouched: they still run the standing-pose path, regression-checked on the same rig. Also: canonicalParentOf/canonicalChildOf topology tables (MotionInbetween), MotionLibrary parses optional per-clip restWorld/restDir (+ unit tests), --dump-canonical emits both blocks, and all three generate surfaces (CLI/GUI/MCP) prefer the clip's restWorld over the library-level CMU one. Co-Authored-By: Claude Fable 5 --- src/AnimationControlController.cpp | 11 +- src/AnimationMerger.cpp | 280 ++++++++++++++++++++++++++++- src/AnimationMerger.h | 16 +- src/CLIPipeline.cpp | 26 ++- src/MCPServer.cpp | 9 +- src/MotionInbetween.cpp | 28 +++ src/MotionInbetween.h | 5 + src/MotionLibrary.cpp | 25 +++ src/MotionLibrary.h | 9 + src/MotionLibrary_test.cpp | 39 ++++ 10 files changed, 440 insertions(+), 8 deletions(-) diff --git a/src/AnimationControlController.cpp b/src/AnimationControlController.cpp index 95ee37a34..b6be4e20b 100644 --- a/src/AnimationControlController.cpp +++ b/src/AnimationControlController.cpp @@ -1693,6 +1693,7 @@ QVariantMap AnimationControlController::generateMotion(const QString& prompt, int fps = 30; bool worldFrame = false; std::vector> cmuRest; + std::vector> clipDirs; bool gotClip = false; if (useModel) { @@ -1724,7 +1725,12 @@ QVariantMap AnimationControlController::generateMotion(const QString& prompt, } const MotionLibrary::Clip& clip = lib.clip(idx); quats = clip.quats; fps = clip.fps; - worldFrame = lib.isWorldFrame(); cmuRest = lib.cmuRestWorld(); + worldFrame = lib.isWorldFrame(); + // Prefer the clip's own source-bind orientations (bind-referenced + // retarget); the library-level block is the v2 CMU legacy. + cmuRest = clip.restWorld.empty() ? lib.cmuRestWorld() + : clip.restWorld; + clipDirs = clip.restDir; clipSource = QStringLiteral("template"); if (duration > 0.05) { const int want = std::max(2, int(duration * clip.fps)); @@ -1744,7 +1750,8 @@ QVariantMap AnimationControlController::generateMotion(const QString& prompt, const auto res = AnimationMerger::applyMotionClip(skel.get(), animName, quats, fps, worldFrame, cmuRest, /*refineWithModel=*/false, - /*refineStride=*/8, yaw180); + /*refineStride=*/8, yaw180, + clipDirs); if (!res.ok) return fail(res.error); out["source"] = clipSource; diff --git a/src/AnimationMerger.cpp b/src/AnimationMerger.cpp index cdf599068..8b3bd0105 100644 --- a/src/AnimationMerger.cpp +++ b/src/AnimationMerger.cpp @@ -985,6 +985,56 @@ AnimationMerger::extractCanonicalClips(Ogre::Entity* entity, int fps, } } + // BIND-pose world orientation per canonical role (conjugated into the + // canonical frame) — the reference the bind-referenced retarget deltas + // against. Captured while the skeleton is still at bind from the + // conjugation block above. + std::vector> restWorld( + static_cast(J), {0.f, 0.f, 0.f, 1.f}); + std::vector> restDir( + static_cast(J), {0.f, 0.f, 0.f}); + { + skel->reset(true); + skel->_updateTransforms(); + for (int j = 0; j < J; ++j) { + Ogre::Bone* b = roleBone[static_cast(j)]; + if (!b) continue; + const Ogre::Quaternion w = + C * b->_getDerivedOrientation() * C.Inverse(); + restWorld[static_cast(j)] = { + static_cast(w.x), static_cast(w.y), + static_cast(w.z), static_cast(w.w)}; + } + // Canonical-topology bind bone directions (conjugated by C): role → + // its canonical child joint, leaf roles take the incoming direction. + // The retarget computes the target's directions the same way and + // aligns per bone — the general fix for cross-rig bone-axis + // conventions (identity-rest rigs carry no info in restWorld). + const auto dirBetween = [&](int a, int bIdx, + std::array& out) { + const Ogre::Bone* ba = roleBone[static_cast(a)]; + const Ogre::Bone* bb = roleBone[static_cast(bIdx)]; + if (!ba || !bb) return false; + Ogre::Vector3 v = bb->_getDerivedPosition() + - ba->_getDerivedPosition(); + if (v.squaredLength() < 1e-12f) return false; + v = C * v; + v.normalise(); + out = {static_cast(v.x), static_cast(v.y), + static_cast(v.z)}; + return true; + }; + for (int j = 0; j < J; ++j) { + const int child = MotionInbetween::canonicalChildOf(j); + if (child >= 0 + && dirBetween(j, child, restDir[static_cast(j)])) + continue; + const int parent = MotionInbetween::canonicalParentOf(j); + if (parent >= 0) + dirBetween(parent, j, restDir[static_cast(j)]); + } + } + // Sample by applying each Animation DIRECTLY to the skeleton instance — // deterministic regardless of the entity's animation-state bookkeeping // (state-set application proved instance-dependent for hand-built @@ -1026,6 +1076,8 @@ AnimationMerger::extractCanonicalClips(Ogre::Entity* entity, int fps, clip.quats.push_back(std::move(pose)); } clip.frames = static_cast(clip.quats.size()); + clip.restWorld = restWorld; + clip.restDir = restDir; out.push_back(std::move(clip)); } @@ -1091,7 +1143,8 @@ AnimationMerger::ApplyMotionResult AnimationMerger::applyMotionClip( const std::vector>& cmuRestWorld, bool refineWithModel, int refineStride, - bool yaw180) + bool yaw180, + const std::vector>& clipRestDir) { ApplyMotionResult res; if (!skel) { res.error = QStringLiteral("no skeleton"); return res; } @@ -1181,6 +1234,231 @@ AnimationMerger::ApplyMotionResult AnimationMerger::applyMotionClip( anim->setInterpolationMode(Ogre::Animation::IM_LINEAR); anim->setRotationInterpolationMode(Ogre::Animation::RIM_LINEAR); + // ===== BIND-REFERENCED WORLD RETARGET (preferred when available) ===== + // Clips extracted by --dump-canonical carry the SOURCE rig's bind-pose + // world orientations (restWorld). Both rigs' binds are T-poses, so the + // correspondence source-bind ↔ target-bind is exact: + // Δworld(f) = Ws(f) · Ws_bind⁻¹ (canonical world axes) + // Wt(f) = Δworld(f) · Wt_bind + // local(f) = Wt_parent(f)⁻¹ · Wt(f) (hierarchy-ordered) + // Generated clips therefore reference the TARGET BIND — no pose from any + // other animation is involved (the standing-pose harvest below is only + // the fallback for restWorld-less clips, e.g. the CMU-built libraries). + bool haveRestWorld = false; + if (worldFrame + && cmuRestWorld.size() == static_cast( + MotionInbetween::canonicalJointCount())) { + for (const auto& q : cmuRestWorld) + if (std::abs(q[0]) > 1e-5f || std::abs(q[1]) > 1e-5f + || std::abs(q[2]) > 1e-5f || std::abs(1.f - std::abs(q[3])) > 1e-5f) { + haveRestWorld = true; + break; + } + } + if (haveRestWorld) { + const int Jc = MotionInbetween::canonicalJointCount(); + // Target bind worlds/locals/positions, read at binding pose. + skel->reset(true); + skel->_updateTransforms(); + std::vector bindWorld(static_cast(nBones)); + std::vector bindLocal(static_cast(nBones)); + std::vector bindPos(static_cast(nBones)); + std::vector parentIdx(static_cast(nBones), -1); + for (int i = 0; i < nBones; ++i) { + Ogre::Bone* b = skel->getBone(static_cast(i)); + bindWorld[static_cast(i)] = b->_getDerivedOrientation(); + bindLocal[static_cast(i)] = b->getOrientation(); + bindPos[static_cast(i)] = b->_getDerivedPosition(); + if (auto* p = dynamic_cast(b->getParent())) + parentIdx[static_cast(i)] = p->getHandle(); + } + // Per-role shortest-arc alignment R0: rotate the TARGET's bind bone + // direction onto the SOURCE's (both canonical-topology directions in + // canonical world axes). With Wt(f) = Δs(f)·R0·Wt_bind every target + // bone POINTS where the source bone points each frame, with twist + // transported by Δs — cross-rig bone-axis conventions cancel. + // (restWorld alone degenerates on identity-rest rigs like Mixamo.) + std::vector roleBoneIdx(static_cast(Jc), -1); + for (int i = 0; i < nBones; ++i) + if (boneToCanon[i] >= 0 && boneToCanon[i] < Jc + && roleBoneIdx[static_cast(boneToCanon[i])] < 0) + roleBoneIdx[static_cast(boneToCanon[i])] = i; + auto restQ = [&](int c) { + const auto& q = cmuRestWorld[static_cast(c)]; + return Ogre::Quaternion(q[3], q[0], q[1], q[2]); + }; + // TARGET-frame conjugation Ct: the clip directions live in the + // CANONICAL frame (the extractor conjugates the source by its own + // bind-geometry-derived frame), while this skeleton's bind data is + // in its RAW file frame (Mixamo FBX rigs are commonly rotated). + // Derive Ct exactly like the extractor (up = hip→head, left = + // rhip→lhip) and express canonical directions in target space via + // Ct⁻¹ before aiming. + Ogre::Quaternion Ct = Ogre::Quaternion::IDENTITY; + { + auto rolePos = [&](int role) -> const Ogre::Vector3* { + const int i = (role >= 0 && role < Jc) + ? roleBoneIdx[static_cast(role)] : -1; + return i >= 0 ? &bindPos[static_cast(i)] : nullptr; + }; + const Ogre::Vector3* hip = rolePos(0); + const Ogre::Vector3* head = rolePos(5); + if (!head) head = rolePos(3); + const Ogre::Vector3* lSide = rolePos(19); + const Ogre::Vector3* rSide = rolePos(15); + if (!lSide || !rSide) { lSide = rolePos(11); rSide = rolePos(7); } + if (hip && head && lSide && rSide) { + Ogre::Vector3 up = *head - *hip; + Ogre::Vector3 left = *lSide - *rSide; + if (up.squaredLength() > 1e-12f + && left.squaredLength() > 1e-12f) { + up.normalise(); + left = left - up * left.dotProduct(up); + if (left.squaredLength() > 1e-12f) { + left.normalise(); + const Ogre::Vector3 fwd = left.crossProduct(up); + Ogre::Matrix3 src; + src.SetColumn(0, left); + src.SetColumn(1, up); + src.SetColumn(2, fwd); + Ct = Ogre::Quaternion(src).Inverse(); + Ct.normalise(); + } + } + } + } + const Ogre::Quaternion CtInv = Ct.Inverse(); + + // PER-FRAME DIRECTION MATCHING. "Bind" poses are NOT trustworthy + // T-poses across importers (Assimp's reset pose for Mixamo FBX is + // whatever the file stored), so bind-to-bind delta transplant breaks + // cross-rig. Each rig IS self-consistent though: the source bone's + // constant LOCAL direction axis a_s = Ws_bind⁻¹·ds_bind rotates to + // ds(f) = Ws(f)·a_s each frame, and the target bone is aimed at + // exactly that world direction: Wt(f) = arc(dt_bind → ds(f))·Wt_bind. + // Twist about the bone is intentionally not transported (roll is the + // least visible DoF; the hip's facing-yaw staying locked matches the + // clip-facing design). + std::vector srcLocalAxis( + static_cast(Jc), Ogre::Vector3::ZERO); + std::vector tgtBindDir( + static_cast(Jc), Ogre::Vector3::ZERO); + if (!clipRestDir.empty()) { + auto targetDir = [&](int role) -> Ogre::Vector3 { + const int child = MotionInbetween::canonicalChildOf(role); + const int parent = MotionInbetween::canonicalParentOf(role); + const int a = roleBoneIdx[static_cast(role)]; + if (a >= 0 && child >= 0) { + const int bIdx = roleBoneIdx[static_cast(child)]; + if (bIdx >= 0) { + Ogre::Vector3 v = bindPos[static_cast(bIdx)] + - bindPos[static_cast(a)]; + if (v.squaredLength() > 1e-12f) return v; + } + } + if (a >= 0 && parent >= 0) { + const int pIdx = roleBoneIdx[static_cast(parent)]; + if (pIdx >= 0) { + Ogre::Vector3 v = bindPos[static_cast(a)] + - bindPos[static_cast(pIdx)]; + if (v.squaredLength() > 1e-12f) return v; + } + } + return Ogre::Vector3::ZERO; + }; + for (int c = 0; c < Jc; ++c) { + const auto& sd = clipRestDir[static_cast(c)]; + Ogre::Vector3 srcDir(sd[0], sd[1], sd[2]); + Ogre::Vector3 tgtDir = targetDir(c); + if (srcDir.squaredLength() > 1e-8f + && tgtDir.squaredLength() > 1e-8f) { + srcDir.normalise(); + tgtDir.normalise(); + srcLocalAxis[static_cast(c)] = + restQ(c).Inverse() * srcDir; + tgtBindDir[static_cast(c)] = tgtDir; + } + } + } + // Hierarchy order: parents before children (indices may not be). + std::vector order; + order.reserve(static_cast(nBones)); + { + std::vector placed(static_cast(nBones), 0); + bool progress = true; + while (static_cast(order.size()) < nBones && progress) { + progress = false; + for (int i = 0; i < nBones; ++i) { + if (placed[static_cast(i)]) continue; + const int pi = parentIdx[static_cast(i)]; + if (pi < 0 || placed[static_cast(pi)]) { + order.push_back(i); + placed[static_cast(i)] = 1; + progress = true; + } + } + } + for (int i = 0; i < nBones; ++i) // cycles/orphans: append + if (!placed[static_cast(i)]) order.push_back(i); + } + auto clipQ = [&](int frame, int joint) -> Ogre::Quaternion { + const auto& q = clipQuats[frame][joint]; + return Ogre::Quaternion(q[3], q[0], q[1], q[2]); // (w,x,y,z) + }; + std::vector tracks( + static_cast(nBones), nullptr); + for (int i = 0; i < nBones; ++i) + if (boneToCanon[i] >= 0 && boneToCanon[i] < Jc) { + tracks[static_cast(i)] = anim->createNodeTrack( + static_cast(i), + skel->getBone(static_cast(i))); + ++res.tracksWritten; + } + std::vector W(static_cast(nBones)); + for (int f = 0; f < frames; ++f) { + for (int i : order) { + const int pi = parentIdx[static_cast(i)]; + const Ogre::Quaternion Wp = (pi >= 0) + ? W[static_cast(pi)] : Ogre::Quaternion::IDENTITY; + const int c = boneToCanon[i]; + Ogre::Quaternion local; + if (c >= 0 && c < Jc + && srcLocalAxis[static_cast(c)].squaredLength() + > 1e-8f) { + Ogre::Vector3 dCan = + clipQ(f, c) * srcLocalAxis[static_cast(c)]; + if (yaw180) { dCan.x = -dCan.x; dCan.z = -dCan.z; } + const Ogre::Vector3 ds = CtInv * dCan; + const Ogre::Quaternion R = + tgtBindDir[static_cast(c)].getRotationTo(ds); + const Ogre::Quaternion Wt = + R * bindWorld[static_cast(i)]; + local = Wp.Inverse() * Wt; + W[static_cast(i)] = Wt; + } else { + local = bindLocal[static_cast(i)]; + W[static_cast(i)] = Wp * local; + } + if (auto* trk = tracks[static_cast(i)]) { + // Ogre skeleton keyframes are DELTAS applied onto the + // binding pose (NodeAnimationTrack::applyToNode rotates + // the reset bone) — convert the absolute local target. + Ogre::TransformKeyFrame* kf = + trk->createNodeKeyFrame(f * dt); + kf->setRotation( + bindLocal[static_cast(i)].Inverse() * local); + kf->setTranslate(Ogre::Vector3::ZERO); + kf->setScale(Ogre::Vector3::UNIT_SCALE); + } + } + } + res.ok = true; + res.canonicalJoints = distinct; + res.frames = frames; + res.length = length; + return res; + } + // The rig's natural STANDING pose. Mixamo skeletons have an identity bone // rest pose — the real standing orientation lives in frame 0 of their // existing animation, not in the bone transform. So harvest each bone's diff --git a/src/AnimationMerger.h b/src/AnimationMerger.h index d4b8bbc52..b2d1fb521 100644 --- a/src/AnimationMerger.h +++ b/src/AnimationMerger.h @@ -180,7 +180,8 @@ class AnimationMerger { const std::vector>& cmuRestWorld = {}, bool refineWithModel = false, int refineStride = 8, - bool yaw180 = false); + bool yaw180 = false, + const std::vector>& clipRestDir = {}); /// One skeletal animation extracted onto the 22-joint canonical skeleton /// (#839, the REVERSE of applyMotionClip's world-frame path): per frame, @@ -193,6 +194,19 @@ class AnimationMerger { int resolvedRoles = 0; ///< canonical roles matched on this rig (≤22) /// frames × 22 × [x,y,z,w]; unresolved roles hold identity. std::vector>> quats; + /// The SOURCE rig's BIND-pose world orientation per canonical role + /// (same conjugated frame as `quats`). Enables the bind-referenced + /// retarget: world deltas are taken against the source bind and + /// applied onto the target bind — no pose from any other animation + /// is ever involved. 22 × [x,y,z,w]; identity for unresolved roles. + std::vector> restWorld; + /// Canonical-frame BIND bone directions per role (unit vectors, + /// canonical topology: role → its canonical child joint; leaf roles + /// use the incoming direction). The target computes its own bind + /// directions the same way; a shortest-arc alignment between the two + /// makes every retargeted bone POINT where the source bone points. + /// 22 × [x,y,z]; zero for unresolved roles. + std::vector> restDir; }; /// Sample every (or one) skeletal animation of `entity` at `fps` and diff --git a/src/CLIPipeline.cpp b/src/CLIPipeline.cpp index edda89de5..8894ce632 100644 --- a/src/CLIPipeline.cpp +++ b/src/CLIPipeline.cpp @@ -2003,6 +2003,7 @@ int CLIPipeline::cmdAnimGenerate(const QString& filePath, const QString& prompt, int fps = 30; bool worldFrame = false; std::vector> cmuRest; // template-only (model has none) + std::vector> clipDirs; QString clipSource; bool gotClip = false; @@ -2052,7 +2053,9 @@ int CLIPipeline::cmdAnimGenerate(const QString& filePath, const QString& prompt, quats = clip.quats; fps = clip.fps; worldFrame = lib.isWorldFrame(); - cmuRest = lib.cmuRestWorld(); + cmuRest = clip.restWorld.empty() ? lib.cmuRestWorld() + : clip.restWorld; + clipDirs = clip.restDir; clipSource = QStringLiteral("template"); // Optionally retime the clip to a requested duration by frame stride/pad. if (duration > 0.05f) { @@ -2085,7 +2088,8 @@ int CLIPipeline::cmdAnimGenerate(const QString& filePath, const QString& prompt, auto res = AnimationMerger::applyMotionClip(skel.get(), animName, quats, fps, worldFrame, cmuRest, /*refineWithModel=*/false, - /*refineStride=*/8, yaw180); + /*refineStride=*/8, yaw180, + clipDirs); if (!res.ok) { err() << "Error: " << res.error << Qt::endl; return 1; } @@ -2417,6 +2421,24 @@ int CLIPipeline::cmdAnim(int argc, char* argv[]) frames.append(row); } co["quats"] = frames; + QJsonArray rest; + for (const auto& q : c.restWorld) { + QJsonArray quat; + for (float v : q) + quat.append(static_cast( + std::round(v * 100000.0f) / 100000.0f)); + rest.append(quat); + } + co["restWorld"] = rest; + QJsonArray dirs; + for (const auto& v : c.restDir) { + QJsonArray vec; + for (float x : v) + vec.append(static_cast( + std::round(x * 100000.0f) / 100000.0f)); + dirs.append(vec); + } + co["restDir"] = dirs; clipArr.append(co); } root["clips"] = clipArr; diff --git a/src/MCPServer.cpp b/src/MCPServer.cpp index ab10b0622..0c4279c56 100644 --- a/src/MCPServer.cpp +++ b/src/MCPServer.cpp @@ -4039,6 +4039,7 @@ QJsonObject MCPServer::toolGenerateMotion(const QJsonObject &args) std::vector>> quats; int fps = 30; bool worldFrame = false; std::vector> cmuRest; + std::vector> clipDirs; bool gotClip = false; if (useModel) { @@ -4068,7 +4069,10 @@ QJsonObject MCPServer::toolGenerateMotion(const QJsonObject &args) } const MotionLibrary::Clip& clip = lib.clip(idx); quats = clip.quats; fps = clip.fps; - worldFrame = lib.isWorldFrame(); cmuRest = lib.cmuRestWorld(); + worldFrame = lib.isWorldFrame(); + cmuRest = clip.restWorld.empty() ? lib.cmuRestWorld() + : clip.restWorld; + clipDirs = clip.restDir; clipSource = QStringLiteral("template"); if (duration > 0.05) { const int want = std::max(2, int(duration * clip.fps)); @@ -4088,7 +4092,8 @@ QJsonObject MCPServer::toolGenerateMotion(const QJsonObject &args) const auto r = AnimationMerger::applyMotionClip(skel.get(), animName, quats, fps, worldFrame, cmuRest, /*refineWithModel=*/false, - /*refineStride=*/8, yaw180); + /*refineStride=*/8, yaw180, + clipDirs); if (!r.ok) return makeErrorResult(QString("Error: %1").arg(r.error)); entity->refreshAvailableAnimationState(); diff --git a/src/MotionInbetween.cpp b/src/MotionInbetween.cpp index ef3d3dcb6..e612adcc8 100644 --- a/src/MotionInbetween.cpp +++ b/src/MotionInbetween.cpp @@ -74,6 +74,34 @@ char sideOf(const QString& n) int MotionInbetween::canonicalJointCount() { return kCanonCount; } +namespace { +// hip abdomen chest neck neck1 head | rcollar rshoulder relbow rhand | +// lcollar lshoulder lelbow lhand | rbuttock rhip rknee rfoot | +// lbuttock lhip lknee lfoot +constexpr int kCanonParent[22] = { + -1, 0, 1, 2, 3, 4, + 2, 6, 7, 8, + 2, 10, 11, 12, + 0, 14, 15, 16, + 0, 18, 19, 20}; +constexpr int kCanonChild[22] = { + 1, 2, 3, 4, 5, -1, + 7, 8, 9, -1, + 11, 12, 13, -1, + 15, 16, 17, -1, + 19, 20, 21, -1}; +} // namespace + +int MotionInbetween::canonicalParentOf(int i) +{ + return (i >= 0 && i < kCanonCount) ? kCanonParent[i] : -1; +} + +int MotionInbetween::canonicalChildOf(int i) +{ + return (i >= 0 && i < kCanonCount) ? kCanonChild[i] : -1; +} + QString MotionInbetween::canonicalJointName(int i) { if (i < 0 || i >= kCanonCount) return {}; diff --git a/src/MotionInbetween.h b/src/MotionInbetween.h index 44f01fea4..2babcf9e5 100644 --- a/src/MotionInbetween.h +++ b/src/MotionInbetween.h @@ -112,6 +112,11 @@ class MotionInbetween { static int canonicalJointCount(); // 22 // Canonical joint name at index i (CMU core-body name), 0..count-1. static QString canonicalJointName(int i); + /// Canonical skeleton topology (22 joints): parent index (-1 for hip) + /// and the primary child used for bone-direction computation (-1 for + /// leaf joints: head, hands, feet). + static int canonicalParentOf(int i); + static int canonicalChildOf(int i); // Map an arbitrary skeleton bone name to a canonical joint index, or -1 if // it doesn't correspond to one of the 22 roles. Case-insensitive; tolerates // common prefixes (mixamorig:, mixamorig1:, bip01 …) and side spellings diff --git a/src/MotionLibrary.cpp b/src/MotionLibrary.cpp index 9803d60d2..6d3719c7d 100644 --- a/src/MotionLibrary.cpp +++ b/src/MotionLibrary.cpp @@ -120,6 +120,31 @@ bool MotionLibrary::parse(const QByteArray& json) clip.quats.push_back(std::move(pose)); } clip.frames = static_cast(clip.quats.size()); + // Optional per-clip source-bind orientations (bind-referenced + // retarget). Malformed/absent → empty, the standing-pose path runs. + const QJsonArray rest = co.value("restWorld").toArray(); + if (rest.size() == kCanonJoints) { + clip.restWorld.reserve(kCanonJoints); + for (int j = 0; j < kCanonJoints; ++j) { + const QJsonArray q = rest[j].toArray(); + clip.restWorld.push_back({ + static_cast(q.at(0).toDouble()), + static_cast(q.at(1).toDouble()), + static_cast(q.at(2).toDouble()), + static_cast(q.at(3).toDouble(1.0))}); + } + } + const QJsonArray rdir = co.value("restDir").toArray(); + if (rdir.size() == kCanonJoints) { + clip.restDir.reserve(kCanonJoints); + for (int j = 0; j < kCanonJoints; ++j) { + const QJsonArray v = rdir[j].toArray(); + clip.restDir.push_back({ + static_cast(v.at(0).toDouble()), + static_cast(v.at(1).toDouble()), + static_cast(v.at(2).toDouble())}); + } + } if (clip.frames > 0 && !clip.action.isEmpty()) m_clips.push_back(std::move(clip)); } diff --git a/src/MotionLibrary.h b/src/MotionLibrary.h index 19366bbc9..1653b9edb 100644 --- a/src/MotionLibrary.h +++ b/src/MotionLibrary.h @@ -35,6 +35,15 @@ class MotionLibrary { // order (size frames × 22). Rotation only; translation/scale are the // caller's (the retarget writes rotation keyframes). std::vector>> quats; + // Optional (schema v3 clips extracted by --dump-canonical): the + // SOURCE rig's bind-pose world orientation per canonical joint. + // Present → the retarget runs bind-referenced (deltas vs the source + // bind onto the target bind); absent (CMU-built libraries) → the + // standing-pose path. + std::vector> restWorld; + // Optional: canonical-frame bind bone directions (22 × [x,y,z]) — + // enables the direction-aligned bind-referenced retarget. + std::vector> restDir; }; MotionLibrary() = default; diff --git a/src/MotionLibrary_test.cpp b/src/MotionLibrary_test.cpp index 3795807b6..66d8c2e61 100644 --- a/src/MotionLibrary_test.cpp +++ b/src/MotionLibrary_test.cpp @@ -93,6 +93,45 @@ TEST(MotionLibrary, IgnoresWrongSizedCmuRest) EXPECT_FALSE(lib.hasCmuRest()); } +TEST(MotionLibrary, ParsesPerClipRestWorldAndRestDir) +{ + // v3 clips extracted by --dump-canonical carry per-clip restWorld (22 + // source-bind quats) + restDir (22 canonical bind bone directions) that + // switch the retarget to the bind-referenced path. Absent → empty. + MotionLibrary plain; + ASSERT_TRUE(plain.loadFromJson(miniLib())) << plain.error().toStdString(); + EXPECT_TRUE(plain.clip(0).restWorld.empty()); + EXPECT_TRUE(plain.clip(0).restDir.empty()); + + QByteArray rest = ",\"restWorld\":["; + for (int j = 0; j < 22; ++j) rest += (j ? ",[0,0,0.5,0.866]" : "[0,0,0.5,0.866]"); + rest += "],\"restDir\":["; + for (int j = 0; j < 22; ++j) rest += (j ? ",[0,1,0]" : "[0,1,0]"); + rest += "]"; + QByteArray v3 = miniLib(); + const int at = v3.indexOf("\"quats\""); // first (walk) clip + const int end = v3.indexOf("},", at); // its closing brace + ASSERT_GT(end, 0); + v3.insert(end, rest); + MotionLibrary lib; + ASSERT_TRUE(lib.loadFromJson(v3)) << lib.error().toStdString(); + ASSERT_EQ(lib.clip(0).restWorld.size(), 22u); + ASSERT_EQ(lib.clip(0).restDir.size(), 22u); + EXPECT_NEAR(lib.clip(0).restWorld[3][2], 0.5f, 1e-6f); + EXPECT_NEAR(lib.clip(0).restDir[3][1], 1.0f, 1e-6f); + EXPECT_TRUE(lib.clip(1).restWorld.empty()); // only the walk clip got it + + // Wrong-sized arrays are ignored, clip still loads. + QByteArray badSz = miniLib(); + badSz.insert(badSz.indexOf("},", badSz.indexOf("\"quats\"")), + ",\"restWorld\":[[0,0,0,1]],\"restDir\":[[0,1,0]]"); + MotionLibrary lib2; + ASSERT_TRUE(lib2.loadFromJson(badSz)) << lib2.error().toStdString(); + EXPECT_TRUE(lib2.clip(0).restWorld.empty()); + EXPECT_TRUE(lib2.clip(0).restDir.empty()); + EXPECT_EQ(lib2.clipCount(), 2); +} + TEST(MotionLibrary, MatchesDirectActionWord) { MotionLibrary lib; From 6560b10331533e2c8bdb9d0c000e7db000b6b7f1 Mon Sep 17 00:00:00 2001 From: Fernando Date: Sat, 11 Jul 2026 02:57:09 -0400 Subject: [PATCH 08/10] feat(scripts): propagate restWorld/restDir into v5 library clips MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The builder now copies the per-clip source-bind orientation + bind bone direction blocks from the --dump-canonical sidecars into the library JSON, lighting up the bind-referenced retarget for every harvested clip. Stale sidecar caches predate these fields — delete *.canonical.json to re-extract. Co-Authored-By: Claude Fable 5 --- scripts/build-motion-library-v5.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/scripts/build-motion-library-v5.py b/scripts/build-motion-library-v5.py index b56cb0fe4..8988bd23a 100644 --- a/scripts/build-motion-library-v5.py +++ b/scripts/build-motion-library-v5.py @@ -270,12 +270,21 @@ def main(): continue seen.add(fp); seen.add(sem) counts[action] = counts.get(action, 0) + 1 - clips.append({ + clip = { "action": action, "source": f"{title} — {c.get('animation')}", "frames": len(w), "quats": w, - }) + } + # Source-bind orientations + canonical bind bone + # directions → the bind-referenced retarget path. + # Older sidecar caches predate these fields; delete + # *.canonical.json to re-extract. + if len(c.get("restWorld", [])) == CANON_COUNT: + clip["restWorld"] = c["restWorld"] + if len(c.get("restDir", [])) == CANON_COUNT: + clip["restDir"] = c["restDir"] + clips.append(clip) print(f" + {action:<10} {title[:38]:<40}" f" {c.get('animation')} ({len(w)}f)") From b22d7437f9af1d6170afa009977eccd65ebdd25c Mon Sep 17 00:00:00 2001 From: Fernando Date: Sat, 11 Jul 2026 11:10:37 -0400 Subject: [PATCH 09/10] fix(anim): measure the clip reference frame at an animated calm frame MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two fixes surfaced by retargeting the full harvested library onto a cartoon rig: 1. Extractor: restWorld/restDir/conjugation are now all measured at the clip's calmest ANIMATED frame instead of the bind/reset pose. On many scraped rigs (Quaternius, Sketchfab glTF) the animation worlds differ from the reset pose by a constant global rotation (an armature-node +/-90 deg X that Assimp bakes into one but not the other), so a bind-measured reference tipped the retargeted body horizontal. One measurement frame makes the (restWorld, restDir, quats) triple consistent by construction — the global offset cancels exactly. Verified: samurai dance / Quaternius knight jump / swim idle now retarget upright; Mixamo self-parity holds at 2.2 deg. 2. Retarget: yaw180 is no longer applied in the direction-matching path. That path anchors facing to the TARGET's own bind (clips are canonical +Z-facing), so the flip aimed collars/arms near-anti-parallel to their bind directions and destabilised getRotationTo (self-parity 2.1 -> 6.9 deg with wild collar swings). The flag keeps its meaning for the legacy standing-pose transport. Co-Authored-By: Claude Fable 5 --- src/AnimationMerger.cpp | 165 +++++++++++++++++++++++++--------------- src/AnimationMerger.h | 23 +++--- 2 files changed, 117 insertions(+), 71 deletions(-) diff --git a/src/AnimationMerger.cpp b/src/AnimationMerger.cpp index 8b3bd0105..6adcd4b1d 100644 --- a/src/AnimationMerger.cpp +++ b/src/AnimationMerger.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include @@ -943,13 +944,22 @@ AnimationMerger::extractCanonicalClips(Ogre::Entity* entity, int fps, // Scraped rigs live in arbitrary file frames (Blender FBX armatures are // commonly Z-up), while the motion library's world convention is Y-up, // +Z-facing, +X-left. Guessing per format is fragile — derive the - // source frame from the rig's own BIND geometry instead: up = hip→head, + // source frame from the rig's own geometry instead: up = hip→head, // left = right-hip→left-hip (shoulders as fallback), forward = left×up. // Every sampled world quat is then conjugated: q' = C · q · C⁻¹. - Ogre::Quaternion C = Ogre::Quaternion::IDENTITY; - { - skel->reset(true); - skel->_updateTransforms(); + // + // CRITICAL: the frame, reference orientations AND bone directions are + // all measured at an ANIMATED calm frame of the clip — never at the + // bind/reset pose. On many scraped rigs (Quaternius, Sketchfab glTF) + // the animation worlds differ from the reset pose by a constant global + // rotation (an armature-node ±90° X that Assimp bakes into one but not + // the other), so mixing bind-measured references with animated quats + // tips the retargeted body over. Measuring everything at one animated + // frame makes the (restWorld, restDir, quats) triple consistent by + // construction — the global offset cancels exactly. + const auto deriveFrame = [&]() -> Ogre::Quaternion { + // Reads the skeleton's CURRENT (applied) pose. + Ogre::Quaternion frameC = Ogre::Quaternion::IDENTITY; auto posOf = [&](int role) -> const Ogre::Bone* { return (role >= 0 && role < J) ? roleBone[static_cast(role)] : nullptr; @@ -978,40 +988,94 @@ AnimationMerger::extractCanonicalClips(Ogre::Entity* entity, int fps, src.SetColumn(0, left); src.SetColumn(1, up); src.SetColumn(2, fwd); - C = Ogre::Quaternion(src).Inverse(); - C.normalise(); + frameC = Ogre::Quaternion(src).Inverse(); + frameC.normalise(); } } } - } + return frameC; + }; - // BIND-pose world orientation per canonical role (conjugated into the - // canonical frame) — the reference the bind-referenced retarget deltas - // against. Captured while the skeleton is still at bind from the - // conjugation block above. - std::vector> restWorld( - static_cast(J), {0.f, 0.f, 0.f, 1.f}); - std::vector> restDir( - static_cast(J), {0.f, 0.f, 0.f}); - { + // Sample by applying each Animation DIRECTLY to the skeleton instance — + // deterministic regardless of the entity's animation-state bookkeeping + // (state-set application proved instance-dependent for hand-built + // skeletons), and it leaves the entity's enabled states untouched. + for (unsigned short a = 0; a < skel->getNumAnimations(); ++a) { + Ogre::Animation* anim = skel->getAnimation(a); + if (!anim) continue; + const QString name = QString::fromStdString(anim->getName()); + if (!onlyAnimation.isEmpty() + && name.compare(onlyAnimation, Qt::CaseInsensitive) != 0) + continue; + const float length = anim->getLength(); + if (length <= 0.0f) + continue; + + CanonicalClip clip; + clip.animation = name; + clip.resolvedRoles = resolved; + const int frames = + std::max(2, static_cast(std::lround(length * fps)) + 1); + + // Pass 1: sample RAW world orientations per frame per role. + std::vector> raw( + static_cast(frames), + std::vector(static_cast(J))); + for (int f = 0; f < frames; ++f) { + skel->reset(true); + anim->apply(skel, std::min(length, + static_cast(f) / static_cast(fps))); + skel->_updateTransforms(); + for (int j = 0; j < J; ++j) + if (Ogre::Bone* b = roleBone[static_cast(j)]) + raw[static_cast(f)][static_cast(j)] = + b->_getDerivedOrientation(); + } + + // Calm reference frame f*: minimum mean joint rotation speed — + // typically a near-neutral lead-in/contact pose, the most reliable + // place to read body geometry (the same heuristic the standing-pose + // harvest and the library builder's window snap use). + int fStar = 0; + double bestE = std::numeric_limits::max(); + for (int f = 1; f < frames; ++f) { + double e = 0.0; + for (int j = 0; j < J; ++j) { + if (!roleBone[static_cast(j)]) continue; + const Ogre::Quaternion d = + raw[static_cast(f - 1)][static_cast(j)] + .Inverse() + * raw[static_cast(f)][static_cast(j)]; + e += 2.0 * std::acos(std::min(1.0, + std::abs(static_cast(d.w)))); + } + if (e < bestE) { bestE = e; fStar = f - 1; } + } + + // Measure the reference triple at f*: canonical frame C, reference + // world orientation and bone directions — all from the SAME applied + // pose, so they share one frame with the sampled quats. skel->reset(true); + anim->apply(skel, std::min(length, + static_cast(fStar) / static_cast(fps))); skel->_updateTransforms(); + const Ogre::Quaternion C = deriveFrame(); + const Ogre::Quaternion Cinv = C.Inverse(); + std::vector> restWorld( + static_cast(J), {0.f, 0.f, 0.f, 1.f}); + std::vector> restDir( + static_cast(J), {0.f, 0.f, 0.f}); for (int j = 0; j < J; ++j) { Ogre::Bone* b = roleBone[static_cast(j)]; if (!b) continue; const Ogre::Quaternion w = - C * b->_getDerivedOrientation() * C.Inverse(); + C * b->_getDerivedOrientation() * Cinv; restWorld[static_cast(j)] = { static_cast(w.x), static_cast(w.y), static_cast(w.z), static_cast(w.w)}; } - // Canonical-topology bind bone directions (conjugated by C): role → - // its canonical child joint, leaf roles take the incoming direction. - // The retarget computes the target's directions the same way and - // aligns per bone — the general fix for cross-rig bone-axis - // conventions (identity-rest rigs carry no info in restWorld). const auto dirBetween = [&](int a, int bIdx, - std::array& out) { + std::array& outDir) { const Ogre::Bone* ba = roleBone[static_cast(a)]; const Ogre::Bone* bb = roleBone[static_cast(bIdx)]; if (!ba || !bb) return false; @@ -1020,8 +1084,8 @@ AnimationMerger::extractCanonicalClips(Ogre::Entity* entity, int fps, if (v.squaredLength() < 1e-12f) return false; v = C * v; v.normalise(); - out = {static_cast(v.x), static_cast(v.y), - static_cast(v.z)}; + outDir = {static_cast(v.x), static_cast(v.y), + static_cast(v.z)}; return true; }; for (int j = 0; j < J; ++j) { @@ -1033,42 +1097,17 @@ AnimationMerger::extractCanonicalClips(Ogre::Entity* entity, int fps, if (parent >= 0) dirBetween(parent, j, restDir[static_cast(j)]); } - } - - // Sample by applying each Animation DIRECTLY to the skeleton instance — - // deterministic regardless of the entity's animation-state bookkeeping - // (state-set application proved instance-dependent for hand-built - // skeletons), and it leaves the entity's enabled states untouched. - for (unsigned short a = 0; a < skel->getNumAnimations(); ++a) { - Ogre::Animation* anim = skel->getAnimation(a); - if (!anim) continue; - const QString name = QString::fromStdString(anim->getName()); - if (!onlyAnimation.isEmpty() - && name.compare(onlyAnimation, Qt::CaseInsensitive) != 0) - continue; - const float length = anim->getLength(); - if (length <= 0.0f) - continue; - CanonicalClip clip; - clip.animation = name; - clip.resolvedRoles = resolved; - const int frames = - std::max(2, static_cast(std::lround(length * fps)) + 1); + // Pass 2: conjugate the stored raw worlds into the canonical frame. clip.quats.reserve(static_cast(frames)); - for (int f = 0; f < frames; ++f) { - skel->reset(true); - anim->apply(skel, std::min(length, - static_cast(f) / static_cast(fps))); - skel->_updateTransforms(); std::vector> pose( static_cast(J), {0.f, 0.f, 0.f, 1.f}); for (int j = 0; j < J; ++j) { - Ogre::Bone* b = roleBone[static_cast(j)]; - if (!b) continue; + if (!roleBone[static_cast(j)]) continue; const Ogre::Quaternion w = - C * b->_getDerivedOrientation() * C.Inverse(); + C * raw[static_cast(f)][static_cast(j)] + * Cinv; pose[static_cast(j)] = { static_cast(w.x), static_cast(w.y), static_cast(w.z), static_cast(w.w)}; @@ -1076,8 +1115,8 @@ AnimationMerger::extractCanonicalClips(Ogre::Entity* entity, int fps, clip.quats.push_back(std::move(pose)); } clip.frames = static_cast(clip.quats.size()); - clip.restWorld = restWorld; - clip.restDir = restDir; + clip.restWorld = std::move(restWorld); + clip.restDir = std::move(restDir); out.push_back(std::move(clip)); } @@ -1425,10 +1464,14 @@ AnimationMerger::ApplyMotionResult AnimationMerger::applyMotionClip( if (c >= 0 && c < Jc && srcLocalAxis[static_cast(c)].squaredLength() > 1e-8f) { - Ogre::Vector3 dCan = - clipQ(f, c) * srcLocalAxis[static_cast(c)]; - if (yaw180) { dCan.x = -dCan.x; dCan.z = -dCan.z; } - const Ogre::Vector3 ds = CtInv * dCan; + // NB: yaw180 is deliberately NOT applied here — this + // path anchors facing to the TARGET's own bind (the clip + // is canonical +Z-facing by construction), so a flip + // would swing every aim near-anti-parallel to its bind + // direction and destabilise getRotationTo. The flag only + // matters for the legacy standing-pose transport below. + const Ogre::Vector3 ds = CtInv * + (clipQ(f, c) * srcLocalAxis[static_cast(c)]); const Ogre::Quaternion R = tgtBindDir[static_cast(c)].getRotationTo(ds); const Ogre::Quaternion Wt = diff --git a/src/AnimationMerger.h b/src/AnimationMerger.h index b2d1fb521..8b4f24c1c 100644 --- a/src/AnimationMerger.h +++ b/src/AnimationMerger.h @@ -194,18 +194,21 @@ class AnimationMerger { int resolvedRoles = 0; ///< canonical roles matched on this rig (≤22) /// frames × 22 × [x,y,z,w]; unresolved roles hold identity. std::vector>> quats; - /// The SOURCE rig's BIND-pose world orientation per canonical role - /// (same conjugated frame as `quats`). Enables the bind-referenced - /// retarget: world deltas are taken against the source bind and - /// applied onto the target bind — no pose from any other animation + /// The SOURCE rig's REFERENCE world orientation per canonical role, + /// measured at the clip's calmest ANIMATED frame (same conjugated + /// frame as `quats` — never the bind/reset pose, which on many + /// scraped rigs differs from the animation worlds by a constant + /// global armature rotation). Enables the bind-referenced retarget + /// onto the TARGET's bind — no pose from any other target animation /// is ever involved. 22 × [x,y,z,w]; identity for unresolved roles. std::vector> restWorld; - /// Canonical-frame BIND bone directions per role (unit vectors, - /// canonical topology: role → its canonical child joint; leaf roles - /// use the incoming direction). The target computes its own bind - /// directions the same way; a shortest-arc alignment between the two - /// makes every retargeted bone POINT where the source bone points. - /// 22 × [x,y,z]; zero for unresolved roles. + /// Canonical-frame bone directions per role at the same reference + /// frame (unit vectors, canonical topology: role → its canonical + /// child joint; leaf roles use the incoming direction). Combined + /// with restWorld this gives the source bone's constant LOCAL + /// direction axis; the target computes its own bind directions the + /// same way, so every retargeted bone POINTS where the source bone + /// points each frame. 22 × [x,y,z]; zero for unresolved roles. std::vector> restDir; }; From 73033aa5750cc80d347578d634a894b87c686ddb Mon Sep 17 00:00:00 2001 From: Fernando Date: Sat, 11 Jul 2026 12:37:02 -0400 Subject: [PATCH 10/10] fix(anim): bind-referenced base pose for MODEL-generated clips MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The trained-model path ("Use trained model" in the GUI, --model on the CLI) still contaminated generations with the rig's other animations: the model emits no reference triple, so its clips rode the legacy transport whose standing pose is harvested from the first authored animation — on the Hip Hop fox every generated walk started from the dance stance. Now the retarget SYNTHESIZES the standing pose instead: callers borrow a template clip's canonical bone directions (new MotionLibrary::referenceDirsForPrompt — deterministic, prefers the prompt's action and the most complete direction set, local library only) and applyMotionClip aims every bone from the TARGET'S BIND at those directions — the same math as the direction retarget at its reference frame, so the base pose is bind-referenced by construction. The harvest only runs when no synthetic pose could be built (v1/v2 CMU libraries, missing local library), preserving the old behavior there. The target-side bind reader (bind worlds/locals/positions, role map, Ct conjugation, per-role bind directions, hierarchy order) is extracted into a shared file-local helper used by both the direction retarget and the synthetic pose; the direction path's math is unchanged (self-parity holds at 2.22 deg, legacy v4 regression-checked). Verified: model walk on the Hip Hop fox and on a Mixamo rig both start upright with arms at the sides; template + v4 paths unchanged. Co-Authored-By: Claude Fable 5 --- src/AnimationControlController.cpp | 4 + src/AnimationMerger.cpp | 462 +++++++++++++++++------------ src/CLIPipeline.cpp | 5 + src/MCPServer.cpp | 4 + src/MotionLibrary.cpp | 35 +++ src/MotionLibrary.h | 10 + 6 files changed, 327 insertions(+), 193 deletions(-) diff --git a/src/AnimationControlController.cpp b/src/AnimationControlController.cpp index b6be4e20b..c725659c0 100644 --- a/src/AnimationControlController.cpp +++ b/src/AnimationControlController.cpp @@ -1704,6 +1704,10 @@ QVariantMap AnimationControlController::generateMotion(const QString& prompt, if (mr.ok) { action = mr.matchedAction; quats = mr.clip.quats; fps = mr.clip.fps; worldFrame = mr.worldFrame; clipSource = QStringLiteral("model"); gotClip = true; + // Borrow a template clip's reference directions so the + // retarget synthesizes a BIND-referenced base pose (no + // harvest from the rig's other animations). + clipDirs = MotionLibrary::referenceDirsForPrompt(prompt); } } if (!gotClip) diff --git a/src/AnimationMerger.cpp b/src/AnimationMerger.cpp index 6adcd4b1d..31b745b5e 100644 --- a/src/AnimationMerger.cpp +++ b/src/AnimationMerger.cpp @@ -1173,6 +1173,132 @@ bool AnimationMerger::detectBackwardFacing(Ogre::Entity* entity) return dz < -0.02f * h; } +namespace { +// Target-side bind data for the canonical retarget paths: per-bone bind +// worlds/locals/positions, parent indices, a parents-before-children +// traversal order, the first bone per canonical role, the target's +// raw-frame → canonical conjugation Ct (same derivation as the extractor: +// up = hip→head, left = rhip→lhip from bind positions), and the target's +// bind bone direction per role (canonical topology: role → child, leaf +// roles take the incoming direction). +struct TargetBindFrame { + std::vector bindWorld, bindLocal; + std::vector bindPos; + std::vector parentIdx, roleBoneIdx, order; + Ogre::Quaternion Ct = Ogre::Quaternion::IDENTITY; + std::vector tgtBindDir; +}; + +TargetBindFrame readTargetBindFrame(Ogre::Skeleton* skel, + const std::vector& boneToCanon) +{ + const int nBones = static_cast(skel->getNumBones()); + const int Jc = MotionInbetween::canonicalJointCount(); + TargetBindFrame tb; + tb.bindWorld.resize(static_cast(nBones)); + tb.bindLocal.resize(static_cast(nBones)); + tb.bindPos.resize(static_cast(nBones)); + tb.parentIdx.assign(static_cast(nBones), -1); + tb.roleBoneIdx.assign(static_cast(Jc), -1); + tb.tgtBindDir.assign(static_cast(Jc), Ogre::Vector3::ZERO); + skel->reset(true); + skel->_updateTransforms(); + for (int i = 0; i < nBones; ++i) { + Ogre::Bone* b = skel->getBone(static_cast(i)); + tb.bindWorld[static_cast(i)] = b->_getDerivedOrientation(); + tb.bindLocal[static_cast(i)] = b->getOrientation(); + tb.bindPos[static_cast(i)] = b->_getDerivedPosition(); + if (auto* p = dynamic_cast(b->getParent())) + tb.parentIdx[static_cast(i)] = p->getHandle(); + } + for (int i = 0; i < nBones; ++i) { + const int c = boneToCanon[static_cast(i)]; + if (c >= 0 && c < Jc && tb.roleBoneIdx[static_cast(c)] < 0) + tb.roleBoneIdx[static_cast(c)] = i; + } + { + auto rolePos = [&](int role) -> const Ogre::Vector3* { + const int i = (role >= 0 && role < Jc) + ? tb.roleBoneIdx[static_cast(role)] : -1; + return i >= 0 ? &tb.bindPos[static_cast(i)] : nullptr; + }; + const Ogre::Vector3* hip = rolePos(0); + const Ogre::Vector3* head = rolePos(5); + if (!head) head = rolePos(3); + const Ogre::Vector3* lSide = rolePos(19); + const Ogre::Vector3* rSide = rolePos(15); + if (!lSide || !rSide) { lSide = rolePos(11); rSide = rolePos(7); } + if (hip && head && lSide && rSide) { + Ogre::Vector3 up = *head - *hip; + Ogre::Vector3 left = *lSide - *rSide; + if (up.squaredLength() > 1e-12f + && left.squaredLength() > 1e-12f) { + up.normalise(); + left = left - up * left.dotProduct(up); + if (left.squaredLength() > 1e-12f) { + left.normalise(); + const Ogre::Vector3 fwd = left.crossProduct(up); + Ogre::Matrix3 src; + src.SetColumn(0, left); + src.SetColumn(1, up); + src.SetColumn(2, fwd); + tb.Ct = Ogre::Quaternion(src).Inverse(); + tb.Ct.normalise(); + } + } + } + } + auto targetDir = [&](int role) -> Ogre::Vector3 { + const int child = MotionInbetween::canonicalChildOf(role); + const int parent = MotionInbetween::canonicalParentOf(role); + const int a = tb.roleBoneIdx[static_cast(role)]; + if (a >= 0 && child >= 0) { + const int bIdx = tb.roleBoneIdx[static_cast(child)]; + if (bIdx >= 0) { + Ogre::Vector3 v = tb.bindPos[static_cast(bIdx)] + - tb.bindPos[static_cast(a)]; + if (v.squaredLength() > 1e-12f) return v; + } + } + if (a >= 0 && parent >= 0) { + const int pIdx = tb.roleBoneIdx[static_cast(parent)]; + if (pIdx >= 0) { + Ogre::Vector3 v = tb.bindPos[static_cast(a)] + - tb.bindPos[static_cast(pIdx)]; + if (v.squaredLength() > 1e-12f) return v; + } + } + return Ogre::Vector3::ZERO; + }; + for (int c = 0; c < Jc; ++c) { + Ogre::Vector3 v = targetDir(c); + if (v.squaredLength() > 1e-12f) { + v.normalise(); + tb.tgtBindDir[static_cast(c)] = v; + } + } + // Parents-before-children traversal (bone indices may not be ordered). + tb.order.reserve(static_cast(nBones)); + std::vector placed(static_cast(nBones), 0); + bool progress = true; + while (static_cast(tb.order.size()) < nBones && progress) { + progress = false; + for (int i = 0; i < nBones; ++i) { + if (placed[static_cast(i)]) continue; + const int pi = tb.parentIdx[static_cast(i)]; + if (pi < 0 || placed[static_cast(pi)]) { + tb.order.push_back(i); + placed[static_cast(i)] = 1; + progress = true; + } + } + } + for (int i = 0; i < nBones; ++i) // cycles/orphans: append + if (!placed[static_cast(i)]) tb.order.push_back(i); + return tb; +} +} // namespace + AnimationMerger::ApplyMotionResult AnimationMerger::applyMotionClip( Ogre::Skeleton* skel, const std::string& animName, @@ -1294,224 +1420,174 @@ AnimationMerger::ApplyMotionResult AnimationMerger::applyMotionClip( break; } } - if (haveRestWorld) { - const int Jc = MotionInbetween::canonicalJointCount(); - // Target bind worlds/locals/positions, read at binding pose. - skel->reset(true); - skel->_updateTransforms(); - std::vector bindWorld(static_cast(nBones)); - std::vector bindLocal(static_cast(nBones)); - std::vector bindPos(static_cast(nBones)); - std::vector parentIdx(static_cast(nBones), -1); - for (int i = 0; i < nBones; ++i) { - Ogre::Bone* b = skel->getBone(static_cast(i)); - bindWorld[static_cast(i)] = b->_getDerivedOrientation(); - bindLocal[static_cast(i)] = b->getOrientation(); - bindPos[static_cast(i)] = b->_getDerivedPosition(); - if (auto* p = dynamic_cast(b->getParent())) - parentIdx[static_cast(i)] = p->getHandle(); - } - // Per-role shortest-arc alignment R0: rotate the TARGET's bind bone - // direction onto the SOURCE's (both canonical-topology directions in - // canonical world axes). With Wt(f) = Δs(f)·R0·Wt_bind every target - // bone POINTS where the source bone points each frame, with twist - // transported by Δs — cross-rig bone-axis conventions cancel. - // (restWorld alone degenerates on identity-rest rigs like Mixamo.) - std::vector roleBoneIdx(static_cast(Jc), -1); - for (int i = 0; i < nBones; ++i) - if (boneToCanon[i] >= 0 && boneToCanon[i] < Jc - && roleBoneIdx[static_cast(boneToCanon[i])] < 0) - roleBoneIdx[static_cast(boneToCanon[i])] = i; - auto restQ = [&](int c) { - const auto& q = cmuRestWorld[static_cast(c)]; - return Ogre::Quaternion(q[3], q[0], q[1], q[2]); - }; - // TARGET-frame conjugation Ct: the clip directions live in the - // CANONICAL frame (the extractor conjugates the source by its own - // bind-geometry-derived frame), while this skeleton's bind data is - // in its RAW file frame (Mixamo FBX rigs are commonly rotated). - // Derive Ct exactly like the extractor (up = hip→head, left = - // rhip→lhip) and express canonical directions in target space via - // Ct⁻¹ before aiming. - Ogre::Quaternion Ct = Ogre::Quaternion::IDENTITY; - { - auto rolePos = [&](int role) -> const Ogre::Vector3* { - const int i = (role >= 0 && role < Jc) - ? roleBoneIdx[static_cast(role)] : -1; - return i >= 0 ? &bindPos[static_cast(i)] : nullptr; - }; - const Ogre::Vector3* hip = rolePos(0); - const Ogre::Vector3* head = rolePos(5); - if (!head) head = rolePos(3); - const Ogre::Vector3* lSide = rolePos(19); - const Ogre::Vector3* rSide = rolePos(15); - if (!lSide || !rSide) { lSide = rolePos(11); rSide = rolePos(7); } - if (hip && head && lSide && rSide) { - Ogre::Vector3 up = *head - *hip; - Ogre::Vector3 left = *lSide - *rSide; - if (up.squaredLength() > 1e-12f - && left.squaredLength() > 1e-12f) { - up.normalise(); - left = left - up * left.dotProduct(up); - if (left.squaredLength() > 1e-12f) { - left.normalise(); - const Ogre::Vector3 fwd = left.crossProduct(up); - Ogre::Matrix3 src; - src.SetColumn(0, left); - src.SetColumn(1, up); - src.SetColumn(2, fwd); - Ct = Ogre::Quaternion(src).Inverse(); - Ct.normalise(); - } - } - } - } - const Ogre::Quaternion CtInv = Ct.Inverse(); - - // PER-FRAME DIRECTION MATCHING. "Bind" poses are NOT trustworthy - // T-poses across importers (Assimp's reset pose for Mixamo FBX is - // whatever the file stored), so bind-to-bind delta transplant breaks - // cross-rig. Each rig IS self-consistent though: the source bone's - // constant LOCAL direction axis a_s = Ws_bind⁻¹·ds_bind rotates to - // ds(f) = Ws(f)·a_s each frame, and the target bone is aimed at - // exactly that world direction: Wt(f) = arc(dt_bind → ds(f))·Wt_bind. - // Twist about the bone is intentionally not transported (roll is the - // least visible DoF; the hip's facing-yaw staying locked matches the - // clip-facing design). - std::vector srcLocalAxis( - static_cast(Jc), Ogre::Vector3::ZERO); - std::vector tgtBindDir( - static_cast(Jc), Ogre::Vector3::ZERO); - if (!clipRestDir.empty()) { - auto targetDir = [&](int role) -> Ogre::Vector3 { - const int child = MotionInbetween::canonicalChildOf(role); - const int parent = MotionInbetween::canonicalParentOf(role); - const int a = roleBoneIdx[static_cast(role)]; - if (a >= 0 && child >= 0) { - const int bIdx = roleBoneIdx[static_cast(child)]; - if (bIdx >= 0) { - Ogre::Vector3 v = bindPos[static_cast(bIdx)] - - bindPos[static_cast(a)]; - if (v.squaredLength() > 1e-12f) return v; - } - } - if (a >= 0 && parent >= 0) { - const int pIdx = roleBoneIdx[static_cast(parent)]; - if (pIdx >= 0) { - Ogre::Vector3 v = bindPos[static_cast(a)] - - bindPos[static_cast(pIdx)]; - if (v.squaredLength() > 1e-12f) return v; - } - } - return Ogre::Vector3::ZERO; - }; + const int Jc = MotionInbetween::canonicalJointCount(); + + // The rig's STANDING pose the legacy transport composes onto — declared + // here so the SYNTHETIC bind-referenced path below can fill it before + // the (contaminating) animation harvest is even considered. + struct StandXform { Ogre::Quaternion rot; Ogre::Vector3 pos; Ogre::Vector3 scale; bool has = false; }; + std::unordered_map standPose; + bool synthStand = false; + + if (!clipRestDir.empty()) { + const TargetBindFrame tb = readTargetBindFrame(skel, boneToCanon); + const Ogre::Quaternion CtInv = tb.Ct.Inverse(); + + if (haveRestWorld) { + // PER-FRAME DIRECTION MATCHING. "Bind" poses are NOT trustworthy + // T-poses across importers (Assimp's reset pose for Mixamo FBX is + // whatever the file stored), so bind-to-bind delta transplant + // breaks cross-rig. Each rig IS self-consistent though: the source + // bone's constant LOCAL direction axis a_s = Ws_ref⁻¹·ds_ref + // rotates to ds(f) = Ws(f)·a_s each frame, and the target bone is + // aimed at exactly that world direction: + // Wt(f) = arc(dt_bind → ds(f)) · Wt_bind + // hierarchy-ordered, twist about the bone intentionally not + // transported (roll is the least visible DoF). The generated clip + // therefore references the TARGET BIND — no pose from any other + // animation is involved. + std::vector srcLocalAxis( + static_cast(Jc), Ogre::Vector3::ZERO); for (int c = 0; c < Jc; ++c) { const auto& sd = clipRestDir[static_cast(c)]; Ogre::Vector3 srcDir(sd[0], sd[1], sd[2]); - Ogre::Vector3 tgtDir = targetDir(c); if (srcDir.squaredLength() > 1e-8f - && tgtDir.squaredLength() > 1e-8f) { + && tb.tgtBindDir[static_cast(c)].squaredLength() + > 1e-8f) { srcDir.normalise(); - tgtDir.normalise(); + const auto& q = cmuRestWorld[static_cast(c)]; + const Ogre::Quaternion restQ(q[3], q[0], q[1], q[2]); srcLocalAxis[static_cast(c)] = - restQ(c).Inverse() * srcDir; - tgtBindDir[static_cast(c)] = tgtDir; + restQ.Inverse() * srcDir; } } - } - // Hierarchy order: parents before children (indices may not be). - std::vector order; - order.reserve(static_cast(nBones)); - { - std::vector placed(static_cast(nBones), 0); - bool progress = true; - while (static_cast(order.size()) < nBones && progress) { - progress = false; - for (int i = 0; i < nBones; ++i) { - if (placed[static_cast(i)]) continue; - const int pi = parentIdx[static_cast(i)]; - if (pi < 0 || placed[static_cast(pi)]) { - order.push_back(i); - placed[static_cast(i)] = 1; - progress = true; + auto clipQ = [&](int frame, int joint) -> Ogre::Quaternion { + const auto& q = clipQuats[frame][joint]; + return Ogre::Quaternion(q[3], q[0], q[1], q[2]); // (w,x,y,z) + }; + std::vector tracks( + static_cast(nBones), nullptr); + for (int i = 0; i < nBones; ++i) + if (boneToCanon[i] >= 0 && boneToCanon[i] < Jc) { + tracks[static_cast(i)] = anim->createNodeTrack( + static_cast(i), + skel->getBone(static_cast(i))); + ++res.tracksWritten; + } + std::vector W(static_cast(nBones)); + for (int f = 0; f < frames; ++f) { + for (int i : tb.order) { + const int pi = tb.parentIdx[static_cast(i)]; + const Ogre::Quaternion Wp = (pi >= 0) + ? W[static_cast(pi)] + : Ogre::Quaternion::IDENTITY; + const int c = boneToCanon[i]; + Ogre::Quaternion local; + if (c >= 0 && c < Jc + && srcLocalAxis[static_cast(c)] + .squaredLength() > 1e-8f) { + // NB: yaw180 is deliberately NOT applied here — this + // path anchors facing to the TARGET's own bind (the + // clip is canonical +Z-facing by construction), so a + // flip would swing every aim near-anti-parallel to + // its bind direction and destabilise getRotationTo. + // The flag only matters for the legacy standing-pose + // transport below. + const Ogre::Vector3 ds = CtInv * + (clipQ(f, c) + * srcLocalAxis[static_cast(c)]); + const Ogre::Quaternion R = + tb.tgtBindDir[static_cast(c)] + .getRotationTo(ds); + const Ogre::Quaternion Wt = + R * tb.bindWorld[static_cast(i)]; + local = Wp.Inverse() * Wt; + W[static_cast(i)] = Wt; + } else { + local = tb.bindLocal[static_cast(i)]; + W[static_cast(i)] = Wp * local; + } + if (auto* trk = tracks[static_cast(i)]) { + // Ogre skeleton keyframes are DELTAS applied onto the + // binding pose (NodeAnimationTrack::applyToNode + // rotates the reset bone) — convert the absolute + // local target. + Ogre::TransformKeyFrame* kf = + trk->createNodeKeyFrame(f * dt); + kf->setRotation( + tb.bindLocal[static_cast(i)].Inverse() + * local); + kf->setTranslate(Ogre::Vector3::ZERO); + kf->setScale(Ogre::Vector3::UNIT_SCALE); } } } - for (int i = 0; i < nBones; ++i) // cycles/orphans: append - if (!placed[static_cast(i)]) order.push_back(i); + res.ok = true; + res.canonicalJoints = distinct; + res.frames = frames; + res.length = length; + return res; } - auto clipQ = [&](int frame, int joint) -> Ogre::Quaternion { - const auto& q = clipQuats[frame][joint]; - return Ogre::Quaternion(q[3], q[0], q[1], q[2]); // (w,x,y,z) - }; - std::vector tracks( - static_cast(nBones), nullptr); - for (int i = 0; i < nBones; ++i) - if (boneToCanon[i] >= 0 && boneToCanon[i] < Jc) { - tracks[static_cast(i)] = anim->createNodeTrack( - static_cast(i), - skel->getBone(static_cast(i))); - ++res.tracksWritten; - } - std::vector W(static_cast(nBones)); - for (int f = 0; f < frames; ++f) { - for (int i : order) { - const int pi = parentIdx[static_cast(i)]; - const Ogre::Quaternion Wp = (pi >= 0) - ? W[static_cast(pi)] : Ogre::Quaternion::IDENTITY; - const int c = boneToCanon[i]; - Ogre::Quaternion local; - if (c >= 0 && c < Jc - && srcLocalAxis[static_cast(c)].squaredLength() - > 1e-8f) { - // NB: yaw180 is deliberately NOT applied here — this - // path anchors facing to the TARGET's own bind (the clip - // is canonical +Z-facing by construction), so a flip - // would swing every aim near-anti-parallel to its bind - // direction and destabilise getRotationTo. The flag only - // matters for the legacy standing-pose transport below. - const Ogre::Vector3 ds = CtInv * - (clipQ(f, c) * srcLocalAxis[static_cast(c)]); + + // MODEL clips carry no reference triple (their quats are learned, + // mixed-convention deltas), so they ride the legacy transport below — + // but its standing pose must NOT be harvested from the rig's other + // animations (that bakes e.g. a dance stance into every generated + // clip). Instead SYNTHESIZE the standing pose from a TEMPLATE clip's + // canonical bone directions (passed via clipRestDir): aim each bone + // from the target's BIND at the template's reference direction — + // bind-referenced by construction, matching the direction path at + // its reference frame. + std::vector Wstand(static_cast(nBones)); + int placedRoles = 0; + for (int i : tb.order) { + const int pi = tb.parentIdx[static_cast(i)]; + const Ogre::Quaternion Wp = (pi >= 0) + ? Wstand[static_cast(pi)] : Ogre::Quaternion::IDENTITY; + const int c = boneToCanon[i]; + bool aimed = false; + if (c >= 0 && c < Jc + && tb.tgtBindDir[static_cast(c)].squaredLength() + > 1e-8f) { + const auto& sd = clipRestDir[static_cast(c)]; + Ogre::Vector3 srcDir(sd[0], sd[1], sd[2]); + if (srcDir.squaredLength() > 1e-8f) { + srcDir.normalise(); + const Ogre::Vector3 ds = CtInv * srcDir; const Ogre::Quaternion R = - tgtBindDir[static_cast(c)].getRotationTo(ds); + tb.tgtBindDir[static_cast(c)] + .getRotationTo(ds); const Ogre::Quaternion Wt = - R * bindWorld[static_cast(i)]; - local = Wp.Inverse() * Wt; - W[static_cast(i)] = Wt; - } else { - local = bindLocal[static_cast(i)]; - W[static_cast(i)] = Wp * local; - } - if (auto* trk = tracks[static_cast(i)]) { - // Ogre skeleton keyframes are DELTAS applied onto the - // binding pose (NodeAnimationTrack::applyToNode rotates - // the reset bone) — convert the absolute local target. - Ogre::TransformKeyFrame* kf = - trk->createNodeKeyFrame(f * dt); - kf->setRotation( - bindLocal[static_cast(i)].Inverse() * local); - kf->setTranslate(Ogre::Vector3::ZERO); - kf->setScale(Ogre::Vector3::UNIT_SCALE); + R * tb.bindWorld[static_cast(i)]; + const Ogre::Quaternion local = Wp.Inverse() * Wt; + Wstand[static_cast(i)] = Wt; + // Keyframe-delta form (relative to the bind local), the + // same convention the harvested standing pose arrives in. + standPose[static_cast(i)] = { + tb.bindLocal[static_cast(i)].Inverse() * local, + Ogre::Vector3::ZERO, Ogre::Vector3::UNIT_SCALE, true }; + aimed = true; + ++placedRoles; } } + if (!aimed) + Wstand[static_cast(i)] = + Wp * tb.bindLocal[static_cast(i)]; } - res.ok = true; - res.canonicalJoints = distinct; - res.frames = frames; - res.length = length; - return res; + synthStand = placedRoles > 0; } + // The rig's natural STANDING pose. Mixamo skeletons have an identity bone // rest pose — the real standing orientation lives in frame 0 of their // existing animation, not in the bone transform. So harvest each bone's // orientation at t=0 of the rig's first existing animation; that is the // "bind" we compose our CMU motion onto (without it the body inverts, since // the bone rest is a meaningless identity). Falls back to the bone's own - // rest if there's no prior animation. - struct StandXform { Ogre::Quaternion rot; Ogre::Vector3 pos; Ogre::Vector3 scale; bool has = false; }; - std::unordered_map standPose; - if (skel->getNumAnimations() > 0) { + // rest if there's no prior animation. Skipped entirely when the synthetic + // bind-referenced standing pose above was built — harvesting another + // animation's pose is exactly the contamination that path eliminates. + if (!synthStand && skel->getNumAnimations() > 0) { Ogre::Animation* ref = nullptr; for (unsigned short ai = 0; ai < skel->getNumAnimations(); ++ai) { Ogre::Animation* a = skel->getAnimation(ai); diff --git a/src/CLIPipeline.cpp b/src/CLIPipeline.cpp index 8894ce632..aa00ad4fc 100644 --- a/src/CLIPipeline.cpp +++ b/src/CLIPipeline.cpp @@ -2022,6 +2022,11 @@ int CLIPipeline::cmdAnimGenerate(const QString& filePath, const QString& prompt, quats = mr.clip.quats; fps = mr.clip.fps; worldFrame = mr.worldFrame; // v4 models: world-frame quats + // Model clips carry no reference triple — borrow a template + // clip's canonical bone directions so the retarget can + // synthesize a BIND-referenced base pose instead of + // harvesting the rig's other animations (contamination). + clipDirs = MotionLibrary::referenceDirsForPrompt(prompt); clipSource = QStringLiteral("model"); gotClip = true; } else { diff --git a/src/MCPServer.cpp b/src/MCPServer.cpp index 0c4279c56..d3f2a005e 100644 --- a/src/MCPServer.cpp +++ b/src/MCPServer.cpp @@ -4050,6 +4050,10 @@ QJsonObject MCPServer::toolGenerateMotion(const QJsonObject &args) if (mr.ok) { action = mr.matchedAction; quats = mr.clip.quats; fps = mr.clip.fps; worldFrame = mr.worldFrame; clipSource = QStringLiteral("model"); gotClip = true; + // Borrow a template clip's reference directions so the + // retarget synthesizes a BIND-referenced base pose (no + // harvest from the rig's other animations). + clipDirs = MotionLibrary::referenceDirsForPrompt(prompt); } } } diff --git a/src/MotionLibrary.cpp b/src/MotionLibrary.cpp index 6d3719c7d..6d321d6fc 100644 --- a/src/MotionLibrary.cpp +++ b/src/MotionLibrary.cpp @@ -265,3 +265,38 @@ QString MotionLibrary::ensureLibraryBlocking() return (ok && !timedOut && QFileInfo::exists(dest)) ? dest : QString(); } + +std::vector> MotionLibrary::referenceDirsForPrompt( + const QString& prompt) +{ + MotionLibrary lib; + if (!libraryPresent() || !lib.loadFromFile(libraryPath())) + return {}; + // Deterministically pick the most COMPLETE direction set (some scraped + // clips resolve fewer roles — an incomplete reference leaves those bones + // at the raw T-pose). Prefer clips matching the prompt's action. + QString action; + const int matched = lib.matchPrompt(prompt, &action); + auto score = [](const Clip& c) { + int n = 0; + for (const auto& d : c.restDir) + if (std::abs(d[0]) > 1e-6f || std::abs(d[1]) > 1e-6f + || std::abs(d[2]) > 1e-6f) + ++n; + return n; + }; + const Clip* best = nullptr; + int bestScore = 0; + bool bestMatchesAction = false; + for (const Clip& c : lib.m_clips) { + const int n = score(c); + if (n == 0) continue; + const bool m = matched >= 0 + && c.action.compare(action, Qt::CaseInsensitive) == 0; + if (!best || (m && !bestMatchesAction) + || (m == bestMatchesAction && n > bestScore)) { + best = &c; bestScore = n; bestMatchesAction = m; + } + } + return best ? best->restDir : std::vector>{}; +} diff --git a/src/MotionLibrary.h b/src/MotionLibrary.h index 1653b9edb..c1efb05b3 100644 --- a/src/MotionLibrary.h +++ b/src/MotionLibrary.h @@ -92,6 +92,16 @@ class MotionLibrary { // / QSettings ai/motionLibraryBaseUrl. Call on a thread with an event loop. static QString ensureLibraryBlocking(); + // Reference bone directions (22 × [x,y,z]) for a MODEL-generated clip: + // model output carries no reference triple, so its base pose is + // synthesized from a TEMPLATE clip's restDir instead of harvesting the + // rig's other animations (which contaminated generations with e.g. a + // dance stance). Prefers a clip matching `prompt`, falls back to any + // clip carrying restDir. Reads the LOCAL library only (no download); + // returns empty when unavailable — callers then keep the harvest path. + static std::vector> referenceDirsForPrompt( + const QString& prompt); + private: bool parse(const QByteArray& json); std::vector m_clips;