diff --git a/gui/public/i18n/en/translation.ftl b/gui/public/i18n/en/translation.ftl index fd804a0ec6..d08e7adcbd 100644 --- a/gui/public/i18n/en/translation.ftl +++ b/gui/public/i18n/en/translation.ftl @@ -639,8 +639,9 @@ settings-general-fk_settings-velocity_settings = Velocity Settings settings-general-fk_settings-velocity_settings-description = Send derived velocity data to SteamVR. Required for Natural Locomotion support. May cause jitter in FBT. settings-general-fk_settings-velocity_settings-send_derived_velocity = Send derived velocity to driver settings-general-fk_settings-arm_fk = Arm tracking -settings-general-fk_settings-arm_fk-description = Force arms to be tracked from the headset (HMD) even if positional hand data is available. -settings-general-fk_settings-arm_fk-force_arms = Force arms from HMD +settings-general-fk_settings-arm_fk-description = Use forward kinematics to track arms between controller and HMD positional tracking. Otherwise, track arms from controllers alone. +settings-general-fk_settings-arm_fk-force_arms = Use forward kinematics (recommended) +settings-general-fk_settings-arm_fk-legacy = Tracking arms from controller alone is not recommended anymore! settings-general-fk_settings-reset_settings = Reset settings settings-general-fk_settings-reset_settings-reset_hmd_pitch-description = Reset the HMD's pitch (vertical rotation) upon doing a full reset. Useful if wearing an HMD on the forehead for VTubing or mocap. Do not enable for VR. settings-general-fk_settings-reset_settings-reset_hmd_pitch = Reset HMD pitch @@ -822,6 +823,9 @@ settings-osc-router-network-port_out = settings-osc-router-network-address = Network address settings-osc-router-network-address-description = Set the address to send out data at. settings-osc-router-network-address-placeholder = IPV4 address +settings-osc-router-external_tracking = External tracking +settings-osc-router-rescale_tracking = Rescale to avatar +settings-osc-router-rescale_tracking-description = Rescale external tracking routed through OSC to avatar scale as determined by the VRM loaded in VMC settings. ## OSC VRChat settings settings-osc-vrchat = VRChat OSC Trackers @@ -879,12 +883,13 @@ settings-osc-vmc-network-port_out = settings-osc-vmc-network-address = Network address settings-osc-vmc-network-address-description = Choose which address to send out data at via VMC. settings-osc-vmc-network-address-placeholder = IPV4 address -settings-osc-vmc-vrm = VRM Model -settings-osc-vmc-vrm-description = Load a VRM model to allow head anchor and enable a higher compatibility with other applications. +settings-osc-vmc-vrm = VRM Avatar +settings-osc-vmc-vrm-description = Load a VRM avatar to allow for positioning using that avatars scale. Otherwise, alignment in external programs that use VMC might be wrong. settings-osc-vmc-vrm-untitled_model = Untitled model settings-osc-vmc-vrm-file_select = Drag & drop a model to use, or browse +settings-osc-vmc-vrm-required = Loading a VRM avatar is required for proper avatar positioning over VMC! settings-osc-vmc-anchor_hip = Anchor at hips -settings-osc-vmc-anchor_hip-description = Anchor the tracking at the hips, useful for seated VTubing. If disabling, load a VRM model. +settings-osc-vmc-anchor_hip-description = Anchor the tracking at the hips, useful for seated VTubing. If disabled, load a VRM avatar to allow for proper positioning. settings-osc-vmc-anchor_hip-label = Anchor at hips settings-osc-vmc-mirror_tracking = Mirror tracking settings-osc-vmc-mirror_tracking-description = Mirror the tracking horizontally. @@ -977,8 +982,8 @@ onboarding-quiz-mocap_preferences-playspace-title = What is your playspace? onboarding-quiz-mocap_preferences-playspace-desc = If standing, SlimeVR will try to track walking movement instead of anchoring you in one spot. onboarding-quiz-mocap_preferences-playspace-sitting = Sitting onboarding-quiz-mocap_preferences-playspace-standing = Standing -onboarding-quiz-mocap_preferences-vrm_model-title = Do you have a VRM model? (Optional) -onboarding-quiz-mocap_preferences-vrm_model-desc = Loading a VRM model will improve tracking quality and compatibility with applications that use VMC. +onboarding-quiz-mocap_preferences-vrm_model-title = Do you have a VRM avatar? (Optional) +onboarding-quiz-mocap_preferences-vrm_model-desc = Load a VRM avatar to allow for positioning using that avatars scale. Otherwise, alignment in external programs that use VMC might be wrong. onboarding-quiz-mocap_preferences-head_tracker-title = Are you wearing a tracker or VR headset on your head? onboarding-quiz-mocap_preferences-head_tracker-yes = Yes onboarding-quiz-mocap_preferences-head_tracker-no = No diff --git a/gui/src/components/onboarding/pages/quiz/MocapPreferencesQuestions.tsx b/gui/src/components/onboarding/pages/quiz/MocapPreferencesQuestions.tsx index c2dbdb4f08..6b3e6fe98c 100644 --- a/gui/src/components/onboarding/pages/quiz/MocapPreferencesQuestions.tsx +++ b/gui/src/components/onboarding/pages/quiz/MocapPreferencesQuestions.tsx @@ -31,29 +31,28 @@ export function QuizMocapPosQuestion() { id="onboarding-quiz-mocap_preferences-desc" /> +
+
setHeadTracker(true)} - icon={} - name="onboarding-quiz-mocap_preferences-head_tracker-yes" + active={playspace === 'sitting'} + onClick={() => setPlayspace('sitting')} + icon={} + name="onboarding-quiz-mocap_preferences-playspace-sitting" /> { - setHeadTracker(false); - setMocapPos(undefined); - }} - icon={} - name="onboarding-quiz-mocap_preferences-head_tracker-no" + active={playspace === 'standing'} + onClick={() => setPlayspace('standing')} + icon={} + name="onboarding-quiz-mocap_preferences-playspace-standing" />
@@ -68,37 +67,40 @@ export function QuizMocapPosQuestion() { />
- + + + + +
+
+
+ +
+
+ setHeadTracker(true)} + icon={} + name="onboarding-quiz-mocap_preferences-head_tracker-yes" + /> + { + setHeadTracker(false); + setMocapPos(undefined); + }} + icon={} + name="onboarding-quiz-mocap_preferences-head_tracker-no" + /> +
{headTracker && ( <> -
-
-
- - -
-
- setPlayspace('sitting')} - icon={} - name="onboarding-quiz-mocap_preferences-playspace-sitting" - /> - setPlayspace('standing')} - icon={} - name="onboarding-quiz-mocap_preferences-playspace-standing" - /> -
-
-
@@ -114,7 +116,7 @@ export function QuizMocapPosQuestion() { icon={ } name="onboarding-quiz-mocap_preferences-head_tracker_location-forehead" @@ -125,7 +127,7 @@ export function QuizMocapPosQuestion() { icon={ } name="onboarding-quiz-mocap_preferences-head_tracker_location-face" diff --git a/gui/src/components/settings/pages/GeneralSettings.tsx b/gui/src/components/settings/pages/GeneralSettings.tsx index a90e725afc..bc5f11a85b 100644 --- a/gui/src/components/settings/pages/GeneralSettings.tsx +++ b/gui/src/components/settings/pages/GeneralSettings.tsx @@ -27,6 +27,7 @@ import { WrenchIcon } from '@/components/commons/icon/WrenchIcons'; import { NumberSelector } from '@/components/commons/NumberSelector'; import { Radio } from '@/components/commons/Radio'; import { Typography } from '@/components/commons/Typography'; +import { TipBox } from '@/components/commons/TipBox'; import { SettingsPageLayout, SettingsPagePaneLayout, @@ -229,6 +230,7 @@ export function GeneralSettings() { const { trackers: { automaticTrackerToggle }, + toggles: { forceArmsFromHmd }, } = watch(); const onSubmit = (values: SettingsForm) => { @@ -924,6 +926,11 @@ export function GeneralSettings() { )} />
+ {!forceArmsFromHmd && ( + + {l10n.getString('settings-general-fk_settings-arm_fk-legacy')} + + )}
diff --git a/gui/src/components/settings/pages/OSCRouterSettings.tsx b/gui/src/components/settings/pages/OSCRouterSettings.tsx index 0e5a68ae2c..762aa706cd 100644 --- a/gui/src/components/settings/pages/OSCRouterSettings.tsx +++ b/gui/src/components/settings/pages/OSCRouterSettings.tsx @@ -19,7 +19,7 @@ import { SettingsPagePaneLayout, } from '@/components/settings/SettingsPageLayout'; import { yupResolver } from '@hookform/resolvers/yup'; -import { object } from 'yup'; +import { boolean, object } from 'yup'; import { OSCSettings, useOscSettingsValidator, @@ -28,6 +28,7 @@ import { interface OSCRouterSettingsForm { router: { oscSettings: OSCSettings; + rescaleTracking: boolean; }; } @@ -39,6 +40,7 @@ const defaultValues = { portOut: 9000, address: '127.0.0.1', }, + rescaleTracking: false, }, }; @@ -56,6 +58,7 @@ export function OSCRouterSettings() { object({ router: object({ oscSettings: oscValidator, + rescaleTracking: boolean().required(), }), }) ), @@ -71,6 +74,7 @@ export function OSCRouterSettings() { new OSCSettingsT(), values.router.oscSettings ); + router.rescaleTracking = values.router.rescaleTracking; settings.oscRouter = router; } @@ -102,6 +106,7 @@ export function OSCRouterSettings() { formData.router.oscSettings.address = settings.oscRouter.oscSettings.address.toString(); } + formData.router.rescaleTracking = settings.oscRouter.rescaleTracking; reset(formData); } @@ -202,6 +207,25 @@ export function OSCRouterSettings() { label="" />
+ + {l10n.getString('settings-osc-router-external_tracking')} + +
+ + {l10n.getString( + 'settings-osc-router-rescale_tracking-description' + )} + +
+
+ +
diff --git a/gui/src/components/settings/pages/VMCSettings.tsx b/gui/src/components/settings/pages/VMCSettings.tsx index f13af7de4b..62c77e8ed8 100644 --- a/gui/src/components/settings/pages/VMCSettings.tsx +++ b/gui/src/components/settings/pages/VMCSettings.tsx @@ -16,6 +16,7 @@ import { FileInput } from '@/components/commons/FileInput'; import { VMCIcon } from '@/components/commons/icon/VMCIcon'; import { Input } from '@/components/commons/Input'; import { Typography } from '@/components/commons/Typography'; +import { TipBox } from '@/components/commons/TipBox'; import { magic } from '@/utils/formatting'; import { SettingsPageLayout, @@ -50,7 +51,7 @@ const defaultValues = { }, }; -export function VMCFileUpload() { +export function VMCFileUpload({ suggested = false }) { const { sendRPCPacket, useRPCPacket } = useWebsocketAPI(); const { l10n } = useLocalization(); const [modelName, setModelName] = useState(null); @@ -106,22 +107,29 @@ export function VMCFileUpload() { }); return ( - + <> + {modelName === null && suggested && ( + + Tip + + )} + + ); } @@ -145,6 +153,8 @@ export function VMCSettings() { ), }); + const anchorHip = watch('vmc.anchorHip'); + const onSubmit = async (values: VMCSettingsForm) => { const settings = new ChangeSettingsRequestT(); @@ -287,17 +297,6 @@ export function VMCSettings() { label="" />
- - {l10n.getString('settings-osc-vmc-vrm')} - -
- - {l10n.getString('settings-osc-vmc-vrm-description')} - -
-
- -
{l10n.getString('settings-osc-vmc-anchor_hip')} @@ -315,6 +314,17 @@ export function VMCSettings() { label={l10n.getString('settings-osc-vmc-anchor_hip-label')} />
+ + {l10n.getString('settings-osc-vmc-vrm')} + +
+ + {l10n.getString('settings-osc-vmc-vrm-description')} + +
+
+ +
{l10n.getString('settings-osc-vmc-mirror_tracking')} @@ -419,7 +429,7 @@ function getVRMName(data: any): string | null { if (typeof name !== 'string') { error( - `The name of the VRM model is not a string, instead it is a ${typeof name}` + `The name of the VRM avatar is not a string, instead it is a ${typeof name}` ); return null; } diff --git a/gui/src/components/widgets/SkeletonVisualizerWidget.tsx b/gui/src/components/widgets/SkeletonVisualizerWidget.tsx index 3954b65235..5fba8543bd 100644 --- a/gui/src/components/widgets/SkeletonVisualizerWidget.tsx +++ b/gui/src/components/widgets/SkeletonVisualizerWidget.tsx @@ -11,14 +11,12 @@ import { GridHelper, Group, PerspectiveCamera, - Quaternion, Scene, Vector2, Vector3, WebGLRenderer, } from 'three'; import { BodyPart, BoneT } from 'solarxr-protocol'; -import { QuaternionFromQuatT, isIdentity } from '@/maths/quaternion'; import classNames from 'classnames'; import { useLocalization } from '@fluent/react'; import { ErrorBoundary } from 'react-error-boundary'; @@ -72,7 +70,10 @@ function initializePreview( }); renderer.setSize(canvas.clientWidth, canvas.clientHeight); - const grid = new GridHelper(10, 50, GROUND_COLOR, GROUND_COLOR); + const gridSize = 20, + gridDiv = 100, + gridSpacing = gridSize / gridDiv; + const grid = new GridHelper(gridSize, gridDiv, GROUND_COLOR, GROUND_COLOR); grid.position.set(0, 0, 0); scene.add(grid); @@ -85,12 +86,8 @@ function initializePreview( scene.add(skeleton[0]); let heightOffset = 0; - let skeletonOffset = 0; - const rebuildSkeleton = ( - newSkeleton: (BoneKind | Bone)[], - bones: Map - ) => { + const rebuildSkeleton = (newSkeleton: (BoneKind | Bone)[]) => { skeletonGroup.remove(skeletonHelper); skeletonHelper.dispose(); scene.remove(skeleton[0]); @@ -101,22 +98,6 @@ function initializePreview( skeletonHelper.resolution.copy(resolution); skeletonGroup.add(skeletonHelper); scene.add(newSkeleton[0]); - - const hmd = bones.get(BodyPart.HEAD); - const chest = bones.get(BodyPart.UPPER_CHEST); - // Check if HMD is identity, if it's then use upper chest's rotation - const quat = isIdentity(hmd?.rotationG) - ? QuaternionFromQuatT(chest?.rotationG).normalize().invert() - : QuaternionFromQuatT(hmd?.rotationG).normalize().invert(); - - // Project quat to (0x, 1y, 0z) - const VEC_Y = new Vector3(0, 1, 0); - const vec = VEC_Y.multiplyScalar( - new Vector3(quat.x, quat.y, quat.z).dot(VEC_Y) / VEC_Y.lengthSq() - ); - const yawReset = new Quaternion(vec.x, vec.y, vec.z, quat.w).normalize(); - - skeletonGroup.rotation.setFromQuaternion(yawReset); }; const computeUserHeight = (bones: Map) => { @@ -132,15 +113,23 @@ function initializePreview( ); }; - const computeSkeletonOffset = (bones: Map) => { + const computeSkeletonPos = (bones: Map) => { const hmd = bones.get(BodyPart.HEAD); - // If I know the head position, don't use an offset - if (hmd?.headPositionG?.y !== undefined && hmd.headPositionG?.y > 0) { - return 0; - } + const pos = new Vector3( + hmd?.headPositionG?.x ?? 0, + hmd?.headPositionG?.y ?? 0, + hmd?.headPositionG?.z ?? 0 + ); + if (pos.length() != 0) return pos; + // Estimate head height based on skeleton + // This path should not occur, server should send some kind of root position const yLength = Y_PARTS.map((x) => bones.get(x)); - if (yLength.some((x) => x === undefined)) return 0; - return (yLength as BoneT[]).reduce((prev, cur) => prev + cur.boneLength, 0); + if (yLength.some((x) => x === undefined)) return pos; + const offset = (yLength as BoneT[]).reduce( + (prev, cur) => prev + cur.boneLength, + 0 + ); + return new Vector3(0, offset, 0); }; const render = (delta: number) => { @@ -220,11 +209,14 @@ function initializePreview( }); } - const newSkeletinOffset = computeSkeletonOffset(bones); - if (newSkeletinOffset != skeletonOffset) { - skeletonOffset = newSkeletinOffset; - skeletonGroup.position.set(0, skeletonOffset, 0); - } + // Apply height to skeleton, and horizontal position inversely to grid (to keep skeleton centered) + const skeletonPos = computeSkeletonPos(bones); + skeletonGroup.position.set(0, skeletonPos.y, 0); + grid.position.set( + -skeletonPos.x % gridSpacing, + 0, + -skeletonPos.z % gridSpacing + ); }, destroy: () => { cancelAnimationFrame(animationFrameId); @@ -320,7 +312,7 @@ function SkeletonVisualizer({ if (bones.size === 0) return; const context = previewContext.current; if (!context || disabled) return; - context.rebuildSkeleton(createChildren(bones, BoneKind.root), bones); + context.rebuildSkeleton(createChildren(bones, BoneKind.root)); }, [bones.size, disabled]); useEffect(() => { diff --git a/gui/src/utils/skeletonHelper.ts b/gui/src/utils/skeletonHelper.ts index eb77d95a48..55be376fa7 100644 --- a/gui/src/utils/skeletonHelper.ts +++ b/gui/src/utils/skeletonHelper.ts @@ -146,7 +146,10 @@ export class BoneKind extends Bone { const parent = BoneKind.parent(this.boneT.bodyPart); const parentBone = parent === null ? undefined : bones.get(parent); if (this.boneT.bodyPart === BoneKind.root) { - this.position.set(0, this.boneT.headPositionG?.y ?? 0, 0); + if (parent !== null) + // otherwise, this logic falls apart + console.error('Expected root bone to not have any parent!'); + this.position.set(0, 0, 0); return; } diff --git a/server/core/src/main/java/dev/slimevr/config/OSCRouterConfig.kt b/server/core/src/main/java/dev/slimevr/config/OSCRouterConfig.kt new file mode 100644 index 0000000000..efeb1f9c35 --- /dev/null +++ b/server/core/src/main/java/dev/slimevr/config/OSCRouterConfig.kt @@ -0,0 +1,7 @@ +package dev.slimevr.config + +class OSCRouterConfig : OSCConfig() { + + // Rescale positional tracking to avatar scale (with VRM specified) + var rescaleTracking = false +} diff --git a/server/core/src/main/java/dev/slimevr/config/VRConfig.kt b/server/core/src/main/java/dev/slimevr/config/VRConfig.kt index 903750aa8a..5458024383 100644 --- a/server/core/src/main/java/dev/slimevr/config/VRConfig.kt +++ b/server/core/src/main/java/dev/slimevr/config/VRConfig.kt @@ -21,7 +21,7 @@ class VRConfig { val driftCompensation: DriftCompensationConfig = DriftCompensationConfig() - val oscRouter: OSCConfig = OSCConfig() + val oscRouter: OSCRouterConfig = OSCRouterConfig() val vrcOSC: VRCOSCConfig = VRCOSCConfig() @@ -78,12 +78,12 @@ class VRConfig { vrcOSC .setOSCTrackerRole( TrackerRole.LEFT_FOOT, - vrcOSC.getOSCTrackerRole(TrackerRole.WAIST, true), + vrcOSC.getOSCTrackerRole(TrackerRole.LEFT_FOOT, true), ) vrcOSC .setOSCTrackerRole( TrackerRole.RIGHT_FOOT, - vrcOSC.getOSCTrackerRole(TrackerRole.WAIST, true), + vrcOSC.getOSCTrackerRole(TrackerRole.RIGHT_FOOT, true), ) // Initialize default settings for VMC diff --git a/server/core/src/main/java/dev/slimevr/osc/OSCRouter.java b/server/core/src/main/java/dev/slimevr/osc/OSCRouter.java index 0621dfa2b8..167fdfb7b5 100644 --- a/server/core/src/main/java/dev/slimevr/osc/OSCRouter.java +++ b/server/core/src/main/java/dev/slimevr/osc/OSCRouter.java @@ -4,7 +4,7 @@ import com.illposed.osc.messageselector.OSCPatternAddressMessageSelector; import com.illposed.osc.transport.OSCPortIn; import com.illposed.osc.transport.OSCPortOut; -import dev.slimevr.config.OSCConfig; +import dev.slimevr.config.OSCRouterConfig; import io.eiren.util.collections.FastList; import io.eiren.util.logging.LogManager; @@ -12,23 +12,26 @@ import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.UnknownHostException; +import java.util.ArrayList; public class OSCRouter { private OSCPortIn oscReceiver; private OSCPortOut oscSender; - private final OSCConfig config; + private final OSCRouterConfig config; private final FastList oscHandlers; private int lastPortIn; private int lastPortOut; private InetAddress lastAddress; private long timeAtLastError; + public float scaleTrackingVolume = 1f; + public OSCRouter( - OSCConfig oscConfig, + OSCRouterConfig oscRouterConfig, FastList oscHandlers ) { - this.config = oscConfig; + this.config = oscRouterConfig; this.oscHandlers = oscHandlers; refreshSettings(false); @@ -147,6 +150,7 @@ public void refreshSettings(boolean refreshHandlersSettings) { // Listens for any message ("//" is a wildcard) MessageSelector selector = new OSCPatternAddressMessageSelector("//"); oscReceiver.getDispatcher().addListener(selector, listener); + oscReceiver.getDispatcher().setAlwaysDispatchingImmediately(true); if (!oscReceiver.isListening()) oscReceiver.startListening(); } @@ -155,12 +159,30 @@ public void refreshSettings(boolean refreshHandlersSettings) { void handleReceivedMessage(OSCMessageEvent event) { if (oscSender != null && oscSender.isConnected()) { - OSCMessage oscMessage = new OSCMessage( - event.getMessage().getAddress(), - event.getMessage().getArguments() - ); + var address = event.getMessage().getAddress(); + var args = event.getMessage().getArguments(); + OSCMessage oscMessageA = null, oscMessageB = null; + if ( + config.getRescaleTracking() + && scaleTrackingVolume != 1.0f + && address.endsWith("/Pos") + ) { + // Original message with coordinates in device scale + oscMessageA = new OSCMessage(address + "/Local", args); + // Modified message with coordinates in avatar scale + ArrayList argsMod = new ArrayList(args); + argsMod.set(1, (Float) argsMod.get(1) * scaleTrackingVolume); + argsMod.set(2, (Float) argsMod.get(2) * scaleTrackingVolume); + argsMod.set(3, (Float) argsMod.get(3) * scaleTrackingVolume); + oscMessageB = new OSCMessage(address, argsMod); + } else { + oscMessageA = new OSCMessage(address, args); + } + try { - oscSender.send(oscMessage); + oscSender.send(oscMessageA); + if (oscMessageB != null) + oscSender.send(oscMessageB); } catch (IOException | OSCSerializeException e) { // Avoid spamming AsynchronousCloseException too many // times per second diff --git a/server/core/src/main/java/dev/slimevr/osc/UnityArmature.kt b/server/core/src/main/java/dev/slimevr/osc/UnityArmature.kt index c355766a81..7328d32d79 100644 --- a/server/core/src/main/java/dev/slimevr/osc/UnityArmature.kt +++ b/server/core/src/main/java/dev/slimevr/osc/UnityArmature.kt @@ -169,6 +169,7 @@ class UnityArmature(localRot: Boolean) { fun update() { // Set the upper chest node's rotation to the chest's + // Otherwise, some applications will apply double yaw rotation (e.g. VNyan) upperChestNode.localTransform.rotation = chestNode.localTransform.rotation // Update the root node hipsNode.update() diff --git a/server/core/src/main/java/dev/slimevr/osc/UnityBone.kt b/server/core/src/main/java/dev/slimevr/osc/UnityBone.kt index 29d89f8f0e..7c2b590a18 100644 --- a/server/core/src/main/java/dev/slimevr/osc/UnityBone.kt +++ b/server/core/src/main/java/dev/slimevr/osc/UnityBone.kt @@ -43,7 +43,7 @@ enum class UnityBone( CHEST("Chest", BoneType.CHEST, TrackerPosition.CHEST), @SerialName("upperChest") - UPPER_CHEST("UpperChest", BoneType.CHEST, TrackerPosition.CHEST), + UPPER_CHEST("UpperChest", BoneType.UPPER_CHEST, TrackerPosition.UPPER_CHEST), @SerialName("neck") NECK("Neck", BoneType.NECK, TrackerPosition.NECK), diff --git a/server/core/src/main/java/dev/slimevr/osc/VMCHandler.kt b/server/core/src/main/java/dev/slimevr/osc/VMCHandler.kt index 52b40325e5..14f0bd704c 100644 --- a/server/core/src/main/java/dev/slimevr/osc/VMCHandler.kt +++ b/server/core/src/main/java/dev/slimevr/osc/VMCHandler.kt @@ -148,6 +148,7 @@ class VMCHandler( .addListener(OSCPatternAddressMessageSelector(address), listener) } + oscReceiver!!.dispatcher.setAlwaysDispatchingImmediately(true) oscReceiver!!.startListening() } } @@ -201,6 +202,7 @@ class VMCHandler( if (trackerPosition != null) { handleReceivedTracker( "VMC-Bone-" + event.message.arguments[0], + "VMC Bone: " + event.message.arguments[0], trackerPosition, null, Quaternion( @@ -213,6 +215,7 @@ class VMCHandler( getByStringVal( event.message.arguments[0].toString(), ), + false, ) } } @@ -221,6 +224,7 @@ class VMCHandler( "/VMC/Ext/Hmd/Pos", "/VMC/Ext/Con/Pos", "/VMC/Ext/Tra/Pos" -> handleReceivedTracker( "VMC-Tracker-" + event.message.arguments[0], + "VMC: " + event.message.arguments[0], null, Vector3( event.message.arguments[1] as Float, @@ -235,6 +239,7 @@ class VMCHandler( ), false, null, + event.message.address == "/VMC/Ext/Hmd/Pos", ) // Is VMC tracking root (offsets all rotations) @@ -261,11 +266,13 @@ class VMCHandler( private fun handleReceivedTracker( name: String, + label: String, trackerPosition: TrackerPosition?, position: Vector3?, rotation: Quaternion, localRotation: Boolean, unityBone: UnityBone?, + isHmd: Boolean, ) { // Create device if it doesn't exist var rot = rotation @@ -283,7 +290,7 @@ class VMCHandler( trackerDevice, getNextLocalTrackerId(), name, - "VMC Tracker #$currentLocalTrackerId", + label, trackerPosition, hasPosition = position != null, hasRotation = true, @@ -291,11 +298,13 @@ class VMCHandler( isComputed = position != null, usesTimeout = true, allowReset = position != null, + isHmd = isHmd, ) trackerDevice!!.trackers[trackerDevice!!.trackers.size] = tracker byTrackerNameTracker[name] = tracker server.registerTracker(tracker) } + tracker.isHmd = isHmd tracker.status = TrackerStatus.OK // Set position @@ -333,6 +342,10 @@ class VMCHandler( oscArgs.add((System.currentTimeMillis() - startTime) / 1000f) oscBundle.addPacket(OSCMessage("/VMC/Ext/T", oscArgs.clone())) + // Rescale tracking to avatar scale if configured with target VRM + val vrmScale = if (vrmHeight > 0) vrmHeight / humanPoseManager.userNeckHeightFromConfig else 1f + server.oSCRouter.scaleTrackingVolume = vrmScale + if (humanPoseManager.isSkeletonPresent) { // Indicate tracking is available oscArgs.clear() @@ -367,17 +380,16 @@ class VMCHandler( if (!anchorHip) { // Anchor from head outputUnityArmature?.let { unityArmature -> - // Scale the SlimeVR neck position with the VRM model + // Scale the SlimeVR neck position with the VRM avatar // We're only getting the height up to the neck because we don't want to factor the neck's length into the scaling - val slimevrScaledRootPos = humanPoseManager.getBone(BoneType.NECK).getTailPosition() * - (vrmHeight / humanPoseManager.userNeckHeightFromConfig) + var rootPos = humanPoseManager.getBone(BoneType.NECK).getTailPosition() * vrmScale // Get the VRM head and hip positions val vrmHeadPos = unityArmature.getHeadNodeOfBone(UnityBone.HEAD)!!.parent!!.worldTransform.translation val vrmHipPos = unityArmature.getHeadNodeOfBone(UnityBone.HIPS)!!.worldTransform.translation // Calculate the new VRM hip position by subtracting the difference head-hip distance from the SlimeVR head - val calculatedVrmHipPos = slimevrScaledRootPos - (vrmHeadPos - vrmHipPos) + val calculatedVrmHipPos = rootPos - (vrmHeadPos - vrmHipPos) // Set the VRM's hip position unityArmature.getHeadNodeOfBone(UnityBone.HIPS)?.localTransform?.translation = calculatedVrmHipPos @@ -415,12 +427,9 @@ class VMCHandler( for (tracker in computedTrackers) { if (!tracker.status.reset) { oscArgs.clear() - - val name = tracker.name - oscArgs.add(name) - + oscArgs.add(tracker.name) addTransformToArgs( - tracker.position, + tracker.position * vrmScale, tracker.getRotation(), ) @@ -433,6 +442,7 @@ class VMCHandler( } else { "/VMC/Ext/Tra/Pos" } + oscBundle .addPacket( OSCMessage( @@ -440,6 +450,22 @@ class VMCHandler( oscArgs.clone(), ), ) + + if (vrmScale != 1f) { + oscArgs.clear() + oscArgs.add(tracker.name) + addTransformToArgs( + tracker.position, + tracker.getRotation(), + ) + oscBundle + .addPacket( + OSCMessage( + address + "/Local", + oscArgs.clone(), + ), + ) + } } } diff --git a/server/core/src/main/java/dev/slimevr/osc/VRCOSCHandler.kt b/server/core/src/main/java/dev/slimevr/osc/VRCOSCHandler.kt index ebb7b75653..1d284ef6e2 100644 --- a/server/core/src/main/java/dev/slimevr/osc/VRCOSCHandler.kt +++ b/server/core/src/main/java/dev/slimevr/osc/VRCOSCHandler.kt @@ -263,6 +263,7 @@ class VRCOSCHandler( ) } + newOscReceiver.dispatcher.setAlwaysDispatchingImmediately(true) newOscReceiver.startListening() // Advertise our new receiving port over OSCQuery diff --git a/server/core/src/main/java/dev/slimevr/protocol/rpc/settings/RPCSettingsBuilder.kt b/server/core/src/main/java/dev/slimevr/protocol/rpc/settings/RPCSettingsBuilder.kt index 05ac3ef16a..da4943f343 100644 --- a/server/core/src/main/java/dev/slimevr/protocol/rpc/settings/RPCSettingsBuilder.kt +++ b/server/core/src/main/java/dev/slimevr/protocol/rpc/settings/RPCSettingsBuilder.kt @@ -8,7 +8,7 @@ import dev.slimevr.config.DriftCompensationConfig import dev.slimevr.config.FiltersConfig import dev.slimevr.config.HIDConfig import dev.slimevr.config.LegTweaksConfig -import dev.slimevr.config.OSCConfig +import dev.slimevr.config.OSCRouterConfig import dev.slimevr.config.ResetsConfig import dev.slimevr.config.SkeletonConfig import dev.slimevr.config.StayAlignedConfig @@ -45,7 +45,7 @@ import solarxr_protocol.rpc.settings.SkeletonHeight fun createOSCRouterSettings( fbb: FlatBufferBuilder, - config: OSCConfig, + config: OSCRouterConfig, ): Int { val addressStringOffset = fbb.createString(config.address) @@ -60,6 +60,7 @@ fun createOSCRouterSettings( OSCRouterSettings.startOSCRouterSettings(fbb) OSCRouterSettings.addOscSettings(fbb, oscSettingOffset) + OSCRouterSettings.addRescaleTracking(fbb, config.rescaleTracking) return OSCRouterSettings.endOSCRouterSettings(fbb) } diff --git a/server/core/src/main/java/dev/slimevr/protocol/rpc/settings/RPCSettingsHandler.kt b/server/core/src/main/java/dev/slimevr/protocol/rpc/settings/RPCSettingsHandler.kt index 4b0a95659b..60e4b964c3 100644 --- a/server/core/src/main/java/dev/slimevr/protocol/rpc/settings/RPCSettingsHandler.kt +++ b/server/core/src/main/java/dev/slimevr/protocol/rpc/settings/RPCSettingsHandler.kt @@ -88,6 +88,7 @@ class RPCSettingsHandler(var rpcHandler: RPCHandler, var api: ProtocolAPI) { oscRouterConfig.portOut = osc.portOut() oscRouterConfig.address = osc.address() } + oscRouterConfig.rescaleTracking = req.oscRouter().rescaleTracking() oscRouter.refreshSettings(true) } diff --git a/server/core/src/main/java/dev/slimevr/tracking/processor/skeleton/HumanSkeleton.kt b/server/core/src/main/java/dev/slimevr/tracking/processor/skeleton/HumanSkeleton.kt index da16526691..01646be2cd 100644 --- a/server/core/src/main/java/dev/slimevr/tracking/processor/skeleton/HumanSkeleton.kt +++ b/server/core/src/main/java/dev/slimevr/tracking/processor/skeleton/HumanSkeleton.kt @@ -545,13 +545,26 @@ class HumanSkeleton( StayAligned.adjustNextTracker(trackerSkeleton, stayAlignedConfig) + // Update bone rotations with direct estimates from their nearest rotational tracker + // Head (and hand) positions are also updated to support head-tracker-only AND legacy path for head+hand trackers + // In both cases, ikSolver won't run and resolve positions automatically, so they need to be set here updateTransforms() - updateBones() + + // Try to solve for positional constrains within the skeletal hierarchy + // This supports not just head+hand trackers like legacy path below, but any kind of positional tracker + val solvedSkeletalHierarchy = ikSolver.solve() + + // Legacy Path: Hierarchy is split for each positional tracker (head+hand), update hierarchies individually + if (!solvedSkeletalHierarchy) headBone.update() + if (isTrackingLeftArmFromController) leftHandTrackerBone.update() + if (isTrackingRightArmFromController) rightHandTrackerBone.update() + if (enforceConstraints) { // TODO re-enable toggling correctConstraints once // https://github.com/SlimeVR/SlimeVR-Server/issues/1297 is solved headBone.updateWithConstraints(false) } + updateComputedTrackers() // Don't run post-processing if the tracking is paused @@ -570,16 +583,6 @@ class HumanSkeleton( } } - /** - * Update all the bones by updating the roots - */ - @ThreadSafe - fun updateBones() { - headBone.update() - if (isTrackingLeftArmFromController) leftHandTrackerBone.update() - if (isTrackingRightArmFromController) rightHandTrackerBone.update() - } - /** * Update all the bones' transforms from trackers */ @@ -999,6 +1002,12 @@ class HumanSkeleton( lowerArmTracker: Tracker?, handTracker: Tracker?, ) { + // Get shoulder rotation + var armRot = shoulderTracker?.getRotation() ?: upperChestBone.getLocalRotation() + // Set shoulder rotation + upperShoulderBone.setRotation(upperChestBone.getLocalRotation()) + shoulderBone.setRotation(armRot) + if (isTrackingFromController) { // From controller // Set hand rotation and position from tracker handTracker?.let { @@ -1008,7 +1017,7 @@ class HumanSkeleton( } // Get lower arm rotation - var armRot = getFirstAvailableTracker(lowerArmTracker, upperArmTracker)?.getRotation() ?: IDENTITY + armRot = getFirstAvailableTracker(lowerArmTracker, upperArmTracker)?.getRotation() ?: IDENTITY // Set lower arm rotation lowerArmBone.setRotation(armRot) @@ -1017,11 +1026,6 @@ class HumanSkeleton( // Set elbow tracker rotation elbowTrackerBone.setRotation(armRot) } else { // From HMD - // Get shoulder rotation - var armRot = shoulderTracker?.getRotation() ?: upperChestBone.getLocalRotation() - // Set shoulder rotation - upperShoulderBone.setRotation(upperChestBone.getLocalRotation()) - shoulderBone.setRotation(armRot) if (upperArmTracker != null || lowerArmTracker != null) { // Get upper arm rotation @@ -1036,7 +1040,6 @@ class HumanSkeleton( lowerArmBone.setRotation(armRot) } else { // Fallback arm rotation as upper chest - armRot = upperChestBone.getLocalRotation() upperArmBone.setRotation(armRot) elbowTrackerBone.setRotation(armRot) lowerArmBone.setRotation(armRot) diff --git a/server/core/src/main/java/dev/slimevr/tracking/processor/skeleton/IKConstraint.kt b/server/core/src/main/java/dev/slimevr/tracking/processor/skeleton/IKConstraint.kt index 0642ce0cbc..5bc188cb6d 100644 --- a/server/core/src/main/java/dev/slimevr/tracking/processor/skeleton/IKConstraint.kt +++ b/server/core/src/main/java/dev/slimevr/tracking/processor/skeleton/IKConstraint.kt @@ -1,9 +1,9 @@ package dev.slimevr.tracking.processor.skeleton import dev.slimevr.tracking.trackers.Tracker +import dev.slimevr.tracking.trackers.TrackerPosition import io.github.axisangles.ktmath.Quaternion import io.github.axisangles.ktmath.Vector3 -import solarxr_protocol.datatypes.BodyPart class IKConstraint(val tracker: Tracker) { private var offset = Vector3.NULL @@ -12,10 +12,20 @@ class IKConstraint(val tracker: Tracker) { fun getPosition(): Vector3 = tracker.position + (tracker.getRotation() * rotationOffset).sandwich(offset) fun reset(nodePosition: Vector3) { - val bodyPartsToSkip = setOf(BodyPart.LEFT_HAND, BodyPart.RIGHT_HAND) + // HMD on Head and Controllers in hands are assumed to be perfectly aligned with the bones they're intended to track + // Other generic trackers need to be calibrated from mounting position to actual body part + // TODO: Make positional mounting calibration configurable with sensible defaults + // Generally HMD on head and Controllers in hands don't need to be calibrated + // But sometimes controllers may be used on top of the hands, e.g. for tracking gloves + + if ((tracker.isHmd && tracker.trackerPosition == TrackerPosition.HEAD) || + tracker.trackerPosition == TrackerPosition.LEFT_HAND || + tracker.trackerPosition == TrackerPosition.RIGHT_HAND + ) { + return + } rotationOffset = tracker.getRotation().inv() - if (tracker.trackerPosition?.bodyPart in bodyPartsToSkip) return offset = nodePosition - tracker.position } } diff --git a/server/core/src/main/java/dev/slimevr/tracking/processor/skeleton/IKSolver.kt b/server/core/src/main/java/dev/slimevr/tracking/processor/skeleton/IKSolver.kt index ad18c2e90d..26739bb47d 100644 --- a/server/core/src/main/java/dev/slimevr/tracking/processor/skeleton/IKSolver.kt +++ b/server/core/src/main/java/dev/slimevr/tracking/processor/skeleton/IKSolver.kt @@ -169,7 +169,8 @@ class IKSolver(private val root: Bone) { for (t in trackers) { if (t.hasPosition && !t.isInternal && - !t.status.reset + !t.status.reset && + t.trackerPosition != null ) { constraintList.add(t) } @@ -182,7 +183,8 @@ class IKSolver(private val root: Bone) { for (t in trackers) { if (t.hasRotation && !t.status.reset && - !t.isInternal + !t.isInternal && + t.trackerPosition != null ) { constrainList.add(t) } @@ -225,8 +227,10 @@ class IKSolver(private val root: Bone) { return solved } - fun solve() { - if (rootChain == null || !enabled) return + fun solve(): Boolean { + if (rootChain == null || !enabled) { + return false + } if (needsReset) { for (c in chainList) { @@ -235,11 +239,12 @@ class IKSolver(private val root: Bone) { needsReset = false } - rootChain?.resetChain() root.update() + rootChain?.resetChain() solve(MAX_ITERATIONS) root.update() + return true } } diff --git a/server/core/src/main/java/dev/slimevr/tracking/trackers/Tracker.kt b/server/core/src/main/java/dev/slimevr/tracking/trackers/Tracker.kt index 4262b2abe9..bc132e54ee 100644 --- a/server/core/src/main/java/dev/slimevr/tracking/trackers/Tracker.kt +++ b/server/core/src/main/java/dev/slimevr/tracking/trackers/Tracker.kt @@ -84,7 +84,7 @@ class Tracker @JvmOverloads constructor( */ var allowVelocity: Boolean = false, - val isHmd: Boolean = false, + var isHmd: Boolean = false, /** * If true, the tracker need the user to perform a reset diff --git a/solarxr-protocol b/solarxr-protocol index 3ffd218a83..9280656e79 160000 --- a/solarxr-protocol +++ b/solarxr-protocol @@ -1 +1 @@ -Subproject commit 3ffd218a83a65be046a6676f1cde79a91bc8bc69 +Subproject commit 9280656e79d9f5f9a7fb1ef148f3d50cb7b39965