diff --git a/Cargo.lock b/Cargo.lock index eb30f8dd5..2a151ff3d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6171,6 +6171,7 @@ dependencies = [ "base64 0.22.1", "bevy", "bevy_console", + "bevy_kira_audio", "clap", "common", "console", @@ -11777,6 +11778,7 @@ dependencies = [ "dcl", "dcl_component", "input_manager", + "ipfs", "rapier3d-f64", "scene_runner", "tween", diff --git a/crates/av/src/audio_loader.rs b/crates/av/src/audio_loader.rs new file mode 100644 index 000000000..76767580d --- /dev/null +++ b/crates/av/src/audio_loader.rs @@ -0,0 +1,51 @@ +use bevy::asset::{io::Reader, AssetLoader, LoadContext}; +use bevy_kira_audio::AudioSource; +use kira::sound::{static_sound::StaticSoundData, FromFileError}; +use std::io::Cursor; +use thiserror::Error; + +// Format-agnostic loader for scene-content audio assets served without a file +// extension on the wire. Kira's `StaticSoundData::from_cursor` internally uses +// symphonia's probe to sniff the container (mp3/ogg/wav/flac) from the byte +// stream, so we don't need to know the original extension up front. +#[derive(Default)] +pub struct AudioAssetLoader; + +#[non_exhaustive] +#[derive(Debug, Error)] +pub enum AudioLoaderError { + #[error("Could not read audio asset: {0}")] + Io(#[from] std::io::Error), + #[error("Error decoding audio asset: {0}")] + FileError(#[from] FromFileError), +} + +impl AssetLoader for AudioAssetLoader { + type Asset = AudioSource; + type Settings = (); + type Error = AudioLoaderError; + + async fn load( + &self, + reader: &mut dyn Reader, + _settings: &(), + load_context: &mut LoadContext<'_>, + ) -> Result { + let mut bytes = Vec::new(); + reader.read_to_end(&mut bytes).await?; + let byte_count = bytes.len(); + let sound = StaticSoundData::from_cursor(Cursor::new(bytes))?; + bevy::log::debug!( + "loaded .audio asset {:?} ({} bytes, {} frames @ {} Hz)", + load_context.path(), + byte_count, + sound.frames.len(), + sound.sample_rate, + ); + Ok(AudioSource { sound }) + } + + fn extensions(&self) -> &[&str] { + &["audio"] + } +} diff --git a/crates/av/src/audio_source.rs b/crates/av/src/audio_source.rs index 211fb8278..c30df0dd8 100644 --- a/crates/av/src/audio_source.rs +++ b/crates/av/src/audio_source.rs @@ -33,6 +33,9 @@ impl Plugin for AudioSourcePlugin { fn build(&self, app: &mut App) { // we use kira for audio source asset management, regardless of native / wasm app.add_plugins(bevy_kira_audio::AudioPlugin); + // Custom `.audio` loader for scene-content clips fetched by content hash + // (no on-wire extension). Format is detected from the byte stream. + app.register_asset_loader(crate::audio_loader::AudioAssetLoader); app.add_event::(); app.add_crdt_lww_component::( SceneComponentId::AUDIO_SOURCE, diff --git a/crates/av/src/audio_source_wasm.rs b/crates/av/src/audio_source_wasm.rs index c1a10b982..0a6bdd81f 100644 --- a/crates/av/src/audio_source_wasm.rs +++ b/crates/av/src/audio_source_wasm.rs @@ -165,7 +165,15 @@ fn manage_audio_sources( Option<&RenderLayers>, Option<&RetryEmitter>, ), - Or<(Changed, With)>, + Or<( + Changed, + With, + // Include entities queued for retry: on wasm we can't start + // playback until the web AudioBuffer is decoded, so the first + // frame often fails and we insert RetryEmitter without a Playing + // marker. Without this term the retry never fires. + With, + )>, >, mut audio: NonSendMut, containing_scene: ContainingScene, diff --git a/crates/av/src/lib.rs b/crates/av/src/lib.rs index ec3d30aa1..2ef0354ed 100644 --- a/crates/av/src/lib.rs +++ b/crates/av/src/lib.rs @@ -16,6 +16,7 @@ pub mod video_context; pub mod video_stream; // audio source (non-streaming audio) +pub mod audio_loader; pub mod audio_source; #[cfg(not(feature = "html"))] pub mod audio_source_native; diff --git a/crates/avatar/src/animate.rs b/crates/avatar/src/animate.rs index 8fe400b0d..e2eec90c4 100644 --- a/crates/avatar/src/animate.rs +++ b/crates/avatar/src/animate.rs @@ -19,7 +19,8 @@ use common::{ sets::SceneSets, structs::{ AudioEmitter, AudioType, AvatarDynamicState, EmoteCommand, MoveKind, PlayerModifiers, - PrimaryUser, + PrimaryUser, SceneDrivenAnim, SceneDrivenAnimationFeedback, + SceneDrivenAnimationFeedbackState, }, util::TryPushChildrenEx, }; @@ -58,6 +59,7 @@ impl Plugin for AvatarAnimationPlugin { ( (handle_trigger_emotes, broadcast_emote, receive_emotes).before(animate), (animate, play_current_emote).chain().after(process_avatar), + play_scene_driven_sounds.after(process_avatar), ) .in_set(SceneSets::PostLoop), ); @@ -196,6 +198,19 @@ fn receive_emotes(mut commands: Commands, mut chat_events: EventReader, + /// Whether a `triggerSceneEmote` should be allowed to take over. Mirrors the movement + /// scene's `idle` flag when `source == SceneMovementAnim`; otherwise unused. + overridable: bool, + /// Set by the animate system when the scene publishes a `playback_time` this frame. + /// Consumed exactly once by `play_current_emote`. + pending_seek: Option, + /// Origin of the current selection; dictates override rules and feedback publishing. + source: ActiveEmoteSource, + /// Source path from `MovementAnimation.src`; used to populate feedback and detect + /// cross-fade boundaries between scene-driven animations. + scene_anim_src: Option, + /// Alternate state to play if the primary URN fails to resolve. Populated by + /// `animate` for scene-driven selections with the velocity-based choice; + /// `play_current_emote` swaps to it on resolution failure. + fallback: Option>, } impl Default for ActiveEmote { @@ -217,6 +247,11 @@ impl Default for ActiveEmote { finished: false, transition_seconds: 0.2, initial_audio_mark: None, + overridable: true, + pending_seek: None, + source: ActiveEmoteSource::VelocitySelected, + scene_anim_src: None, + fallback: None, } } } @@ -236,6 +271,7 @@ fn animate( Option<&ContainerEntity>, Option<&PrimaryUser>, Option<&mut LastEmoteCommand>, + Option<&SceneDrivenAnim>, )>, mut velocities: Local>, mut current_emote_min_velocities: Local>, @@ -265,6 +301,7 @@ fn animate( maybe_container, maybe_primary, last_emote, + maybe_scene_anim, ) in avatars.iter_mut() { let Some(mut active_emote) = active_emote else { @@ -286,6 +323,8 @@ fn animate( let damped_velocity_len = damped_velocity.xz().length(); velocities.insert(avatar_ent, damped_velocity); + let scene_anim = maybe_scene_anim.and_then(|a| a.active.as_ref()); + // get requested emote let (mut requested_emote, given_urn, request_loop) = if let Some(EmoteCommand { urn, r#loop, .. }) = emote { @@ -301,13 +340,28 @@ fn animate( requested_emote = None; } + // If the current animation is a non-overridable scene-driven one, silently drop + // any triggerSceneEmote request so the movement scene retains control. + if active_emote.source == ActiveEmoteSource::SceneMovementAnim && !active_emote.overridable + { + requested_emote = None; + } + // check / cancel requested emote if Some(&active_emote.urn) == requested_emote.as_ref() { let playing_min_vel = prior_min_velocities .get(&avatar_ent) .copied() .unwrap_or_default(); - if damped_velocity_len * 0.9 > playing_min_vel { + // A non-idle scene-driven animation takes precedence over a triggered emote. + let scene_cancels = scene_anim.is_some_and(|req| !req.idle); + // Scene-driven animations handle their own motion semantics; don't cancel on move. + let velocity_cancels = + scene_anim.is_none() && active_emote.source != ActiveEmoteSource::SceneMovementAnim; + if scene_cancels { + debug!("clear on scene anim {:?}", active_emote.urn); + requested_emote = None; + } else if velocity_cancels && damped_velocity_len * 0.9 > playing_min_vel { // stop emotes on move debug!( "clear on motion {} > {}", @@ -327,6 +381,89 @@ fn animate( current_emote_min_velocities.insert(avatar_ent, damped_velocity_len); } + // Precompute the velocity-based selection up-front so we can use it both as the + // fallback for scene-driven anims (in case the URN fails to resolve) and as the + // final default when nothing else claims the avatar. + let time_to_peak = (jump_height * -gravity * 2.0).sqrt() / -gravity; + let just_jumped = + dynamic_state.jump_time > (time.elapsed_secs() - time_to_peak / 2.0).max(0.0); + let (velocity_emote, velocity_move_kind) = if dynamic_state.ground_height > 0.2 + || (dynamic_state.velocity.y > 0.0 && just_jumped) + { + let move_kind = if just_jumped { + MoveKind::Jump + } else { + MoveKind::Falling + }; + ( + ActiveEmote { + urn: EmoteUrn::new("jump").unwrap(), + speed: time_to_peak.recip() * 0.5, + repeat: true, + restart: dynamic_state.jump_time > time.elapsed_secs() - time.delta_secs(), + transition_seconds: 0.1, + initial_audio_mark: if !just_jumped { Some(0.1) } else { None }, + ..Default::default() + }, + move_kind, + ) + } else if active_emote.urn == EmoteUrn::new("jump").unwrap() && !active_emote.finished { + ( + ActiveEmote { + urn: EmoteUrn::new("jump").unwrap(), + speed: 1.5, + repeat: false, + restart: false, + transition_seconds: 0.1, + initial_audio_mark: Some(0.1), + ..Default::default() + }, + dynamic_state.move_kind, + ) + } else { + let directional_velocity_len = + (damped_velocity * (Vec3::X + Vec3::Z)).dot(gt.forward().as_vec3()); + if damped_velocity_len.abs() > 0.1 { + if damped_velocity_len.abs() <= 2.6 { + ( + ActiveEmote { + urn: EmoteUrn::new("walk").unwrap(), + speed: directional_velocity_len / 1.5, + restart: false, + repeat: true, + transition_seconds: 0.4, + ..Default::default() + }, + MoveKind::Walk, + ) + } else { + ( + ActiveEmote { + urn: EmoteUrn::new("run").unwrap(), + speed: directional_velocity_len / 4.5, + restart: false, + repeat: true, + transition_seconds: 0.4, + ..Default::default() + }, + MoveKind::Jog, + ) + } + } else { + ( + ActiveEmote { + urn: EmoteUrn::new("idle_male").unwrap(), + speed: 1.0, + restart: false, + repeat: true, + transition_seconds: 0.4, + ..Default::default() + }, + MoveKind::Idle, + ) + } + }; + // play requested emote *active_emote = if let Some(requested_emote) = requested_emote { if emote_changed { @@ -380,85 +517,46 @@ fn animate( urn: requested_emote, restart: emote_changed, repeat: request_loop, + source: ActiveEmoteSource::TriggeredEmote, ..Default::default() } - } else { - // otherwise play a default emote based on motion - let time_to_peak = (jump_height * -gravity * 2.0).sqrt() / -gravity; - let just_jumped = - dynamic_state.jump_time > (time.elapsed_secs() - time_to_peak / 2.0).max(0.0); - if dynamic_state.ground_height > 0.2 || (dynamic_state.velocity.y > 0.0 && just_jumped) - { - if just_jumped { - dynamic_state.move_kind = MoveKind::Jump; - } else { - dynamic_state.move_kind = MoveKind::Falling; - } - ActiveEmote { - urn: EmoteUrn::new("jump").unwrap(), - speed: time_to_peak.recip() * 0.5, - repeat: true, - restart: dynamic_state.jump_time > time.elapsed_secs() - time.delta_secs(), - transition_seconds: 0.1, - initial_audio_mark: if !just_jumped { Some(0.1) } else { None }, - ..Default::default() - } - } else if active_emote.urn == EmoteUrn::new("jump").unwrap() && !active_emote.finished { - // finish the jump - we use `repeat: false` to signal that we are landing... - ActiveEmote { - urn: EmoteUrn::new("jump").unwrap(), - speed: 1.5, - repeat: false, - restart: false, - transition_seconds: 0.1, - initial_audio_mark: Some(0.1), - ..Default::default() - } + } else if let Some(scene_anim_req) = + scene_anim.and_then(|req| EmoteUrn::new(req.urn.as_str()).ok().map(|urn| (req, urn))) + { + let (req, urn) = scene_anim_req; + dynamic_state.move_kind = if req.idle { + MoveKind::Idle } else { - let directional_velocity_len = - (damped_velocity * (Vec3::X + Vec3::Z)).dot(gt.forward().as_vec3()); - - if damped_velocity_len.abs() > 0.1 { - if damped_velocity_len.abs() <= 2.6 { - dynamic_state.move_kind = MoveKind::Walk; - ActiveEmote { - urn: EmoteUrn::new("walk").unwrap(), - speed: directional_velocity_len / 1.5, - restart: false, - repeat: true, - transition_seconds: 0.4, - ..Default::default() - } - } else { - dynamic_state.move_kind = MoveKind::Jog; - ActiveEmote { - urn: EmoteUrn::new("run").unwrap(), - speed: directional_velocity_len / 4.5, - restart: false, - repeat: true, - transition_seconds: 0.4, - ..Default::default() - } - } - } else { - dynamic_state.move_kind = MoveKind::Idle; - ActiveEmote { - urn: EmoteUrn::new("idle_male").unwrap(), - speed: 1.0, - restart: false, - repeat: true, - transition_seconds: 0.4, - ..Default::default() - } - } + MoveKind::Walk + }; + // Detect anim change via URN (stable across local/remote sources) rather than src, + // which is empty for requests received over the network. + let is_new_anim = active_emote.source != ActiveEmoteSource::SceneMovementAnim + || active_emote.urn != urn; + ActiveEmote { + urn, + speed: req.speed, + restart: is_new_anim, + repeat: req.r#loop, + finished: false, + transition_seconds: req.transition_seconds, + initial_audio_mark: None, + overridable: req.idle, + pending_seek: req.seek, + source: ActiveEmoteSource::SceneMovementAnim, + scene_anim_src: Some(req.src.clone()), + fallback: Some(Box::new(velocity_emote)), } + } else { + dynamic_state.move_kind = velocity_move_kind; + velocity_emote } } } struct SpawnedExtras { urn: EmoteUrn, - scene: Option, + scene: Option<(Entity, InstanceId)>, scene_initialized: bool, audio: Option<(Entity, f32)>, clip: Option<(AnimationNodeIndex, Handle)>, @@ -479,7 +577,13 @@ impl SpawnedExtras { #[allow(clippy::too_many_arguments, clippy::type_complexity)] fn play_current_emote( mut commands: Commands, - mut q: Query<(Entity, &mut ActiveEmote, &AvatarAnimPlayer, &Children)>, + mut q: Query<( + Entity, + &mut ActiveEmote, + &AvatarAnimPlayer, + &Children, + Option<&PrimaryUser>, + )>, definitions: Query<&AvatarDefinition>, mut emote_loader: CollectibleManager, mut gltfs: ResMut>, @@ -501,11 +605,15 @@ fn play_current_emote( ), mut emitters: Query<&mut AudioEmitter>, prop_details: Query<(Option<&Name>, &Transform, &ChildOf)>, + (mut feedback, mut frozen_feedback): ( + ResMut, + Local>, + ), ) { let prior_playing = std::mem::take(&mut *playing); let mut prev_spawned_extras = std::mem::take(&mut *spawned_extras); - for (entity, mut active_emote, target_entity, children) in q.iter_mut() { + for (entity, mut active_emote, target_entity, children, maybe_primary) in q.iter_mut() { debug!("emote {}", active_emote.urn); let Some(definition) = children.iter().flat_map(|c| definitions.get(c).ok()).next() else { warn!("no definition"); @@ -515,8 +623,11 @@ fn play_current_emote( // clean up old extras if let Some(extras) = prev_spawned_extras.remove(&entity) { if extras.urn != active_emote.urn { - if let Some(scene) = extras.scene { + if let Some((wrapper, scene)) = extras.scene { scene_spawner.despawn_instance(scene); + if let Ok(mut commands) = commands.get_entity(wrapper) { + commands.despawn(); + } } if let Some((audio_ent, _)) = extras.audio.as_ref() { @@ -532,109 +643,159 @@ fn play_current_emote( let ent = target_entity.0; let bodyshape = &definition.body_shape; - if let Some(scene_emote) = active_emote.urn.scene_emote() { - debug!("got {scene_emote:?}"); - let mut split = scene_emote.split('-'); - let maybe_hash = if split.next() == Some("b64") { - // stupid to have "-" as a separator and also part of the hash itself for local hashes - let parts = split.skip(1).take(2).collect::>(); - (parts.len() == 2).then_some(parts.join("-")) - } else { - split.next().map(ToOwned::to_owned) - }; - let Some(hash) = maybe_hash.as_ref() else { - debug!("failed to split scene emote {scene_emote:?}"); - active_emote.finished = true; - continue; - }; - - if emote_loader - .get_representation(&active_emote.urn, bodyshape.as_str()) - .is_err() - { - // load the gltf - let handle = ipfas.load_hash::(hash); - let gltf = match gltfs.get_mut(handle.id()) { - Some(gltf) => { - cached_gltf_handles.remove(&handle); - gltf + // Resolve the URN. On permanent failure, if a fallback is present (populated by + // `animate` for scene-driven anims with the velocity-based choice), swap to it + // and retry. The swap persists so downstream `SceneDrivenAnimationFeedback` + // publishing reports the fallback source, not the failed scene-driven one. + enum Outcome { + Ready, + Loading, + Failed, + } + let outcome = 'resolve: loop { + if let Some(scene_emote) = active_emote.urn.scene_emote() { + debug!("got {scene_emote:?}"); + let mut split = scene_emote.split('-').peekable(); + // take_hash reads a hash, recombining "b64-" back into one + // token because we used '-' as the separator and b64 hashes also + // contain '-'. for non-b64 hashes it just takes the next token. + let take_hash = + |split: &mut std::iter::Peekable>| -> Option { + let first = split.next()?; + if first == "b64" { + let tail = split.next()?; + Some(format!("b64-{tail}")) + } else { + Some(first.to_owned()) + } + }; + let Some(scene_hash) = take_hash(&mut split) else { + debug!("failed to split scene emote {scene_emote:?}"); + if let Some(fb) = active_emote.fallback.take() { + *active_emote = *fb; + continue 'resolve; } - None => { - cached_gltf_handles.insert(handle); - continue; + break 'resolve Outcome::Failed; + }; + let Some(hash) = take_hash(&mut split) else { + debug!("failed to split scene emote {scene_emote:?}"); + if let Some(fb) = active_emote.fallback.take() { + *active_emote = *fb; + continue 'resolve; } + break 'resolve Outcome::Failed; }; - // fix up the gltf if possible/required - if !gltf.named_animations.keys().any(|k| k.ends_with("_Avatar")) { - let Some(anim) = gltf.animations.first() else { - warn!("scene emote has no animations"); - active_emote.finished = true; - continue; + if emote_loader + .get_representation(&active_emote.urn, bodyshape.as_str()) + .is_err() + { + // load the gltf through the scene's modifier context so b64 + // hashes (local preview / portable) resolve to the scene's + // origin rather than the realm content URL. + let handle = ipfas.load_scene_content_hash::(&scene_hash, &hash); + let gltf = match gltfs.get_mut(handle.id()) { + Some(gltf) => { + cached_gltf_handles.remove(&handle); + gltf + } + None => { + cached_gltf_handles.insert(handle); + break 'resolve Outcome::Loading; + } }; - gltf.named_animations.insert("_Avatar".into(), anim.clone()); - } + // fix up the gltf if possible/required + if !gltf.named_animations.keys().any(|k| k.ends_with("_Avatar")) { + let Some(anim) = gltf.animations.first() else { + warn!("scene emote has no animations"); + if let Some(fb) = active_emote.fallback.take() { + *active_emote = *fb; + continue 'resolve; + } + break 'resolve Outcome::Failed; + }; + + gltf.named_animations.insert("_Avatar".into(), anim.clone()); + } - // add repr - emote_loader.add_builtin( - active_emote.urn.clone(), - Collectible { - representations: HashMap::from_iter([( - bodyshape.to_owned(), - Emote { - gltf: handle, - default_repeat: false, - sound: Vec::default(), + // add repr + emote_loader.add_builtin( + active_emote.urn.clone(), + Collectible { + representations: HashMap::from_iter([( + bodyshape.to_owned(), + Emote { + gltf: handle, + default_repeat: false, + sound: Vec::default(), + }, + )]), + data: CollectibleData:: { + hash: hash.to_owned(), + urn: active_emote.urn.as_str().to_owned(), + thumbnail: "embedded://images/redx.png".to_owned(), + available_representations: HashSet::from_iter([ + bodyshape.to_owned() + ]), + name: active_emote.urn.to_string(), + description: active_emote.urn.to_string(), + extra_data: (), }, - )]), - data: CollectibleData:: { - hash: hash.to_owned(), - urn: active_emote.urn.as_str().to_owned(), - thumbnail: "embedded://images/redx.png".to_owned(), - available_representations: HashSet::from_iter([bodyshape.to_owned()]), - name: active_emote.urn.to_string(), - description: active_emote.urn.to_string(), - extra_data: (), }, - }, - ); + ); + } } - } - let emote = match emote_loader.get_representation(&active_emote.urn, bodyshape.as_str()) { - Ok(emote) => emote, - e @ Err(CollectibleError::Failed) - | e @ Err(CollectibleError::Missing) - | e @ Err(CollectibleError::NoRepresentation) => { - debug!("{} -> {:?}", active_emote.urn, e); - active_emote.finished = true; - continue; + match emote_loader.get_representation(&active_emote.urn, bodyshape.as_str()) { + Ok(emote) => match emote.avatar_animation(&gltfs) { + Ok(Some(_)) => break 'resolve Outcome::Ready, + Err(e) => { + debug!("animation error: {:?}", e); + break 'resolve Outcome::Loading; + } + Ok(None) => { + debug!("{} -> no clip", active_emote.urn); + if let Some(fb) = active_emote.fallback.take() { + *active_emote = *fb; + continue 'resolve; + } + break 'resolve Outcome::Failed; + } + }, + Err(CollectibleError::Loading) => { + debug!("{} -> loading", active_emote.urn); + break 'resolve Outcome::Loading; + } + Err(e) => { + debug!("{} -> {:?}", active_emote.urn, e); + if let Some(fb) = active_emote.fallback.take() { + *active_emote = *fb; + continue 'resolve; + } + break 'resolve Outcome::Failed; + } } - Err(CollectibleError::Loading) => { - debug!("{} -> loading", active_emote.urn); + }; + + match outcome { + Outcome::Ready => {} + Outcome::Loading => continue, + Outcome::Failed => { + active_emote.finished = true; continue; } + } + + let emote = match emote_loader.get_representation(&active_emote.urn, bodyshape.as_str()) { + Ok(emote) => emote, + _ => continue, }; active_emote.repeat |= emote.default_repeat; let clip = match emote.avatar_animation(&gltfs) { - Err(e) => { - debug!("animation error: {:?}", e); - continue; - } - Ok(None) => { - debug!("{} -> no clip", active_emote.urn); - debug!( - "available : {:?}", - gltfs - .get(emote.gltf.id()) - .map(|gltf| gltf.named_animations.keys().collect::>()) - ); - active_emote.finished = true; - continue; - } Ok(Some(clip)) => clip, + _ => continue, }; // extract props and prop anim @@ -642,7 +803,7 @@ fn play_current_emote( if let Ok(Some(props)) = emote.prop_scene(&gltfs) { debug!("got props"); if let Some(extras) = spawned_extras.get_mut(&entity) { - let Some(instance) = extras.scene else { + let Some((wrapper, instance)) = extras.scene else { continue; }; @@ -667,19 +828,18 @@ fn play_current_emote( } } - if parent.parent() == entity { + if parent.parent() == wrapper { // children of root nodes -> rotate - if parent.parent() == entity { - let mut rotated = *transform; - rotated.rotate_around( - Vec3::ZERO, - Quat::from_rotation_y(std::f32::consts::PI), - ); - commands.entity(spawned_ent).try_insert(rotated); - } + let mut rotated = *transform; + rotated.rotate_around( + Vec3::ZERO, + Quat::from_rotation_y(std::f32::consts::PI), + ); + commands.entity(spawned_ent).try_insert(rotated); } } } + commands.entity(wrapper).try_insert(Visibility::Inherited); extras.scene_initialized = true; } @@ -709,11 +869,14 @@ fn play_current_emote( } } } else { - let scene = scene_spawner.spawn_as_child(props, entity); + let wrapper = commands + .spawn((Transform::default(), Visibility::Hidden, ChildOf(entity))) + .id(); + let scene = scene_spawner.spawn_as_child(props, wrapper); spawned_extras .entry(entity) .or_insert_with(|| SpawnedExtras::new(active_emote.urn.clone())) - .scene = Some(scene); + .scene = Some((wrapper, scene)); continue; } } @@ -772,7 +935,8 @@ fn play_current_emote( let play = |transitions: Option>, player: &mut AnimationPlayer, clip_ix: AnimationNodeIndex, - active_emote: &ActiveEmote| + active_emote: &ActiveEmote, + pending_seek: Option| -> f32 { let active_animation = if Some(&active_emote.urn) != prior_playing.get(&ent) || active_emote.restart { @@ -804,6 +968,16 @@ fn play_current_emote( // println!("active weight {}", active_animation.weight()); active_animation.set_speed(active_emote.speed); + if let Some(seek) = pending_seek { + // `replay()` in bevy_animation resets `seek_time` to 0.0 + // (among other state), so it must run BEFORE `seek_to` + // or the seek is clobbered. We still call it so a non- + // looping clip that has completed can be restarted by a + // new seek. + active_animation.replay(); + active_animation.seek_to(seek.clamp(0.0, clip_duration)); + } + // nasty hack for falling animation if active_emote.urn.as_str() == "urn:decentraland:off-chain:base-emotes:jump" && active_animation.seek_time() >= 0.4 @@ -843,7 +1017,14 @@ fn play_current_emote( (graph.add_clip(clip, 1.0, graph.root), 0.0) }); - let elapsed = play(transitions, &mut player, *clip_ix, &active_emote); + let pending_seek = active_emote.pending_seek.take(); + let elapsed = play( + transitions, + &mut player, + *clip_ix, + &active_emote, + pending_seek, + ); // reset audio mark if we've rewound (jump hacks again) if let Some(mark) = spawned_extras .get_mut(&entity) @@ -862,7 +1043,43 @@ fn play_current_emote( if let Some((prop_player_ents, clip_ix)) = prop_player_and_clip { for ent in prop_player_ents { if let Ok((mut player, transitions, _, _)) = players.get_mut(ent) { - play(transitions, &mut player, clip_ix, &active_emote); + play(transitions, &mut player, clip_ix, &active_emote, None); + } + } + } + + if maybe_primary.is_some() { + match active_emote.source { + ActiveEmoteSource::SceneMovementAnim => { + let loops = elapsed / clip_duration; + let playback_time = if active_emote.repeat && clip_duration > 0.0 { + elapsed - loops.floor() * clip_duration + } else { + elapsed.min(clip_duration) + }; + let loop_count = if clip_duration > 0.0 { + loops.floor().max(0.0) as u32 + } else { + 0 + }; + let state = SceneDrivenAnimationFeedbackState { + src: active_emote.scene_anim_src.clone().unwrap_or_default(), + r#loop: active_emote.repeat, + speed: active_emote.speed, + idle: active_emote.overridable, + playback_time, + duration: clip_duration, + loop_count, + }; + *frozen_feedback = Some(state.clone()); + feedback.state = Some(state); + } + ActiveEmoteSource::TriggeredEmote => { + feedback.state = frozen_feedback.clone(); + } + ActiveEmoteSource::VelocitySelected => { + *frozen_feedback = None; + feedback.state = None; } } } @@ -954,3 +1171,51 @@ fn emote_console_command( input.ok(); } } + +// Plays avatar-bus audio clips requested by a scene-driven movement animation. +// Dedups against the last observed sound list per avatar so that the scene holding +// the same list across frames doesn't re-fire sounds — a new play is triggered +// only when the list transitions to a different value (including the scene clearing +// and re-asserting it on a later frame). +fn play_scene_driven_sounds( + mut commands: Commands, + avatars: Query<(Entity, &SceneDrivenAnim)>, + ipfas: IpfsAssetServer, + mut last_sounds: Local>>, +) { + let mut seen: HashSet = HashSet::default(); + for (entity, scene_anim) in avatars.iter() { + seen.insert(entity); + let Some(active) = scene_anim.active.as_ref() else { + last_sounds.remove(&entity); + continue; + }; + let prev = last_sounds.get(&entity); + if prev.map(|v| v.as_slice()) == Some(active.sounds.as_slice()) { + continue; + } + for content_hash in &active.sounds { + let handle = ipfas.load_scene_content_hash::( + &active.scene_hash, + content_hash, + ); + let audio_entity = commands + .spawn(( + Transform::default(), + Visibility::default(), + AudioEmitter { + handle, + ty: AudioType::Avatar, + ..Default::default() + }, + )) + .id(); + if let Ok(mut entity_commands) = commands.get_entity(entity) { + entity_commands.try_push_children(&[audio_entity]); + } + } + last_sounds.insert(entity, active.sounds.clone()); + } + // Drop tracked state for avatars that no longer have the component. + last_sounds.retain(|e, _| seen.contains(e)); +} diff --git a/crates/avatar/src/foreign_dynamics.rs b/crates/avatar/src/foreign_dynamics.rs index e029f3303..89bd3633c 100644 --- a/crates/avatar/src/foreign_dynamics.rs +++ b/crates/avatar/src/foreign_dynamics.rs @@ -1,6 +1,9 @@ use bevy::prelude::*; -use common::{structs::AvatarDynamicState, util::QuatNormalizeExt}; +use common::{ + structs::{AvatarDynamicState, SceneDrivenAnim, SceneDrivenAnimationRequest}, + util::QuatNormalizeExt, +}; use comms::{ global_crdt::{ForeignPlayer, PlayerPositionEvent}, @@ -35,6 +38,12 @@ struct PlayerTargetPosition { update_freq: f32, grounded: Option, jumping: Option, + // Scene-driven animation state carried with this target. Applied to + // `SceneDrivenAnim` at `anim_apply_at` so it lines up with the interpolated + // position, then `anim_applied` blocks re-application until the next packet. + scene_anim: Option, + anim_apply_at: f32, + anim_applied: bool, } fn update_foreign_user_target_position( @@ -85,6 +94,15 @@ fn update_foreign_user_target_position( let update_freq = LAG_DECAY_SECS / ((LAG_DECAY_SECS - delta).max(0.0) / pos.update_freq + (LAG_DECAY_SECS / delta).min(1.0)); + // Apply-before-overwrite: if the previous event's scene_anim never + // reached its deadline, push it now so bursts of events (stalls, + // multi-event frames) don't silently drop one-shot seeks or + // intermediate transitions. + if !pos.anim_applied { + commands.entity(ev.player).try_insert(SceneDrivenAnim { + active: pos.scene_anim.clone(), + }); + } *pos = PlayerTargetPosition { time: ev.time, timestamp: ev.timestamp, @@ -95,6 +113,9 @@ fn update_foreign_user_target_position( update_freq, grounded: ev.grounded, jumping: ev.jumping, + scene_anim: ev.scene_anim.clone(), + anim_apply_at: ev.time + update_freq, + anim_applied: false, } } } else { @@ -109,6 +130,9 @@ fn update_foreign_user_target_position( update_freq: 0.01, grounded: ev.grounded, jumping: ev.jumping, + scene_anim: ev.scene_anim.clone(), + anim_apply_at: ev.time + 0.01, + anim_applied: false, }, AvatarDynamicState::default(), )); @@ -118,9 +142,10 @@ fn update_foreign_user_target_position( } fn update_foreign_user_actual_position( + mut commands: Commands, mut avatars: Query<( Entity, - &PlayerTargetPosition, + &mut PlayerTargetPosition, &mut Transform, &mut AvatarDynamicState, )>, @@ -128,7 +153,7 @@ fn update_foreign_user_actual_position( containing_scene: ContainingScene, time: Res