Skip to content

lore: Discard reverted uncommitted directory adds on scan #104

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
lehuan5062 wants to merge 10 commits into
base: main
Choose a base branch
from
+475 −122
Open

lore: Discard reverted uncommitted directory adds on scan #104

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
7befc87
lore: Discard reverted uncommitted directory adds on scan
lehuan5062 Jul 9, 2026
d335160
lore: Address maintainer feedback on directory discard
lehuan5062 Jul 10, 2026
e89c9a5
lore: Fix smoke test to actually exercise the directory discard scan …
lehuan5062 Jul 10, 2026
409f2c7
lore: Move shutil import to module scope in discard test
lehuan5062 Jul 10, 2026
5cc9f50
lore: Trim organizational comments from discard test
lehuan5062 Jul 10, 2026
04c827d
Merge branch 'EpicGames:main' into discard-reverted-directory
lehuan5062 Jul 14, 2026
a15c41b
Merge branch 'EpicGames:main' into discard-reverted-directory
lehuan5062 Jul 14, 2026
4da0419
fix: scan discards reverted uncommitted directories and fixes ordering
lehuan5062 Jul 14, 2026
102f674
Merge branch 'EpicGames:main' into discard-reverted-directory
lehuan5062 Jul 16, 2026
6a12b52
Merge branch 'EpicGames:main' into discard-reverted-directory
lehuan5062 Jul 17, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
236 changes: 119 additions & 117 deletions lore-revision/src/repository/status.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1165,6 +1165,125 @@ pub async fn status(
return Ok(());
}

// Scan before the staged diff: the scan discards reverted uncommitted
// adds from the shared staged state, which the staged diff must not
// report as stale adds.
if show_scan {
lore_debug!(
"Calculating deltas against filesystem for {} paths",
paths.len()
);

let mut tasks = JoinSet::new();
for path in paths.iter() {
let repository = repository.clone();
let state_current = state_current.clone();
let state_staged = state_staged.clone();
let path = path.clone();
let layer_mounts = layer_mounts.clone();
let summary = summary.clone();
let exists = if let Some(path) = path.as_ref() {
let mut exists_in_state = false;
let mut exists_in_filesystem = false;

let state = if has_staged {
state_staged.clone()
} else {
state_current.clone()
};

let node_link = state
.find_node_link(repository.clone(), path.as_str())
.await
.unwrap_or_default();
if node_link.is_valid() {
exists_in_state = true;
} else {
let absolute_path = path.to_absolute_path(repository.require_path()?);
exists_in_filesystem = std::fs::exists(absolute_path).unwrap_or_default();
}

if !exists_in_state && !exists_in_filesystem {
emit_path_ignore(path.as_str()).await;
lore_trace!("Ignoring invalid path: {path}");
}

exists_in_state || exists_in_filesystem
} else {
true
};

if exists {
lore_spawn!(tasks, {
async move {
if let Some(path) = path.as_ref() {
lore_debug!(
"Calculating deltas against filesystem path: {}",
path.as_str()
);
} else {
lore_debug!(
"Calculating deltas against filesystem for full repository"
);
}

let start = Instant::now();

// Scan uses staged state as diff base with scan_dirty=true.
// Content hashes in staged state are either zero (add nodes)
// or equal to current revision hashes, so the comparison is
// effectively filesystem vs committed content.
// The current revision is passed as the second pair so the
// walk can distinguish "node exists in staged but not in
// committed" — i.e. unstaged adds — from regular tracked
// files. Dirty flags are set/cleared inline during the walk.
let (changes, _stats) = state::diff_filesystem_ex(
repository.clone(),
state_staged.clone(),
repository.clone(),
state_current.clone(),
path,
FilterMode::Full,
true, // scan_dirty
layer_mounts.clone(),
)
.await
.forward::<StatusError>("computing diff against filesystem")?;

lore_debug!(
"Scan found {} file system changes in {:.3}s",
changes.len(),
start.elapsed().as_secs_f64(),
);

for change in changes.iter() {
let size =
file_size_from_node_change_path(repository.require_path()?, change)
.await?;

// Emit event for display (dirty set/clear handled inline by diff)
if !change.flags.is_stage() {
summary.classify(change);
event::LoreEvent::RepositoryStatusFile(
LoreRepositoryStatusFileEventData::from_node_change(
change, size,
),
)
.send();
} else {
lore_debug!("Ignore staged file {}", change.path);
}
}

Ok(())
}
});
}

lore_drain_tasks!(tasks, StatusError::internal("Recursion task failed"))?;
}
}

// Compare current state against staged state
if show_staged && has_staged {
lore_debug!("Calculating deltas against staged revision");
Expand Down Expand Up @@ -1295,123 +1414,6 @@ pub async fn status(
lore_drain_tasks!(tasks, StatusError::internal("Recursion task failed"))?;
}

// Compare current/staged state against filesystem
if show_scan {
lore_debug!(
"Calculating deltas against filesystem for {} paths",
paths.len()
);

let mut tasks = JoinSet::new();
for path in paths.iter() {
let repository = repository.clone();
let state_current = state_current.clone();
let state_staged = state_staged.clone();
let path = path.clone();
let layer_mounts = layer_mounts.clone();
let summary = summary.clone();
let exists = if let Some(path) = path.as_ref() {
let mut exists_in_state = false;
let mut exists_in_filesystem = false;

let state = if has_staged {
state_staged.clone()
} else {
state_current.clone()
};

let node_link = state
.find_node_link(repository.clone(), path.as_str())
.await
.unwrap_or_default();
if node_link.is_valid() {
exists_in_state = true;
} else {
let absolute_path = path.to_absolute_path(repository.require_path()?);
exists_in_filesystem = std::fs::exists(absolute_path).unwrap_or_default();
}

if !exists_in_state && !exists_in_filesystem {
emit_path_ignore(path.as_str()).await;
lore_trace!("Ignoring invalid path: {path}");
}

exists_in_state || exists_in_filesystem
} else {
true
};

if exists {
lore_spawn!(tasks, {
async move {
if let Some(path) = path.as_ref() {
lore_debug!(
"Calculating deltas against filesystem path: {}",
path.as_str()
);
} else {
lore_debug!(
"Calculating deltas against filesystem for full repository"
);
}

let start = Instant::now();

// Scan uses staged state as diff base with scan_dirty=true.
// Content hashes in staged state are either zero (add nodes)
// or equal to current revision hashes, so the comparison is
// effectively filesystem vs committed content.
// The current revision is passed as the second pair so the
// walk can distinguish "node exists in staged but not in
// committed" — i.e. unstaged adds — from regular tracked
// files. Dirty flags are set/cleared inline during the walk.
let (changes, _stats) = state::diff_filesystem_ex(
repository.clone(),
state_staged.clone(),
repository.clone(),
state_current.clone(),
path,
FilterMode::Full,
true, // scan_dirty
layer_mounts.clone(),
)
.await
.forward::<StatusError>("computing diff against filesystem")?;

lore_debug!(
"Scan found {} file system changes in {:.3}s",
changes.len(),
start.elapsed().as_secs_f64(),
);

for change in changes.iter() {
let size =
file_size_from_node_change_path(repository.require_path()?, change)
.await?;

// Emit event for display (dirty set/clear handled inline by diff)
if !change.flags.is_stage() {
summary.classify(change);
event::LoreEvent::RepositoryStatusFile(
LoreRepositoryStatusFileEventData::from_node_change(
change, size,
),
)
.send();
} else {
lore_debug!("Ignore staged file {}", change.path);
}
}

Ok(())
}
});
}

lore_drain_tasks!(tasks, StatusError::internal("Recursion task failed"))?;
}
}

// Emit the aggregate dirty-node summary for reconciling status runs. For
// --scan these are the changes detected against the filesystem; for
// --check-dirty they are the nodes that stayed dirty after verification.
Expand Down
58 changes: 53 additions & 5 deletions lore-revision/src/state.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4722,6 +4722,30 @@ async fn apply_pending_discards(
}

let initial_ancestor = discard_node.parent;

// For a directory, discard the whole subtree below it first so its node
// slots are reclaimed; the node itself is unlinked from its parent and
// discarded by node_discard_patch below. Each child's sibling pointer is
// captured before discarding it, since discard_node repurposes that
// pointer for the block's free list.
if discard_node.is_directory() {
let mut child_ref = discard_node.child();
while let Some(child_id) = child_ref {
let child_node = state.node(repository.clone(), child_id).await?;
let next_sibling = child_node.sibling();
node_discard_recurse(
state.clone(),
repository.clone(),
child_id,
true, /* recurse */
true, /* discard */
|_, _| {},
)
.await?;
child_ref = next_sibling;
}
}

node_discard_patch(
state.clone(),
repository.clone(),
Expand Down Expand Up @@ -5644,6 +5668,14 @@ async fn emit_filesystem_subtree_deletes(
Ok(false)
}

/// Match each filesystem item from `file_receiver` against `node_list` (the
/// `from` state's children) and `current_node_list` (the `current` state's
/// children), emitting changes into `changes`, marking matched entries in
/// `node_list_found`, spawning subtree-recursion tasks into `tasks`, and
/// queueing stale directory nodes into `pending_discards`. Items with no
/// match in `node_list` are buffered and processed as new adds once the
/// receiver is drained. `node_list` and `current_node_list` must be sorted by
/// name; the binary searches here rely on that ordering.
#[allow(clippy::too_many_arguments)]
async fn diff_filesystem_directory_walk(
ctx: &DiffFilesystemContext,
Expand Down Expand Up @@ -5975,6 +6007,25 @@ async fn diff_filesystem_directory_walk(
continue;
};

// Directory staged then removed from disk before any commit: discard it
// rather than emit a Delete, since nothing committed backs it.
if ctx.scan_dirty && from_node.node.is_directory() {
let in_current = current_node_list
.children
.as_slice()
.binary_search_by(|child| child.name.cmp(&from_named_node.name))
.is_ok();
if !in_current {
lore_trace!(
"Queueing reverted uncommitted directory node {} (no entry at {}, not in current)",
from_named_node.node,
from_node.path
);
pending_discards.push(from_named_node.node);
continue;
}
}

// Emit deletes only for the materialized portion of the subtree,
// suppressing directories the filter merely descended through but never
// wrote to disk (see emit_filesystem_subtree_deletes).
Expand All @@ -5995,11 +6046,8 @@ async fn diff_filesystem_directory_walk(
continue;
}

// A leaf node present in state_from but not in state_current, with
// no file on disk, is an unstaged add that the user reverted by
// removing the file. Discard the node so state_staged matches the
// filesystem rather than emitting a Delete change for a node that
// shouldn't exist.
// Leaf staged but absent from both the commit and disk: a reverted
// unstaged add. Discard it rather than emit a Delete.
let in_current = current_node_list
.children
.as_slice()
Expand Down
Loading