Bug fixed for ceph RGW diff fix

pkg/objstore/s3_common.go

@@ -264,34 +264,19 @@ func (s *s3CommonClient) ObjectInfo(ctx context.Context, bucket string, name str
 
 // https://github.com/minio/minio/issues/17356
 // While working with lists, minio will output object keys as a/b/c, i.e, without leading slash.
-// It will also consider a/b as a valid value for prefix search parameter.
-// Prefix value /a/b will be considered invalid, search will return no results.
-// At the same time, other implementations, e.g. Ceph, will act the other way.
-// Meaning object keys will be listed as /a/b/c, with leading slash.
-// List operation with prefix /a/b will give results, with prefix a/b won't.
-// Therefore, in order to allow interoperability between providers, we should brind prefix parameter
-// and object keys in list to the expected format.
+// Ceph RGW (as deployed) also lists keys without leading slash and does not accept
+// leading slash in HeadObject/GetObject calls.
+// Therefore, we normalize by stripping any leading slash from both input and output names.
 func (s *s3CommonClient) normalizeInputName(name string) string {
 	if name == "" {
 		return name
 	}
-	isMinio := s.provider == s3.ProviderMinIO
-	hasLeadingSlash := strings.HasPrefix(name, "/")
-	if isMinio && hasLeadingSlash {
-		return strings.TrimPrefix(name, "/")
-	}
-	if !isMinio && !hasLeadingSlash {
-		return "/" + name
-	}
-	return name
+	return strings.TrimPrefix(name, "/")
 }
 
 func (s *s3CommonClient) normalizeOutputName(name string) string {
 	if name == "" {
 		return name
 	}
-	if !strings.HasPrefix(name, "/") {
-		return "/" + name
-	}
-	return name
+	return strings.TrimPrefix(name, "/")
 }

service/worker/handler/diff_handlers.go

@@ -23,6 +23,7 @@ import (
 
 	"github.com/hibiken/asynq"
 	"github.com/redis/go-redis/v9"
+	"github.com/rs/zerolog"
 
 	"github.com/clyso/chorus/pkg/dom"
 	"github.com/clyso/chorus/pkg/entity"
@@ -803,6 +804,7 @@ func (r *DiffSvc) EnsureObjectsDeleted(ctx context.Context, id entity.DiffFixID,
 		return fmt.Errorf("unable to get objects to remove: %w", err)
 	}
 
+	zerolog.Ctx(ctx).Warn().Int("count", len(objectsToRemove)).Msg("EnsureObjectsDeleted: checking objects to remove")
 	client, err := r.clients.AsCommon(ctx, location.Storage, user)
 	if err != nil {
 		return fmt.Errorf("unable to obtain client: %w", err)
@@ -833,6 +835,7 @@ func (r *DiffSvc) EnsureObjectsDeleted(ctx context.Context, id entity.DiffFixID,
 		return fmt.Errorf("unable to get objects to copy: %w", err)
 	}
 
+	zerolog.Ctx(ctx).Warn().Int("count", len(objectsToCopy)).Msg("EnsureObjectsDeleted: enqueuing copy tasks")
 	for _, object := range objectsToCopy {
 		payload, err := makePayload(object.Name, object.IsDir)
 		if err != nil {
@@ -847,5 +850,9 @@ func (r *DiffSvc) EnsureObjectsDeleted(ctx context.Context, id entity.DiffFixID,
 		return fmt.Errorf("unable to clean fix set store: %w", err)
 	}
 
+	if _, err := r.fixCopySetStore.Drop(ctx, id); err != nil {
+		return fmt.Errorf("unable to clean fix copy set store: %w", err)
+	}
+
 	return nil
 }

service/worker/handler/migration_obj_copy_handler.go

@@ -74,10 +74,14 @@ func (s *svc) HandleMigrationObjCopy(ctx context.Context, t *asynq.Task) (err er
 	}
 	fromVer, toVer := versions.From, versions.To
 
-	if fromVer != 0 && fromVer <= toVer {
+	isDiffFix := t.Type() == tasks.TypeDiffFixCopyS3
+	if !isDiffFix && fromVer != 0 && fromVer <= toVer {
 		logger.Info().Int("from_ver", fromVer).Int("to_ver", toVer).Msg("migration obj copy: identical from/to obj version: skip copy")
 		return nil
 	}
+	if isDiffFix {
+		logger.Warn().Int("from_ver", fromVer).Int("to_ver", toVer).Msg("diff fix copy: proceeding with copy despite version check")
+	}
 	// 1. sync obj meta and content
 	err = lock.Do(ctx, time.Second*2, func() error {
 		return s.copySvc.CopyObject(ctx, p.ID.User(), copy.File{
@@ -105,7 +109,11 @@ func (s *svc) HandleMigrationObjCopy(ctx context.Context, t *asynq.Task) (err er
 			return fmt.Errorf("migration obj copy: unable to update obj meta: %w", err)
 		}
 	}
-	logger.Info().Msg("migration obj copy: done")
+	if isDiffFix {
+		logger.Warn().Msg("diff fix copy: object copied successfully")
+	} else {
+		logger.Info().Msg("migration obj copy: done")
+	}
 
 	return nil
 }