Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGES/pulp_file/+sync-optimization.feature
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Added sync optimization that skips re-syncing when the remote manifest has not changed. An `optimize` flag on the sync endpoint (default `True`) allows forcing a full sync when needed.
20 changes: 19 additions & 1 deletion pulp_file/app/serializers.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
PublicationSerializer,
RemoteSerializer,
RepositorySerializer,
RepositorySyncURLSerializer,
SingleArtifactContentUploadSerializer,
)
from pulpcore.plugin.util import get_domain_pk
Expand Down Expand Up @@ -139,11 +140,28 @@ class FileRepositorySerializer(RepositorySerializer):
allow_null=True,
)

last_sync_details = serializers.JSONField(
help_text=_("Details about the last sync of this repository."),
read_only=True,
)

class Meta:
fields = RepositorySerializer.Meta.fields + ("autopublish", "manifest")
fields = RepositorySerializer.Meta.fields + ("autopublish", "manifest", "last_sync_details")
model = FileRepository


class FileRepositorySyncURLSerializer(RepositorySyncURLSerializer):
"""
Serializer for File Repository Sync URL.
"""

optimize = serializers.BooleanField(
help_text=_("Whether or not to optimize sync."),
required=False,
default=True,
)


class FileRemoteSerializer(RemoteSerializer):
"""
Serializer for File Remotes.
Expand Down
8 changes: 5 additions & 3 deletions pulp_file/app/viewsets.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@
)
from pulpcore.plugin.serializers import (
AsyncOperationResponseSerializer,
RepositorySyncURLSerializer,
TaskGroupOperationResponseSerializer,
)
from pulpcore.plugin.tasking import dispatch
Expand Down Expand Up @@ -55,6 +54,7 @@
FilePublicationSerializer,
FileRemoteSerializer,
FileRepositorySerializer,
FileRepositorySyncURLSerializer,
)


Expand Down Expand Up @@ -257,14 +257,14 @@ class FileRepositoryViewSet(RepositoryViewSet, ModifyRepositoryActionMixin, Role
summary="Sync from a remote",
responses={202: AsyncOperationResponseSerializer},
)
@action(detail=True, methods=["post"], serializer_class=RepositorySyncURLSerializer)
@action(detail=True, methods=["post"], serializer_class=FileRepositorySyncURLSerializer)
def sync(self, request, pk):
"""
Synchronizes a repository.

The ``repository`` field has to be provided.
"""
serializer = RepositorySyncURLSerializer(
serializer = FileRepositorySyncURLSerializer(
data=request.data, context={"request": request, "repository_pk": pk}
)
serializer.is_valid(raise_exception=True)
Expand All @@ -273,6 +273,7 @@ def sync(self, request, pk):
remote = serializer.validated_data.get("remote", repository.remote)

mirror = serializer.validated_data.get("mirror", False)
optimize = serializer.validated_data.get("optimize", True) # noqa
if mirror and repository.autopublish:
raise ValidationError("Cannot use mirror mode with autopublished repository.")
result = dispatch(
Expand All @@ -283,6 +284,7 @@ def sync(self, request, pk):
"remote_pk": str(remote.pk),
"repository_pk": str(repository.pk),
"mirror": mirror,
"optimize": optimize,
},
)
return OperationPostponedResponse(result, request)
Expand Down
8 changes: 4 additions & 4 deletions pulp_file/tests/functional/api/test_acs.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@

import pytest

from pulpcore.client.pulp_file import RepositorySyncURL
from pulpcore.client.pulp_file import FileRepositorySyncURL
from pulpcore.client.pulp_file.exceptions import ApiException
from pulpcore.tests.functional.utils import (
download_file,
Expand Down Expand Up @@ -108,7 +108,7 @@ def test_acs_sync(
assert acs_server.requests_record[0].path == basic_manifest_path

# Sync the repository
repository_sync_data = RepositorySyncURL(remote=main_remote.pulp_href)
repository_sync_data = FileRepositorySyncURL(remote=main_remote.pulp_href)
monitor_task(
file_bindings.RepositoriesFileApi.sync(file_repo.pulp_href, repository_sync_data).task
)
Expand Down Expand Up @@ -167,7 +167,7 @@ def test_acs_sync_with_paths(
assert expected_request_paths == actual_requested_paths

# Sync the repository
repository_sync_data = RepositorySyncURL(remote=main_remote.pulp_href)
repository_sync_data = FileRepositorySyncURL(remote=main_remote.pulp_href)
monitor_task(
file_bindings.RepositoriesFileApi.sync(file_repo.pulp_href, repository_sync_data).task
)
Expand Down Expand Up @@ -231,7 +231,7 @@ def test_serving_acs_content(
)

# Sync the repository
repository_sync_data = RepositorySyncURL(remote=main_remote.pulp_href)
repository_sync_data = FileRepositorySyncURL(remote=main_remote.pulp_href)
monitor_task(
file_bindings.RepositoriesFileApi.sync(file_repo.pulp_href, repository_sync_data).task
)
Expand Down
6 changes: 3 additions & 3 deletions pulp_file/tests/functional/api/test_auto_publish.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
import pytest

from pulpcore.client.pulp_file import (
RepositorySyncURL,
FileRepositorySyncURL,
)
from pulpcore.tests.functional.utils import get_files_in_manifest

Expand Down Expand Up @@ -48,7 +48,7 @@ def test_auto_publish_and_distribution(
expected_files = get_files_in_manifest(remote.url)

# Sync from the remote
body = RepositorySyncURL(remote=remote.pulp_href)
body = FileRepositorySyncURL(remote=remote.pulp_href)
monitor_task(file_bindings.RepositoriesFileApi.sync(repo.pulp_href, body).task)
repo = file_bindings.RepositoriesFileApi.read(repo.pulp_href)

Expand All @@ -73,7 +73,7 @@ def test_auto_publish_and_distribution(
assert files_in_first_publication == expected_files

# Assert that mirror=True is not allowed when autopublish=True
body = RepositorySyncURL(remote=remote.pulp_href, mirror=True)
body = FileRepositorySyncURL(remote=remote.pulp_href, mirror=True)
with pytest.raises(file_bindings.ApiException) as exc:
file_bindings.RepositoriesFileApi.sync(repo.pulp_href, body)
assert exc.value.status == 400
Expand Down
4 changes: 2 additions & 2 deletions pulp_file/tests/functional/api/test_bad_sync.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
import pytest
from aiohttp import web

from pulpcore.client.pulp_file import RepositorySyncURL
from pulpcore.client.pulp_file import FileRepositorySyncURL


@pytest.fixture
Expand All @@ -23,7 +23,7 @@ def _perform_sync(url, policy="immediate"):
}
remote = gen_object_with_cleanup(file_bindings.RemotesFileApi, remote_data)

body = RepositorySyncURL(remote=remote.pulp_href)
body = FileRepositorySyncURL(remote=remote.pulp_href)
monitor_task(file_bindings.RepositoriesFileApi.sync(file_repo.pulp_href, body).task)
return file_repo

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,11 @@
import requests
from django.conf import settings

from pulpcore.client.pulp_file import FileFileDistribution, FileFilePublication, RepositorySyncURL
from pulpcore.client.pulp_file import (
FileFileDistribution,
FileFilePublication,
FileRepositorySyncURL,
)


@dataclass
Expand Down Expand Up @@ -82,7 +86,7 @@ def ctx(
# Sync to get files into the repo
monitor_task(
file_bindings.RepositoriesFileApi.sync(
repo.pulp_href, RepositorySyncURL(remote=remote.pulp_href)
repo.pulp_href, FileRepositorySyncURL(remote=remote.pulp_href)
).task
)
repo = file_bindings.RepositoriesFileApi.read(repo.pulp_href)
Expand Down
4 changes: 2 additions & 2 deletions pulp_file/tests/functional/api/test_download_policies.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
from aiohttp.client_exceptions import ClientResponseError
from bs4 import BeautifulSoup

from pulpcore.client.pulp_file import FileFilePublication, RepositorySyncURL
from pulpcore.client.pulp_file import FileFilePublication, FileRepositorySyncURL
from pulpcore.tests.functional.utils import download_file, get_files_in_manifest

OBJECT_STORAGES = (
Expand Down Expand Up @@ -67,7 +67,7 @@ def test_download_policy(
expected_files = get_files_in_manifest(remote.url)

# Sync from the remote and assert that a new repository version is created
body = RepositorySyncURL(remote=remote.pulp_href)
body = FileRepositorySyncURL(remote=remote.pulp_href)
monitor_task(file_bindings.RepositoriesFileApi.sync(file_repo.pulp_href, body).task)
file_repo = file_bindings.RepositoriesFileApi.read(file_repo.pulp_href)
assert file_repo.latest_version_href.endswith("/versions/1/")
Expand Down
12 changes: 6 additions & 6 deletions pulp_file/tests/functional/api/test_git_sync.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

import pytest

from pulpcore.client.pulp_file import RepositorySyncURL
from pulpcore.client.pulp_file import FileRepositorySyncURL
from pulpcore.tests.functional.utils import PulpTaskError

GIT_REMOTE_URL = "https://github.com/pulp/pulp-smash.git"
Expand Down Expand Up @@ -65,7 +65,7 @@ def test_git_sync(file_bindings, file_repo, file_git_remote_factory, monitor_tas
"""Test syncing from a public Git repository."""
remote = file_git_remote_factory(url=GIT_REMOTE_URL, git_ref=git_ref)

body = RepositorySyncURL(remote=remote.pulp_href)
body = FileRepositorySyncURL(remote=remote.pulp_href)
monitor_task(file_bindings.RepositoriesFileApi.sync(file_repo.pulp_href, body).task)

file_repo = file_bindings.RepositoriesFileApi.read(file_repo.pulp_href)
Expand All @@ -81,7 +81,7 @@ def test_git_sync_idempotent(file_bindings, file_repo, file_git_remote_factory,
"""Syncing the same Git ref twice should not create a new repository version."""
remote = file_git_remote_factory(url=GIT_REMOTE_URL, git_ref="main")

body = RepositorySyncURL(remote=remote.pulp_href)
body = FileRepositorySyncURL(remote=remote.pulp_href)
monitor_task(file_bindings.RepositoriesFileApi.sync(file_repo.pulp_href, body).task)

file_repo = file_bindings.RepositoriesFileApi.read(file_repo.pulp_href)
Expand All @@ -104,7 +104,7 @@ def test_git_sync_invalid_url(file_bindings, file_repo, file_git_remote_factory,
"""Syncing with an invalid Git URL should raise a task error."""
remote = file_git_remote_factory(url="https://invalid.example.com/no-such-repo.git")

body = RepositorySyncURL(remote=remote.pulp_href)
body = FileRepositorySyncURL(remote=remote.pulp_href)
with pytest.raises(PulpTaskError) as exc:
monitor_task(file_bindings.RepositoriesFileApi.sync(file_repo.pulp_href, body).task)
assert "Failed to clone git repository" in exc.value.task.error["description"]
Expand All @@ -115,7 +115,7 @@ def test_git_sync_invalid_ref(file_bindings, file_repo, file_git_remote_factory,
"""Syncing with a non-existent git ref should raise a task error."""
remote = file_git_remote_factory(url=GIT_REMOTE_URL, git_ref="this-ref-does-not-exist-abc123")

body = RepositorySyncURL(remote=remote.pulp_href)
body = FileRepositorySyncURL(remote=remote.pulp_href)
with pytest.raises(PulpTaskError) as exc:
monitor_task(file_bindings.RepositoriesFileApi.sync(file_repo.pulp_href, body).task)
error_desc = exc.value.task.error["description"]
Expand All @@ -135,7 +135,7 @@ def test_git_sync_lfs(
remote = file_git_remote_factory(url=LFS_REMOTE_URL)
file_repo = file_repo_with_auto_publish

body = RepositorySyncURL(remote=remote.pulp_href)
body = FileRepositorySyncURL(remote=remote.pulp_href)
monitor_task(file_bindings.RepositoriesFileApi.sync(file_repo.pulp_href, body).task)

file_repo = file_bindings.RepositoriesFileApi.read(file_repo.pulp_href)
Expand Down
4 changes: 2 additions & 2 deletions pulp_file/tests/functional/api/test_publish.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@

from pulpcore.client.pulp_file import (
FileFilePublication,
RepositorySyncURL,
FileRepositorySyncURL,
)
from pulpcore.client.pulp_file.exceptions import ApiException
from pulpcore.tests.functional.utils import download_file
Expand All @@ -29,7 +29,7 @@ def test_crd_publications(

# Sync from the remote
initial_repo_version = file_repo.latest_version_href
body = RepositorySyncURL(remote=remote.pulp_href)
body = FileRepositorySyncURL(remote=remote.pulp_href)
monitor_task(file_bindings.RepositoriesFileApi.sync(file_repo.pulp_href, body).task)
first_repo_version_href = file_bindings.RepositoriesFileApi.read(
file_repo.pulp_href
Expand Down
4 changes: 2 additions & 2 deletions pulp_file/tests/functional/api/test_remote_settings.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
import pytest

from pulpcore.client.pulp_file import (
RepositorySyncURL,
FileRepositorySyncURL,
)
from pulpcore.client.pulp_file.exceptions import BadRequestException

Expand Down Expand Up @@ -88,7 +88,7 @@


def _run_basic_sync_and_assert(file_bindings, remote, file_repo, monitor_task):
body = RepositorySyncURL(remote=remote.pulp_href)
body = FileRepositorySyncURL(remote=remote.pulp_href)
monitor_task(file_bindings.RepositoriesFileApi.sync(file_repo.pulp_href, body).task)

# Check content is present, but no artifacts are there
Expand Down
16 changes: 8 additions & 8 deletions pulp_file/tests/functional/api/test_sync.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@

import pytest

from pulpcore.client.pulp_file import RepositorySyncURL
from pulpcore.client.pulp_file import FileRepositorySyncURL
from pulpcore.tests.functional.utils import PulpTaskError


Expand All @@ -29,7 +29,7 @@ def test_sync_file_protocol_handler(
remote = gen_object_with_cleanup(file_bindings.RemotesFileApi, remote_kwargs)
files = set(os.listdir(tmp_path / "file"))

body = RepositorySyncURL(remote=remote.pulp_href)
body = FileRepositorySyncURL(remote=remote.pulp_href)
monitor_task(file_bindings.RepositoriesFileApi.sync(file_repo.pulp_href, body).task)

# test that all the files are still present
Expand All @@ -54,7 +54,7 @@ def test_mirrored_sync(
"""Assert that syncing the repository w/ mirror=True creates a publication."""
remote = file_remote_ssl_factory(manifest_path=basic_manifest_path, policy="on_demand")

repository_sync_data = RepositorySyncURL(remote=remote.pulp_href, mirror=True)
repository_sync_data = FileRepositorySyncURL(remote=remote.pulp_href, mirror=True)
sync_response = file_bindings.RepositoriesFileApi.sync(
file_repo.pulp_href, repository_sync_data
)
Expand All @@ -76,7 +76,7 @@ def test_invalid_url(file_bindings, file_repo, gen_object_with_cleanup, monitor_
}
remote = gen_object_with_cleanup(file_bindings.RemotesFileApi, remote_kwargs)

body = RepositorySyncURL(remote=remote.pulp_href)
body = FileRepositorySyncURL(remote=remote.pulp_href)
with pytest.raises(PulpTaskError):
monitor_task(file_bindings.RepositoriesFileApi.sync(file_repo.pulp_href, body).task)

Expand All @@ -87,7 +87,7 @@ def test_invalid_file(
):
"""Sync a repository using an invalid file repository."""
remote = file_remote_factory(manifest_path=invalid_manifest_path, policy="immediate")
body = RepositorySyncURL(remote=remote.pulp_href)
body = FileRepositorySyncURL(remote=remote.pulp_href)
with pytest.raises(PulpTaskError):
monitor_task(file_bindings.RepositoriesFileApi.sync(file_repo.pulp_href, body).task)

Expand All @@ -103,7 +103,7 @@ def test_duplicate_file_sync(
remote = file_remote_factory(manifest_path=duplicate_filename_paths[0], policy="on_demand")
remote2 = file_remote_factory(manifest_path=duplicate_filename_paths[1], policy="on_demand")

body = RepositorySyncURL(remote=remote.pulp_href)
body = FileRepositorySyncURL(remote=remote.pulp_href)
monitor_task(file_bindings.RepositoriesFileApi.sync(file_repo.pulp_href, body).task)
file_repo = file_bindings.RepositoriesFileApi.read(file_repo.pulp_href)

Expand All @@ -112,7 +112,7 @@ def test_duplicate_file_sync(
assert version.content_summary.added["file.file"]["count"] == 3
assert file_repo.latest_version_href.endswith("/1/")

body = RepositorySyncURL(remote=remote2.pulp_href)
body = FileRepositorySyncURL(remote=remote2.pulp_href)
monitor_task(file_bindings.RepositoriesFileApi.sync(file_repo.pulp_href, body).task)
file_repo = file_bindings.RepositoriesFileApi.read(file_repo.pulp_href)

Expand All @@ -133,7 +133,7 @@ def test_filepath_includes_commas(
"""Sync a repository using a manifest file with a file whose relative_path includes commas"""
remote = file_remote_factory(manifest_path=manifest_path_with_commas, policy="on_demand")

body = RepositorySyncURL(remote=remote.pulp_href)
body = FileRepositorySyncURL(remote=remote.pulp_href)
monitor_task(file_bindings.RepositoriesFileApi.sync(file_repo.pulp_href, body).task)
file_repo = file_bindings.RepositoriesFileApi.read(file_repo.pulp_href)

Expand Down
Loading
Loading