diff --git a/.github/workflows/build-core-services.yml b/.github/workflows/build-core-services.yml index c3389672797..fed807ecf8b 100644 --- a/.github/workflows/build-core-services.yml +++ b/.github/workflows/build-core-services.yml @@ -1,110 +1,294 @@ -name: Build Core Services +name: Publish Generated Service Images on: - push: - branches: [main] - paths: - - "packages/phlo-observatory/**" - - "packages/phlo-api/**" - - ".github/workflows/build-core-services.yml" - pull_request: - branches: [main] - paths: - - "packages/phlo-observatory/**" - - "packages/phlo-api/**" + workflow_dispatch: + inputs: + services: + description: Optional comma-separated generated services to republish + required: false + type: string release: types: [published] -env: - REGISTRY: ghcr.io +concurrency: + group: generated-service-images-${{ github.ref }} + cancel-in-progress: false jobs: - build-observatory: - name: Build Observatory + prepare: + name: Prepare generated build contexts runs-on: ubuntu-latest permissions: contents: read - packages: write + outputs: + targets: ${{ steps.matrix.outputs.targets }} + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd + with: + persist-credentials: false + + - name: Install uv + uses: astral-sh/setup-uv@5a095e7a2014a4212f075830d4f7277575a9d098 + with: + version: "0.10.10" + enable-cache: false + + - name: Install workspace service plugins + run: | + uv python install 3.11 + uv sync --python 3.11 --all-packages + + - name: Render every installed service through the public CLI + id: matrix + env: + PUBLISH_SERVICES: ${{ inputs.services || '' }} + shell: bash + run: | + set -euo pipefail + project="$RUNNER_TEMP/generated-service-project" + mkdir -p "$project" + cd "$project" + uv --project "$GITHUB_WORKSPACE" run phlo services init --no-dev --force + mapfile -t services < <( + uv --project "$GITHUB_WORKSPACE" run phlo services list --all --json | + jq -r '.[].name' + ) + add_args=() + for service in "${services[@]}"; do + add_args+=(--service "$service") + done + uv --project "$GITHUB_WORKSPACE" run phlo services add "${add_args[@]}" --no-start + docker compose \ + --profile '*' \ + -f "$project/.phlo/docker-compose.yml" \ + --project-directory "$project/.phlo" \ + config --format json > "$RUNNER_TEMP/generated-compose.json" + matrix_args=() + if [[ -n "$PUBLISH_SERVICES" ]]; then + matrix_args+=(--services "$PUBLISH_SERVICES") + fi + matrix="$( + uv --project "$GITHUB_WORKSPACE" run python \ + "$GITHUB_WORKSPACE/scripts/generated_image_matrix.py" \ + "$RUNNER_TEMP/generated-compose.json" \ + "$project" \ + "$GITHUB_WORKSPACE" \ + "${matrix_args[@]}" + )" + echo "targets=$(jq -c '.include' <<< "$matrix")" >> "$GITHUB_OUTPUT" + + - name: Upload generated build contexts + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + name: generated-service-build-contexts + path: ${{ runner.temp }}/generated-service-project + include-hidden-files: true + if-no-files-found: error + build: + name: Build ${{ matrix.target.service }} (${{ matrix.architecture.name }}) + needs: prepare + runs-on: ${{ matrix.architecture.runner }} + timeout-minutes: 45 + permissions: + contents: read + packages: write + strategy: + fail-fast: false + max-parallel: 8 + matrix: + target: ${{ fromJSON(needs.prepare.outputs.targets) }} + architecture: + - name: amd64 + platform: linux/amd64 + runner: ubuntu-24.04 + - name: arm64 + platform: linux/arm64 + runner: ubuntu-24.04-arm steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd with: + path: source persist-credentials: false + - name: Download generated build contexts + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 + with: + name: generated-service-build-contexts + path: generated + - name: Set up Docker Buildx uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f - - name: Log in to Container Registry - if: github.event_name != 'pull_request' + - name: Log in to GitHub Container Registry uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 with: - registry: ${{ env.REGISTRY }} + registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Extract metadata - id: meta - uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 - with: - images: ${{ env.REGISTRY }}/${{ github.repository_owner }}/phlo-observatory - tags: | - type=ref,event=branch - type=ref,event=pr - type=semver,pattern={{version}} - type=semver,pattern={{major}}.{{minor}} - type=raw,value=latest,enable={{is_default_branch}} - - - name: Build and push + - name: Render build arguments + id: args + env: + BUILD_ARGS_JSON: ${{ toJSON(matrix.target.build_args) }} + shell: bash + run: | + { + echo 'value<> "$GITHUB_OUTPUT" + + - name: Resolve digest repository + id: image + env: + IMAGE: ${{ matrix.target.image }} + shell: bash + run: | + set -euo pipefail + repository="${IMAGE%@*}" + repository="${repository%:*}" + echo "repository=$repository" >> "$GITHUB_OUTPUT" + + - name: Build and publish native architecture image + id: build uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 with: - context: packages/phlo-observatory/src/phlo_observatory - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - cache-from: type=gha - cache-to: type=gha,mode=max - - build-phlo-api: - name: Build phlo-api - runs-on: ubuntu-latest + context: ${{ matrix.target.root }}/${{ matrix.target.context }} + file: ${{ matrix.target.root }}/${{ matrix.target.dockerfile }} + build-args: ${{ steps.args.outputs.value }} + platforms: ${{ matrix.architecture.platform }} + outputs: type=image,name=${{ steps.image.outputs.repository }},push-by-digest=true,name-canonical=true,push=true + labels: | + org.opencontainers.image.source=https://github.com/${{ github.repository }} + org.opencontainers.image.revision=${{ github.sha }} + provenance: mode=max + sbom: true + cache-from: type=gha,scope=${{ matrix.target.service }}-${{ matrix.architecture.name }} + cache-to: type=gha,mode=max,scope=${{ matrix.target.service }}-${{ matrix.architecture.name }} + + - name: Record architecture digest + env: + DIGEST: ${{ steps.build.outputs.digest }} + shell: bash + run: | + set -euo pipefail + hex="${DIGEST#sha256:}" + [[ "$hex" =~ ^[0-9a-f]{64}$ ]] + mkdir -p digests + touch "digests/$hex" + + - name: Upload architecture digest + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + name: digest-${{ matrix.target.service }}-${{ matrix.architecture.name }} + path: digests/* + if-no-files-found: error + + merge: + name: Publish ${{ matrix.target.service }} manifest + needs: [prepare, build] + if: always() && needs.prepare.result == 'success' + runs-on: ubuntu-24.04 + timeout-minutes: 15 permissions: contents: read packages: write - + strategy: + fail-fast: false + matrix: + target: ${{ fromJSON(needs.prepare.outputs.targets) }} steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd + - name: Download amd64 digest + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 with: - persist-credentials: false + name: digest-${{ matrix.target.service }}-amd64 + path: digests + + - name: Download arm64 digest + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 + with: + name: digest-${{ matrix.target.service }}-arm64 + path: digests - name: Set up Docker Buildx uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f - - name: Log in to Container Registry - if: github.event_name != 'pull_request' + - name: Log in to GitHub Container Registry uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 with: - registry: ${{ env.REGISTRY }} + registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Extract metadata - id: meta - uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 + - name: Publish multi-architecture manifest + id: manifest + env: + IMAGE: ${{ matrix.target.image }} + shell: bash + run: | + set -euo pipefail + refs=() + for digest_file in digests/*; do + refs+=("$IMAGE@sha256:${digest_file##*/}") + done + if [[ "${#refs[@]}" -ne 2 ]]; then + echo "expected two architecture digests, found ${#refs[@]}" >&2 + exit 1 + fi + docker buildx imagetools create --tag "$IMAGE" "${refs[@]}" + digest="$( + docker buildx imagetools inspect \ + --format '{{json .Manifest.Digest}}' \ + "$IMAGE" | jq -r . + )" + [[ "$digest" =~ ^sha256:[0-9a-f]{64}$ ]] + echo "digest=$digest" >> "$GITHUB_OUTPUT" + + - name: Record published digest + env: + DIGEST: ${{ steps.manifest.outputs.digest }} + IMAGE: ${{ matrix.target.image }} + SERVICE: ${{ matrix.target.service }} + SERVICES: ${{ toJSON(matrix.target.services) }} + shell: bash + run: | + jq -n \ + --arg image "$IMAGE" \ + --arg digest "$DIGEST" \ + --argjson services "$SERVICES" \ + '{image: $image, digest: $digest, services: $services}' \ + > "published-${SERVICE}.json" + + - name: Upload published digest + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f with: - images: ${{ env.REGISTRY }}/${{ github.repository_owner }}/phlo-api - tags: | - type=ref,event=branch - type=ref,event=pr - type=semver,pattern={{version}} - type=semver,pattern={{major}}.{{minor}} - type=raw,value=latest,enable={{is_default_branch}} - - - name: Build and push - uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 + name: published-${{ matrix.target.service }} + path: published-${{ matrix.target.service }}.json + if-no-files-found: error + + manifest: + name: Published image digest manifest + needs: merge + if: always() && needs.merge.result == 'success' + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Download published digests + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 + with: + pattern: published-* + path: published + merge-multiple: true + + - name: Assemble digest manifest + shell: bash + run: jq -s 'sort_by(.image)' published/*.json > generated-service-images.json + + - name: Upload digest manifest + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f with: - context: packages/phlo-api/src/phlo_api - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - cache-from: type=gha - cache-to: type=gha,mode=max + name: generated-service-images + path: generated-service-images.json + if-no-files-found: error diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 41d5996f704..99118bf36c3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -295,6 +295,119 @@ jobs: exit 1 fi + generated-container-checks: + name: containers / generated service files + runs-on: ubuntu-latest + timeout-minutes: 120 + permissions: + contents: read + packages: read + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd + with: + persist-credentials: false + + - name: Install uv + uses: astral-sh/setup-uv@5a095e7a2014a4212f075830d4f7277575a9d098 + with: + version: "0.10.10" + + - name: Set up Python + run: uv python install 3.11 + + - name: Build Phlo and service wheels + run: uv build --all-packages --wheel --out-dir "$RUNNER_TEMP/phlo-container-wheelhouse" + + - name: Hydrate the container wheelhouse + shell: bash + run: | + set -euo pipefail + wheelhouse="$RUNNER_TEMP/phlo-container-wheelhouse" + distributions=( + phlo phlo-alloy phlo-api phlo-clickhouse phlo-clickstack phlo-dagster + phlo-grafana phlo-hasura phlo-loki phlo-minio phlo-nessie phlo-oauth2-proxy + phlo-openmetadata phlo-observatory phlo-pgweb phlo-postgres phlo-postgrest + phlo-prometheus phlo-rustfs phlo-superset phlo-traefik phlo-trino + ) + local_wheels=() + shopt -s nullglob + for distribution in "${distributions[@]}"; do + normalized="${distribution//-/_}" + matches=("$wheelhouse"/"$normalized"-*.whl) + if [ "${#matches[@]}" -ne 1 ]; then + echo "Expected one wheel for $distribution, found ${#matches[@]}" >&2 + exit 1 + fi + local_wheels+=("${matches[0]}") + done + uvx --from pip==26.1.2 pip download \ + --dest "$wheelhouse" \ + --find-links "$wheelhouse" \ + "${local_wheels[@]}" + + - name: Cache Trivy vulnerability and scan data + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 + with: + path: ${{ runner.temp }}/phlo-trivy-cache + key: trivy-${{ runner.os }}-v2-${{ github.run_number }} + restore-keys: | + trivy-${{ runner.os }}-v2- + + - name: Install wheels into a clean environment + shell: bash + run: | + set -euo pipefail + env_dir="$RUNNER_TEMP/phlo-container-env" + uv venv "$env_dir" --python 3.11 + uv pip install \ + --python "$env_dir/bin/python" \ + --no-index \ + --find-links "$RUNNER_TEMP/phlo-container-wheelhouse" \ + phlo \ + phlo-alloy \ + phlo-api \ + phlo-clickhouse \ + phlo-clickstack \ + phlo-dagster \ + phlo-grafana \ + phlo-hasura \ + phlo-loki \ + phlo-minio \ + phlo-nessie \ + phlo-oauth2-proxy \ + phlo-openmetadata \ + phlo-observatory \ + phlo-pgweb \ + phlo-postgres \ + phlo-postgrest \ + phlo-prometheus \ + phlo-rustfs \ + phlo-superset \ + phlo-traefik \ + phlo-trino + echo "$env_dir/bin" >> "$GITHUB_PATH" + + - name: Log in to GitHub Container Registry + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Check generated service containers + env: + PHLO_TRIVY_CACHE_DIR: ${{ runner.temp }}/phlo-trivy-cache + run: | + docker pull aquasec/trivy@sha256:cffe3f5161a47a6823fbd23d985795b3ed72a4c806da4c4df16266c02accdd6f + phlo --no-color plugin check --containers --remote-images \ + --allow-vulnerable-image "alloy=ghcr.io/phlohouse/phlo-alloy:v1.18.0-go1.26.5=6a7628c7ba66eb8a0fb70569a36b90012bdb9e0e5b0fa4241331d62179b0528b=Alloy 1.18.0 retains two Docker module advisories without fixes and one whose fix requires an incompatible Docker client major" \ + --allow-vulnerable-image "clickstack=ghcr.io/phlohouse/phlo-clickstack:2.31.0-security-patches=adc46e9eb99e4f3c0ea11a6cab55ebf0d720c844128a33b5bb8c7c7efae79224=HyperDX 2.31.0 bundles source-built OTel binaries with grpc 1.81.1 and Go 1.26.4; replacing them requires rebuilding the upstream OCB distribution, while its legacy MongoDB OpenSSL, root orchestration, and Next findings remain" \ + --allow-vulnerable-image "grafana=ghcr.io/phlohouse/phlo-grafana:13.1.1-go1.26.5=776f3f74541a97642ef470c93c4e04aed1325f81547e268564c87368f8fce669=Trivy reports a stale Tempo pseudo-version although the image rebuilds exact Tempo 2.10.7 source containing both fixes" \ + --allow-vulnerable-image "minio=ghcr.io/phlohouse/phlo-minio:7aac2a2c5b7c=b8d9b4a71a6e9c05e3ba3931ada391b3dce6b46404d303ee373ec46acf994163=Archived public source still has six MinIO-module advisories without public patches; OS, standard library, and third-party Go findings are remediated" \ + --allow-vulnerable-image "openmetadata-elasticsearch=ghcr.io/phlohouse/phlo-openmetadata-elasticsearch:8.11.4-java-patches=d2d0eaf6881362fd497f981d3f9f5e4b431fd072044decda8acf3612ef2d3f3e=Elasticsearch 8.11.4 shades three old Jackson versions, requires Tika 2.x, and has one unpublished and one unfixed LZ4 advisory" \ + --allow-vulnerable-image "superset=ghcr.io/phlohouse/phlo-superset:6.1.0-security-patches=064bf36c4a8366c2077b576e6d08b703a7d0fd114110f0e9243b5090fdaf051e=Superset requires three vulnerable Python versions and Debian 12 has no fixed packages for the remaining operating system advisories" \ + --allow-vulnerable-image "trino=ghcr.io/phlohouse/phlo-trino:483-launcher318-go1.26.5=3f18aa318a5cdca2cffe51a1f6768d3e8c3bc6feee880f61ee0f53e0c7e8add6=Trino 483 requires Jetty 11 while the remaining HTTP advisory is fixed only in incompatible Jetty 12" + release-golden-path: name: python / release golden path runs-on: ubuntu-latest @@ -496,6 +609,7 @@ jobs: - python-quality - python-core-tests - quickstart-smoke + - generated-container-checks - release-golden-path - recovery-continuity-drill - windows-release-contract @@ -509,6 +623,7 @@ jobs: PYTHON_QUALITY: ${{ needs.python-quality.result }} PYTHON_CORE_TESTS: ${{ needs.python-core-tests.result }} QUICKSTART_SMOKE: ${{ needs.quickstart-smoke.result }} + GENERATED_CONTAINER_CHECKS: ${{ needs.generated-container-checks.result }} RELEASE_GOLDEN_PATH: ${{ needs.release-golden-path.result }} RECOVERY_CONTINUITY_DRILL: ${{ needs.recovery-continuity-drill.result }} WINDOWS_RELEASE_CONTRACT: ${{ needs.windows-release-contract.result }} @@ -526,6 +641,7 @@ jobs: echo "| python / quality | ${PYTHON_QUALITY} |" echo "| python / core tests | ${PYTHON_CORE_TESTS} |" echo "| python / quickstart smoke | ${QUICKSTART_SMOKE} |" + echo "| containers / generated service files | ${GENERATED_CONTAINER_CHECKS} |" echo "| python / release golden path | ${RELEASE_GOLDEN_PATH} |" echo "| python / recovery continuity drill | ${RECOVERY_CONTINUITY_DRILL} |" echo "| windows / release golden path contract | ${WINDOWS_RELEASE_CONTRACT} |" @@ -538,6 +654,7 @@ jobs: "${PYTHON_QUALITY}" \ "${PYTHON_CORE_TESTS}" \ "${QUICKSTART_SMOKE}" \ + "${GENERATED_CONTAINER_CHECKS}" \ "${RELEASE_GOLDEN_PATH}" \ "${RECOVERY_CONTINUITY_DRILL}" \ "${WINDOWS_RELEASE_CONTRACT}" \ diff --git a/packages/phlo-alloy/pyproject.toml b/packages/phlo-alloy/pyproject.toml index 14d17929363..b2f6a67ce01 100644 --- a/packages/phlo-alloy/pyproject.toml +++ b/packages/phlo-alloy/pyproject.toml @@ -41,6 +41,7 @@ include-package-data = true [tool.setuptools.package-data] phlo_alloy = [ + "Dockerfile", "service.yaml", "config.alloy", ] diff --git a/packages/phlo-alloy/src/phlo_alloy/Dockerfile b/packages/phlo-alloy/src/phlo_alloy/Dockerfile new file mode 100644 index 00000000000..58f2c5794c0 --- /dev/null +++ b/packages/phlo-alloy/src/phlo_alloy/Dockerfile @@ -0,0 +1,26 @@ +FROM grafana/alloy-build-image:v0.1.34@sha256:4371598809230ffd7611367ba0dcb92bbc8b74bda88471293e5d478b48a0d186 AS build + +RUN git init /src/alloy && \ + git -C /src/alloy remote add origin https://github.com/grafana/alloy.git && \ + git -C /src/alloy fetch --depth 1 origin a435563ff073d5355952c1a8d1821110b1392691 && \ + git -C /src/alloy checkout --detach FETCH_HEAD && \ + rm -rf /src/alloy/.git +WORKDIR /src/alloy +# hadolint ignore=DL3062 +RUN export GOCACHE=/tmp/go-cache GOMODCACHE=/tmp/go-mod && \ + go get google.golang.org/grpc@v1.82.1 && \ + go mod tidy && \ + npm --prefix internal/web/ui ci --no-audit --no-fund && \ + npm --prefix internal/web/ui run build && \ + RELEASE_BUILD=1 \ + VERSION=v1.18.0 \ + GO_TAGS="netgo embedalloyui promtail_journal_enabled" \ + SKIP_UI_BUILD=1 \ + make alloy && \ + rm -rf internal/web/ui/node_modules /tmp/go-cache /tmp/go-mod + +FROM grafana/alloy:v1.18.0@sha256:491b0578c04983fd54fe99b587b6fab4404dc46d0dc16677bd6b00cc1140b308 + +COPY --from=build /src/alloy/build/alloy /bin/alloy + +USER "473" diff --git a/packages/phlo-alloy/src/phlo_alloy/service.yaml b/packages/phlo-alloy/src/phlo_alloy/service.yaml index e50b67a4c52..0ca0534561e 100644 --- a/packages/phlo-alloy/src/phlo_alloy/service.yaml +++ b/packages/phlo-alloy/src/phlo_alloy/service.yaml @@ -4,7 +4,10 @@ category: observability default: false profile: observability -image: grafana/alloy:v1.4.2 +image: ghcr.io/phlohouse/phlo-alloy:v1.18.0-go1.26.5 +build: + context: . + dockerfile: alloy/Dockerfile compose: restart: unless-stopped @@ -36,5 +39,7 @@ env_vars: description: Alloy HTTP port files: + - source: Dockerfile + dest: alloy/Dockerfile - source: config.alloy dest: alloy/config.alloy diff --git a/packages/phlo-alloy/tests/test_alloy_plugin.py b/packages/phlo-alloy/tests/test_alloy_plugin.py index c8a547978b0..dea30982a72 100644 --- a/packages/phlo-alloy/tests/test_alloy_plugin.py +++ b/packages/phlo-alloy/tests/test_alloy_plugin.py @@ -1,5 +1,7 @@ """Tests for Alloy service plugin.""" +from importlib import resources + from phlo_alloy.plugin import AlloyServicePlugin @@ -12,6 +14,31 @@ def test_alloy_service_definition(): assert defn["profile"] == "observability" +def test_alloy_service_builds_patched_release_image() -> None: + """Generated Alloy uses the stable release with fixed embedded Go dependencies.""" + definition = AlloyServicePlugin().service_definition + + assert definition["image"] == "ghcr.io/phlohouse/phlo-alloy:v1.18.0-go1.26.5" + assert definition["build"] == {"context": ".", "dockerfile": "alloy/Dockerfile"} + + +def test_alloy_runtime_image_sets_the_upstream_non_root_user() -> None: + """The generated Dockerfile keeps Alloy's upstream runtime identity explicit.""" + dockerfile = resources.files("phlo_alloy").joinpath("Dockerfile").read_text() + + assert dockerfile.rstrip().endswith('USER "473"') + + +def test_alloy_builder_discards_source_control_and_dependency_caches() -> None: + """The generated build must not retain multi-gigabyte transient dependency trees.""" + dockerfile = resources.files("phlo_alloy").joinpath("Dockerfile").read_text() + + assert "git init /src/alloy" in dockerfile + assert "git -C /src/alloy fetch --depth 1 origin" in dockerfile + assert "rm -rf /src/alloy/.git" in dockerfile + assert "rm -rf internal/web/ui/node_modules /tmp/go-cache /tmp/go-mod" in dockerfile + + def test_alloy_plugin_metadata(): """Validate Alloy plugin metadata.""" plugin = AlloyServicePlugin() diff --git a/packages/phlo-api/src/phlo_api/Dockerfile b/packages/phlo-api/src/phlo_api/Dockerfile index 4e66366d1fc..206be784b97 100644 --- a/packages/phlo-api/src/phlo_api/Dockerfile +++ b/packages/phlo-api/src/phlo_api/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.11-slim AS phlo-build-context +FROM python:3.11-alpine AS phlo-build-context WORKDIR /opt/phlo-build-context @@ -8,7 +8,7 @@ RUN set -eux; \ install -m 0755 "$entrypoint" /opt/phlo-build-context/phlo-api-entrypoint.sh; \ mkdir -p /opt/phlo-build-context/wheelhouse -FROM python:3.11-slim +FROM python:3.11-alpine WORKDIR /app @@ -16,7 +16,15 @@ ARG PHLO_VERSION= ARG PHLO_API_VERSION= ARG PHLO_WHEELHOUSE= -RUN pip install --no-cache-dir uv +RUN apk upgrade --no-cache \ + && apk add --no-cache ca-certificates=20260611-r0 \ + && apk add --no-cache --virtual .build-deps \ + gcc=15.2.0-r5 \ + musl-dev=1.2.6-r2 \ + && pip install --no-cache-dir \ + "setuptools==83.0.0" \ + "wheel==0.47.0" \ + "uv==0.8.13" COPY --from=phlo-build-context /opt/phlo-build-context/wheelhouse /opt/phlo-wheelhouse @@ -34,10 +42,19 @@ RUN set -eux; \ uv pip install --system --prerelease explicit "$PHLO_REQUIREMENT" "$PHLO_API_REQUIREMENT"; \ else \ uv pip install --system phlo "$PHLO_API_REQUIREMENT"; \ - fi + fi && apk del .build-deps + +# Do not retain uv's downloaded archives in the runtime image; they are not needed to run Phlo. +RUN rm -rf /root/.cache/uv COPY --from=phlo-build-context /opt/phlo-build-context/phlo-api-entrypoint.sh /usr/local/bin/phlo-api-entrypoint.sh +RUN addgroup -S phlo \ + && adduser -S -G phlo -h /app -H phlo \ + && chown -R phlo:phlo /app + +USER phlo + EXPOSE 4000 ENTRYPOINT ["/usr/local/bin/phlo-api-entrypoint.sh"] diff --git a/packages/phlo-api/src/phlo_api/service.yaml b/packages/phlo-api/src/phlo_api/service.yaml index 3baac5289b8..4da56f973c7 100644 --- a/packages/phlo-api/src/phlo_api/service.yaml +++ b/packages/phlo-api/src/phlo_api/service.yaml @@ -5,6 +5,7 @@ default: false profile: api phlo_dev: true +image: ghcr.io/phlohouse/phlo-api:0.7.0 build: context: . dockerfile: phlo-api/Dockerfile diff --git a/packages/phlo-clickhouse/README.md b/packages/phlo-clickhouse/README.md index 396cd773388..77de420bde2 100644 --- a/packages/phlo-clickhouse/README.md +++ b/packages/phlo-clickhouse/README.md @@ -41,7 +41,7 @@ The following environment variables can be used to configure ClickHouse: | Variable | Default | Description | |----------|---------|-------------| -| `CLICKHOUSE_VERSION` | `latest` | ClickHouse server version tag | +| `CLICKHOUSE_VERSION` | `26.5.6.64-alpine` | ClickHouse server version tag | | `CLICKHOUSE_HTTP_PORT` | `8123` | ClickHouse HTTP interface port | | `CLICKHOUSE_NATIVE_PORT` | `19000` | ClickHouse native protocol port | | `CLICKHOUSE_METRICS_PORT` | `9363` | ClickHouse Prometheus metrics port | diff --git a/packages/phlo-clickhouse/src/phlo_clickhouse/clickhouse-setup.yaml b/packages/phlo-clickhouse/src/phlo_clickhouse/clickhouse-setup.yaml index 7ffb7908601..39225b35b9c 100644 --- a/packages/phlo-clickhouse/src/phlo_clickhouse/clickhouse-setup.yaml +++ b/packages/phlo-clickhouse/src/phlo_clickhouse/clickhouse-setup.yaml @@ -4,7 +4,7 @@ description: Initialize ClickHouse databases for data plane category: data default: false -image: clickhouse/clickhouse-server:${CLICKHOUSE_VERSION:-latest} +image: clickhouse/clickhouse-server:${CLICKHOUSE_VERSION:-26.5.6.64-alpine} depends_on: - clickhouse diff --git a/packages/phlo-clickhouse/src/phlo_clickhouse/service.yaml b/packages/phlo-clickhouse/src/phlo_clickhouse/service.yaml index 21cbd98b7b0..7636ed1292b 100644 --- a/packages/phlo-clickhouse/src/phlo_clickhouse/service.yaml +++ b/packages/phlo-clickhouse/src/phlo_clickhouse/service.yaml @@ -3,7 +3,7 @@ description: ClickHouse analytical database for data plane category: data default: false -image: clickhouse/clickhouse-server:${CLICKHOUSE_VERSION:-latest} +image: clickhouse/clickhouse-server:${CLICKHOUSE_VERSION:-26.5.6.64-alpine} compose: restart: unless-stopped @@ -44,7 +44,7 @@ compose: env_vars: CLICKHOUSE_VERSION: - default: "latest" + default: "26.5.6.64-alpine" description: ClickHouse server version tag CLICKHOUSE_HTTP_PORT: default: 8123 diff --git a/packages/phlo-clickhouse/tests/test_clickhouse_plugin.py b/packages/phlo-clickhouse/tests/test_clickhouse_plugin.py index f07ee9c7ac8..d2823b7a2c5 100644 --- a/packages/phlo-clickhouse/tests/test_clickhouse_plugin.py +++ b/packages/phlo-clickhouse/tests/test_clickhouse_plugin.py @@ -19,6 +19,16 @@ def test_clickhouse_service_definition(): assert "clickhouse-logs:/var/log/clickhouse-server" in service_definition["compose"]["volumes"] +def test_clickhouse_service_pins_generated_image_version() -> None: + """The generated environment must not replace the pinned image tag with latest.""" + service_definition = ClickHouseServicePlugin().service_definition + + assert service_definition["image"] == ( + "clickhouse/clickhouse-server:${CLICKHOUSE_VERSION:-26.5.6.64-alpine}" + ) + assert service_definition["env_vars"]["CLICKHOUSE_VERSION"]["default"] == ("26.5.6.64-alpine") + + def test_clickhouse_service_metadata(): """Validate ClickHouse service plugin metadata.""" diff --git a/packages/phlo-clickstack/pyproject.toml b/packages/phlo-clickstack/pyproject.toml index 406f3b33b66..95d28a1e626 100644 --- a/packages/phlo-clickstack/pyproject.toml +++ b/packages/phlo-clickstack/pyproject.toml @@ -49,7 +49,7 @@ target-version = "py311" include-package-data = true [tool.setuptools.package-data] -phlo_clickstack = ["service.yaml"] +phlo_clickstack = ["Dockerfile", "service.yaml"] [tool.setuptools.package-dir] "" = "src" diff --git a/packages/phlo-clickstack/src/phlo_clickstack/Dockerfile b/packages/phlo-clickstack/src/phlo_clickstack/Dockerfile new file mode 100644 index 00000000000..86669e597a0 --- /dev/null +++ b/packages/phlo-clickstack/src/phlo_clickstack/Dockerfile @@ -0,0 +1,69 @@ +FROM node:22.22-alpine@sha256:e58326d0d441090181ac150dc2078d3e2cf6a0d42e809aebba3ef5880935ffdd AS node-patches + +RUN npm install --prefix /patches \ + axios@1.18.0 \ + brace-expansion@2.1.2 \ + fast-uri@3.1.4 \ + immutable@4.3.9 \ + js-yaml@4.3.0 \ + minimatch@3.1.4 \ + path-to-regexp@0.1.13 \ + protobufjs@7.6.1 \ + serialize-javascript@7.0.3 \ + sharp@0.35.0 \ + shell-quote@1.9.0 \ + systeminformation@5.31.7 \ + validator@13.15.22 + +FROM golang:1.26.5-alpine3.24@sha256:0178a641fbb4858c5f1b48e34bdaabe0350a330a1b1149aabd498d0699ff5fb2 AS go-patches + +RUN apk add --no-cache git=2.54.0-r0 +WORKDIR /src +RUN git clone --branch v0.25.1 --depth 1 https://github.com/evanw/esbuild.git +WORKDIR /src/esbuild +RUN CGO_ENABLED=0 go build -trimpath -ldflags='-s -w' -o /esbuild ./cmd/esbuild +WORKDIR /src +RUN git clone --branch v4.3.3 --depth 1 https://github.com/hairyhenderson/gomplate.git +WORKDIR /src/gomplate +RUN go get \ + github.com/go-git/go-billy/v5@v5.9.0 \ + github.com/go-git/go-git/v5@v5.19.0 \ + github.com/go-jose/go-jose/v4@v4.1.4 \ + go.opentelemetry.io/otel@v1.43.0 \ + go.opentelemetry.io/otel/sdk@v1.43.0 \ + golang.org/x/crypto@v0.52.0 \ + golang.org/x/net@v0.55.0 \ + google.golang.org/grpc@v1.82.1 \ + && CGO_ENABLED=0 go build -trimpath -ldflags='-s -w' -o /gomplate ./cmd/gomplate + +FROM docker.io/hyperdx/hyperdx-all-in-one:2.31.0@sha256:b01cc48cb5aaf30d630865a88217c826ab86fb9828374201f6cd7c539d5beed1 + +RUN apk update \ + && apk upgrade --no-cache --available \ + && rm -f /usr/local/bin/goose + +COPY --from=go-patches /esbuild /tmp/esbuild +COPY --from=go-patches /gomplate /usr/local/bin/gomplate +RUN find /app/packages/app/node_modules/@esbuild -path '*/bin/esbuild' \ + -exec install -m 0755 /tmp/esbuild {} \; \ + && rm /tmp/esbuild +COPY --from=node-patches /patches/node_modules/axios /app/node_modules/axios +COPY --from=node-patches /patches/node_modules/axios /app/packages/app/node_modules/axios +COPY --from=node-patches /patches/node_modules/brace-expansion /app/packages/app/node_modules/brace-expansion +COPY --from=node-patches /patches/node_modules/fast-uri /app/node_modules/fast-uri +COPY --from=node-patches /patches/node_modules/fast-uri /app/packages/app/node_modules/fast-uri +COPY --from=node-patches /patches/node_modules/immutable /app/packages/app/node_modules/immutable +COPY --from=node-patches /patches/node_modules/js-yaml /app/packages/app/node_modules/@apidevtools/json-schema-ref-parser/node_modules/js-yaml +COPY --from=node-patches /patches/node_modules/minimatch /app/packages/app/node_modules/minimatch +COPY --from=node-patches /patches/node_modules/path-to-regexp /app/node_modules/path-to-regexp +COPY --from=node-patches /patches/node_modules/path-to-regexp /app/packages/app/node_modules/path-to-regexp +COPY --from=node-patches /patches/node_modules/protobufjs /app/node_modules/@opentelemetry/otlp-transformer/node_modules/protobufjs +COPY --from=node-patches /patches/node_modules/protobufjs /app/packages/app/node_modules/@opentelemetry/otlp-transformer/node_modules/protobufjs +COPY --from=node-patches /patches/node_modules/serialize-javascript /app/packages/app/node_modules/serialize-javascript +COPY --from=node-patches /patches/node_modules/sharp /app/packages/app/node_modules/sharp +COPY --from=node-patches /patches/node_modules/shell-quote /app/node_modules/shell-quote +COPY --from=node-patches /patches/node_modules/systeminformation /app/node_modules/systeminformation +COPY --from=node-patches /patches/node_modules/validator /app/packages/app/node_modules/validator + +# hadolint ignore=DL3002 +USER "0" diff --git a/packages/phlo-clickstack/src/phlo_clickstack/service.yaml b/packages/phlo-clickstack/src/phlo_clickstack/service.yaml index aebd8328518..11c07330257 100644 --- a/packages/phlo-clickstack/src/phlo_clickstack/service.yaml +++ b/packages/phlo-clickstack/src/phlo_clickstack/service.yaml @@ -4,7 +4,13 @@ category: observability default: false profile: observability -image: ${CLICKSTACK_IMAGE:-docker.hyperdx.io/hyperdx/hyperdx-all-in-one} +image: ghcr.io/phlohouse/phlo-clickstack:2.31.0-security-patches +build: + context: . + dockerfile: clickstack/Dockerfile +files: + - source: Dockerfile + dest: clickstack/Dockerfile compose: restart: unless-stopped @@ -32,9 +38,6 @@ compose: retries: 10 env_vars: - CLICKSTACK_IMAGE: - default: "docker.hyperdx.io/hyperdx/hyperdx-all-in-one" - description: Official ClickStack all-in-one image reference CLICKSTACK_PORT: default: 18080 description: ClickStack UI port diff --git a/packages/phlo-clickstack/tests/test_clickstack_plugin.py b/packages/phlo-clickstack/tests/test_clickstack_plugin.py index 3a48088dace..e465ae087cd 100644 --- a/packages/phlo-clickstack/tests/test_clickstack_plugin.py +++ b/packages/phlo-clickstack/tests/test_clickstack_plugin.py @@ -1,5 +1,7 @@ """Tests for ClickStack service plugin.""" +from importlib import resources + from click.testing import CliRunner from phlo_clickstack.cli import clickstack_group @@ -21,6 +23,18 @@ def test_clickstack_service_definition() -> None: assert "./volumes/clickstack:/var/lib/clickhouse" not in defn["compose"]["volumes"] +def test_clickstack_builds_the_patched_stable_image() -> None: + """The generated service builds the audited stable derivative.""" + definition = ClickStackServicePlugin().service_definition + dockerfile = resources.files("phlo_clickstack").joinpath("Dockerfile").read_text() + + assert definition["image"] == "ghcr.io/phlohouse/phlo-clickstack:2.31.0-security-patches" + assert definition["build"] == {"context": ".", "dockerfile": "clickstack/Dockerfile"} + assert "FROM docker.io/hyperdx/hyperdx-all-in-one:2.31.0@sha256:b01cc48" in dockerfile + assert "docker.hyperdx.io" not in dockerfile + assert "CLICKSTACK_IMAGE" not in definition["env_vars"] + + def test_clickstack_plugin_metadata() -> None: """Validate ClickStack plugin metadata.""" plugin = ClickStackServicePlugin() diff --git a/packages/phlo-dagster/src/phlo_dagster/Dockerfile b/packages/phlo-dagster/src/phlo_dagster/Dockerfile index 21ddff875be..3b74f5ae4a1 100644 --- a/packages/phlo-dagster/src/phlo_dagster/Dockerfile +++ b/packages/phlo-dagster/src/phlo_dagster/Dockerfile @@ -1,11 +1,11 @@ -FROM python:3.12-slim AS phlo-build-context +FROM python:3.12-alpine AS phlo-build-context WORKDIR /opt/phlo-build-context COPY . . RUN mkdir -p /opt/phlo-build-context/wheelhouse -FROM python:3.12-slim +FROM python:3.12-alpine WORKDIR /opt/dagster @@ -14,11 +14,18 @@ ARG PHLO_DBT_VERSION="" ARG PHLO_WHEELHOUSE="" # Install system dependencies and uv -RUN apt-get update && apt-get install -y --no-install-recommends \ - curl \ - git \ - && rm -rf /var/lib/apt/lists/* \ - && pip install uv +RUN apk upgrade --no-cache \ + && apk add --no-cache \ + bash=5.3.9-r1 \ + ca-certificates=20260611-r0 \ + curl=8.21.0-r0 \ + git=2.54.0-r0 \ + && apk add --no-cache --virtual .build-deps \ + gcc=15.2.0-r5 \ + musl-dev=1.2.6-r2 \ + && pip install --no-cache-dir "uv==0.8.13" + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] # Copy only the artifact directory from the context stage; normal generated builds receive an # empty directory and retain the existing PyPI installation path. @@ -38,15 +45,26 @@ RUN \ else \ uv pip install --system --no-deps --prerelease explicit "phlo==$PHLO_VERSION"; \ PHLO_PRERELEASE_REQUIREMENTS="$(python -c 'import importlib.metadata as md, re; print(" ".join(req.split(";")[0].strip() for req in (md.metadata("phlo").get_all("Requires-Dist") or []) if "extra == '\''defaults'\''" in req and re.search(r"(a|b|rc|dev)[0-9]+", req)))')"; \ - uv pip install --system --prerelease explicit "phlo[defaults]==$PHLO_VERSION" "$PHLO_DBT_REQUIREMENT" $PHLO_PRERELEASE_REQUIREMENTS dagster-webserver dagster-postgres "psycopg[binary]"; \ + base_requirements=("phlo[defaults]==$PHLO_VERSION" "$PHLO_DBT_REQUIREMENT" dagster-webserver dagster-postgres "psycopg[binary]"); \ + if [ -n "$PHLO_PRERELEASE_REQUIREMENTS" ]; then read -r -a prerelease_requirements <<< "$PHLO_PRERELEASE_REQUIREMENTS"; base_requirements+=("${prerelease_requirements[@]}"); fi; \ + uv pip install --system --prerelease explicit "${base_requirements[@]}"; \ fi; \ else \ uv pip install --system "phlo[defaults]" "$PHLO_DBT_REQUIREMENT" dagster-webserver dagster-postgres "psycopg[binary]"; \ - fi + fi && apk del .build-deps + +# Build caches contain package manifests that vulnerability scanners treat as runtime +# dependencies. They are unnecessary after installation and must not ship in the image. +RUN rm -rf /root/.cache/uv /root/.cache/puccinialin # Keep entrypoint outside /opt/dagster so dev volume mounts never hide it. COPY dagster/entrypoint.sh /usr/local/bin/phlo-dagster-entrypoint.sh -RUN chmod +x /usr/local/bin/phlo-dagster-entrypoint.sh +RUN chmod +x /usr/local/bin/phlo-dagster-entrypoint.sh \ + && addgroup -S phlo \ + && adduser -S -G phlo -h /opt/dagster -H phlo \ + && chown -R phlo:phlo /opt/dagster + +USER phlo # Copy workspace configuration COPY dagster/workspace.yaml /opt/dagster/workspace.yaml diff --git a/packages/phlo-dagster/src/phlo_dagster/dagster-daemon.yaml b/packages/phlo-dagster/src/phlo_dagster/dagster-daemon.yaml index a86315ee37f..c4eb84a3bf4 100644 --- a/packages/phlo-dagster/src/phlo_dagster/dagster-daemon.yaml +++ b/packages/phlo-dagster/src/phlo_dagster/dagster-daemon.yaml @@ -5,6 +5,7 @@ category: orchestration default: true phlo_dev: true +image: ghcr.io/phlohouse/phlo-dagster:0.6.0 build: context: . dockerfile: dagster/Dockerfile diff --git a/packages/phlo-dagster/src/phlo_dagster/service.yaml b/packages/phlo-dagster/src/phlo_dagster/service.yaml index f15dbe4c523..203a834f9cc 100644 --- a/packages/phlo-dagster/src/phlo_dagster/service.yaml +++ b/packages/phlo-dagster/src/phlo_dagster/service.yaml @@ -9,6 +9,7 @@ gitignore: - dagster/schedules/ - dagster/logs/ +image: ghcr.io/phlohouse/phlo-dagster:0.6.0 build: context: . dockerfile: dagster/Dockerfile diff --git a/packages/phlo-dagster/tests/test_runtime_image_contract.py b/packages/phlo-dagster/tests/test_runtime_image_contract.py index 2ff4f7001f4..41374914997 100644 --- a/packages/phlo-dagster/tests/test_runtime_image_contract.py +++ b/packages/phlo-dagster/tests/test_runtime_image_contract.py @@ -7,16 +7,16 @@ def test_dagster_runtime_image_installs_prerelease_phlo_with_postgres_driver() -> None: dockerfile = resources.files("phlo_dagster").joinpath("Dockerfile").read_text() - assert dockerfile.startswith("FROM python:3.12-slim") + assert dockerfile.startswith("FROM python:3.12-alpine") assert ( 'uv pip install --system --no-deps --prerelease explicit "phlo==$PHLO_VERSION"' in dockerfile ) assert "PHLO_PRERELEASE_REQUIREMENTS" in dockerfile assert ( - 'uv pip install --system --prerelease explicit "phlo[defaults]==$PHLO_VERSION"' - in dockerfile + 'base_requirements=("phlo[defaults]==$PHLO_VERSION" "$PHLO_DBT_REQUIREMENT"' in dockerfile ) + assert 'uv pip install --system --prerelease explicit "${base_requirements[@]}"' in dockerfile assert 'dagster-postgres "psycopg[binary]"' in dockerfile @@ -46,6 +46,12 @@ def test_dagster_runtime_image_keeps_dbt_unpinned_when_provider_version_is_empty assert '"phlo[defaults]" "$PHLO_DBT_REQUIREMENT" dagster-webserver' in dockerfile +def test_dagster_runtime_image_removes_all_python_build_caches() -> None: + dockerfile = resources.files("phlo_dagster").joinpath("Dockerfile").read_text() + + assert "rm -rf /root/.cache/uv /root/.cache/puccinialin" in dockerfile + + def test_dagster_runtime_entrypoint_installs_mounted_project() -> None: entrypoint = resources.files("phlo_dagster").joinpath("entrypoint.sh").read_text() diff --git a/packages/phlo-grafana/README.md b/packages/phlo-grafana/README.md index 6b8d301eb99..f8b3b77a54e 100644 --- a/packages/phlo-grafana/README.md +++ b/packages/phlo-grafana/README.md @@ -23,7 +23,7 @@ Part of the `observability` profile. | Variable | Default | Description | | ------------------------ | -------- | ------------------- | | `GRAFANA_PORT` | `3003` | Grafana web UI port | -| `GRAFANA_VERSION` | `11.3.1` | Grafana version | +| `GRAFANA_VERSION` | `13.1.1` | Grafana version | | `GRAFANA_ADMIN_USER` | `admin` | Admin username | | `GRAFANA_ADMIN_PASSWORD` | `admin` | Admin password | diff --git a/packages/phlo-grafana/pyproject.toml b/packages/phlo-grafana/pyproject.toml index ead7ef6fbda..f511a81edde 100644 --- a/packages/phlo-grafana/pyproject.toml +++ b/packages/phlo-grafana/pyproject.toml @@ -41,6 +41,7 @@ include-package-data = true [tool.setuptools.package-data] phlo_grafana = [ + "Dockerfile", "service.yaml", "provisioning/datasources/datasources.yml", "provisioning/dashboards/dashboards.yml", diff --git a/packages/phlo-grafana/src/phlo_grafana/Dockerfile b/packages/phlo-grafana/src/phlo_grafana/Dockerfile new file mode 100644 index 00000000000..670df16b37f --- /dev/null +++ b/packages/phlo-grafana/src/phlo_grafana/Dockerfile @@ -0,0 +1,37 @@ +FROM golang:1.26.5-alpine3.23@sha256:622e56dbc11a8cfe87cafa2331e9a201877271cbff918af53d3be315f3da88cc AS build + +RUN apk add --no-cache git=2.52.0-r0 + +WORKDIR /src/grafana +RUN git clone https://github.com/grafana/grafana.git . && \ + git checkout a9cee6e1724a455676bb6c05eef7fc54aa4b19f4 +RUN go get \ + github.com/getkin/kin-openapi@v0.144.0 \ + github.com/grafana/tempo@8100f5a7b8f0986edf1fcd2ff6552d174b25ec18 \ + google.golang.org/grpc@v1.82.1 && \ + CGO_ENABLED=0 go build -buildvcs=false -trimpath \ + -ldflags="-s -w -X main.version=13.1.1 -X main.commit=a9cee6e1724a455676bb6c05eef7fc54aa4b19f4" \ + -o /out/grafana ./pkg/cmd/grafana + +WORKDIR /src/elasticsearch +RUN git clone https://github.com/grafana/grafana-elasticsearch-datasource.git . && \ + git checkout 525671446900cf0c7c5d696f931b95c2cec81d9c && \ + go get google.golang.org/grpc@v1.82.1 && \ + CGO_ENABLED=0 go build -buildvcs=false -trimpath -ldflags="-s -w" \ + -o /out/elasticsearch ./pkg + +WORKDIR /src/zipkin +RUN git clone https://github.com/grafana/grafana-zipkin-datasource.git . && \ + git checkout daafdfe11421dd2cacc8bfdb9ab604da1875b671 && \ + go get golang.org/x/net@v0.55.0 google.golang.org/grpc@v1.82.1 && \ + CGO_ENABLED=0 go build -buildvcs=false -trimpath -ldflags="-s -w" \ + -o /out/zipkin ./pkg + +FROM grafana/grafana:13.1.1@sha256:7cb8c64c4d57a57e734073f3cc94620adb24a0acb929bd80ba9f14017e3a975b + +ARG TARGETARCH +COPY --from=build /out/grafana /usr/share/grafana/bin/grafana +COPY --from=build /out/elasticsearch /usr/share/grafana/data/plugins-bundled/elasticsearch/gpx_grafana_elasticsearch_datasource_linux_${TARGETARCH} +COPY --from=build /out/zipkin /usr/share/grafana/data/plugins-bundled/zipkin/gpx_grafana-zipkin-datasource_linux_${TARGETARCH} + +USER "472" diff --git a/packages/phlo-grafana/src/phlo_grafana/dashboards/cascade-overview.json b/packages/phlo-grafana/src/phlo_grafana/dashboards/cascade-overview.json index be0bf7e0368..7a22a2b4043 100644 --- a/packages/phlo-grafana/src/phlo_grafana/dashboards/cascade-overview.json +++ b/packages/phlo-grafana/src/phlo_grafana/dashboards/cascade-overview.json @@ -67,7 +67,7 @@ }, "textMode": "auto" }, - "pluginVersion": "11.3.1", + "pluginVersion": "13.1.1", "targets": [ { "datasource": { @@ -127,7 +127,7 @@ }, "textMode": "auto" }, - "pluginVersion": "11.3.1", + "pluginVersion": "13.1.1", "targets": [ { "datasource": { @@ -187,7 +187,7 @@ }, "textMode": "auto" }, - "pluginVersion": "11.3.1", + "pluginVersion": "13.1.1", "targets": [ { "datasource": { @@ -247,7 +247,7 @@ }, "textMode": "auto" }, - "pluginVersion": "11.3.1", + "pluginVersion": "13.1.1", "targets": [ { "datasource": { @@ -369,7 +369,7 @@ "sort": "none" } }, - "pluginVersion": "11.3.1", + "pluginVersion": "13.1.1", "targets": [ { "datasource": { @@ -456,7 +456,7 @@ "sort": "none" } }, - "pluginVersion": "11.3.1", + "pluginVersion": "13.1.1", "targets": [ { "datasource": { diff --git a/packages/phlo-grafana/src/phlo_grafana/dashboards/infrastructure.json b/packages/phlo-grafana/src/phlo_grafana/dashboards/infrastructure.json index 8073f0418d6..87b0c1b050a 100644 --- a/packages/phlo-grafana/src/phlo_grafana/dashboards/infrastructure.json +++ b/packages/phlo-grafana/src/phlo_grafana/dashboards/infrastructure.json @@ -87,7 +87,7 @@ "sizing": "auto", "valueMode": "color" }, - "pluginVersion": "11.3.1", + "pluginVersion": "13.1.1", "targets": [ { "datasource": { @@ -235,7 +235,7 @@ "sort": "none" } }, - "pluginVersion": "11.3.1", + "pluginVersion": "13.1.1", "targets": [ { "datasource": { @@ -371,7 +371,7 @@ "sort": "none" } }, - "pluginVersion": "11.3.1", + "pluginVersion": "13.1.1", "targets": [ { "datasource": { @@ -467,7 +467,7 @@ "sort": "none" } }, - "pluginVersion": "11.3.1", + "pluginVersion": "13.1.1", "targets": [ { "datasource": { @@ -567,7 +567,7 @@ "sort": "none" } }, - "pluginVersion": "11.3.1", + "pluginVersion": "13.1.1", "targets": [ { "datasource": { @@ -664,7 +664,7 @@ "sort": "none" } }, - "pluginVersion": "11.3.1", + "pluginVersion": "13.1.1", "targets": [ { "datasource": { diff --git a/packages/phlo-grafana/src/phlo_grafana/service.yaml b/packages/phlo-grafana/src/phlo_grafana/service.yaml index 82b144acd81..b9a49514191 100644 --- a/packages/phlo-grafana/src/phlo_grafana/service.yaml +++ b/packages/phlo-grafana/src/phlo_grafana/service.yaml @@ -4,7 +4,11 @@ category: observability default: false profile: observability -image: grafana/grafana:${GRAFANA_VERSION:-11.3.1} +image: ghcr.io/phlohouse/phlo-grafana:13.1.1-go1.26.5 + +build: + context: ./grafana + dockerfile: Dockerfile depends_on: - prometheus @@ -12,12 +16,12 @@ depends_on: compose: restart: unless-stopped - user: "0" labels: phlo.metrics.enabled: "true" phlo.metrics.port: "grafana:3000" phlo.metrics.path: "/metrics" environment: + GF_PLUGINS_PREINSTALL_DISABLED: "true" GF_SECURITY_ADMIN_USER: ${GRAFANA_ADMIN_USER:-admin} GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-admin} GF_USERS_ALLOW_SIGN_UP: "false" @@ -43,7 +47,7 @@ compose: env_vars: GRAFANA_VERSION: - default: "11.3.1" + default: "13.1.1" description: Grafana version GRAFANA_PORT: default: 3003 @@ -67,3 +71,5 @@ files: dest: grafana/provisioning - source: dashboards dest: grafana/dashboards + - source: Dockerfile + dest: grafana/Dockerfile diff --git a/packages/phlo-grafana/tests/test_integration_grafana.py b/packages/phlo-grafana/tests/test_integration_grafana.py index 09e124d0286..628c3451a5e 100644 --- a/packages/phlo-grafana/tests/test_integration_grafana.py +++ b/packages/phlo-grafana/tests/test_integration_grafana.py @@ -24,3 +24,18 @@ def test_grafana_service_definition(): assert isinstance(service_def, dict) assert "grafana-data:/var/lib/grafana" in service_def["compose"]["volumes"] assert "./volumes/grafana:/var/lib/grafana" not in service_def["compose"]["volumes"] + + +def test_grafana_uses_rebuilt_image(): + from phlo_grafana.plugin import GrafanaServicePlugin + + service_def = GrafanaServicePlugin().service_definition + + assert service_def["image"] == "ghcr.io/phlohouse/phlo-grafana:13.1.1-go1.26.5" + assert service_def["build"] == { + "context": "./grafana", + "dockerfile": "Dockerfile", + } + assert {"source": "Dockerfile", "dest": "grafana/Dockerfile"} in service_def["files"] + assert service_def["compose"].get("user") is None + assert service_def["compose"]["environment"]["GF_PLUGINS_PREINSTALL_DISABLED"] == "true" diff --git a/packages/phlo-hasura/README.md b/packages/phlo-hasura/README.md index 8a08927b6cd..2fed0e36325 100644 --- a/packages/phlo-hasura/README.md +++ b/packages/phlo-hasura/README.md @@ -23,7 +23,7 @@ Part of the `api` profile. | Variable | Default | Description | | --------------------- | -------------------------- | ----------------------- | | `HASURA_PORT` | `8082` | Hasura console/API port | -| `HASURA_VERSION` | `v2.46.0` | Hasura version | +| `HASURA_VERSION` | `v2.49.5` | Hasura version | | `HASURA_ADMIN_SECRET` | `phlo-hasura-admin-secret` | Admin secret | ## Auto-Configuration diff --git a/packages/phlo-hasura/src/phlo_hasura/service.yaml b/packages/phlo-hasura/src/phlo_hasura/service.yaml index 49d8801ca2f..e842207e722 100644 --- a/packages/phlo-hasura/src/phlo_hasura/service.yaml +++ b/packages/phlo-hasura/src/phlo_hasura/service.yaml @@ -4,7 +4,7 @@ category: api default: false profile: api -image: hasura/graphql-engine:${HASURA_VERSION:-v2.46.0} +image: hasura/graphql-engine:${HASURA_VERSION:-v2.49.5} depends_on: - postgres @@ -32,7 +32,7 @@ compose: env_vars: HASURA_VERSION: - default: v2.46.0 + default: v2.49.5 description: Hasura GraphQL Engine version HASURA_PORT: default: 8082 diff --git a/packages/phlo-iceberg/pyproject.toml b/packages/phlo-iceberg/pyproject.toml index 7e456ca7497..0c16e040806 100644 --- a/packages/phlo-iceberg/pyproject.toml +++ b/packages/phlo-iceberg/pyproject.toml @@ -9,7 +9,8 @@ requires = [ dependencies = [ "phlo>=0.1.0", "pandera>=0.26.1", - "pyiceberg[s3fs,pyarrow]>=0.11.0", + "pyarrow>=21.0.0", + "pyiceberg[s3fs]>=0.11.0", ] description = "Iceberg table-store capability plugin for Phlo" name = "phlo-iceberg" diff --git a/packages/phlo-loki/README.md b/packages/phlo-loki/README.md index 09712a00a0b..fe2b92241fc 100644 --- a/packages/phlo-loki/README.md +++ b/packages/phlo-loki/README.md @@ -23,7 +23,6 @@ Part of the `observability` profile. | Variable | Default | Description | | -------------- | ------- | ------------- | | `LOKI_PORT` | `3100` | Loki API port | -| `LOKI_VERSION` | `3.2.1` | Loki version | ## Auto-Configuration diff --git a/packages/phlo-loki/pyproject.toml b/packages/phlo-loki/pyproject.toml index d4e4b854ac6..b1f529c9d28 100644 --- a/packages/phlo-loki/pyproject.toml +++ b/packages/phlo-loki/pyproject.toml @@ -45,6 +45,7 @@ include-package-data = true [tool.setuptools.package-data] phlo_loki = [ + "Dockerfile", "service.yaml", "loki-config.yml", "observatory_assets/*", diff --git a/packages/phlo-loki/src/phlo_loki/Dockerfile b/packages/phlo-loki/src/phlo_loki/Dockerfile new file mode 100644 index 00000000000..11920f2e830 --- /dev/null +++ b/packages/phlo-loki/src/phlo_loki/Dockerfile @@ -0,0 +1,19 @@ +FROM golang:1.26.5-alpine3.24@sha256:0178a641fbb4858c5f1b48e34bdaabe0350a330a1b1149aabd498d0699ff5fb2 AS build + +RUN apk add --no-cache git=2.54.0-r0 +RUN git clone https://github.com/grafana/loki.git /src/loki && \ + git -C /src/loki checkout b318f2829f0ae2094ab3a1e90780450e9e4b03be +WORKDIR /src/loki +RUN go get google.golang.org/grpc@v1.82.1 && go mod tidy +RUN CGO_ENABLED=0 go build -mod=mod -trimpath -tags netgo \ + -ldflags="-s -w \ + -X github.com/grafana/loki/v3/pkg/util/build.Branch=release-3.7.x \ + -X github.com/grafana/loki/v3/pkg/util/build.Version=3.7.4 \ + -X github.com/grafana/loki/v3/pkg/util/build.Revision=b318f2829f0ae2094ab3a1e90780450e9e4b03be" \ + -o /out/loki ./cmd/loki + +FROM grafana/loki:3.7.4@sha256:87f0a067673756a3cede1bcbf0c74875f7df9b09fddb53e399d0c576f756cfcc + +COPY --from=build /out/loki /usr/bin/loki + +USER 10001 diff --git a/packages/phlo-loki/src/phlo_loki/service.yaml b/packages/phlo-loki/src/phlo_loki/service.yaml index f7e8f91cd22..dc6b980d429 100644 --- a/packages/phlo-loki/src/phlo_loki/service.yaml +++ b/packages/phlo-loki/src/phlo_loki/service.yaml @@ -4,7 +4,10 @@ category: observability default: false profile: observability -image: grafana/loki:${LOKI_VERSION:-3.2.1} +image: ghcr.io/phlohouse/phlo-loki:3.7.4-grpc1.82.1 +build: + context: . + dockerfile: loki/Dockerfile compose: restart: unless-stopped @@ -34,9 +37,6 @@ compose: retries: 5 env_vars: - LOKI_VERSION: - default: "3.2.1" - description: Loki version LOKI_PORT: default: 3100 description: Loki API port @@ -48,5 +48,7 @@ env_vars: description: Path used to build Loki log query links files: + - source: Dockerfile + dest: loki/Dockerfile - source: loki-config.yml dest: loki/loki-config.yml diff --git a/packages/phlo-loki/tests/test_loki_plugin.py b/packages/phlo-loki/tests/test_loki_plugin.py index e9604453d4d..0e1e7a8a0fc 100644 --- a/packages/phlo-loki/tests/test_loki_plugin.py +++ b/packages/phlo-loki/tests/test_loki_plugin.py @@ -15,6 +15,15 @@ def test_loki_service_definition(): assert "./volumes/loki:/loki" not in defn["compose"]["volumes"] +def test_loki_service_builds_patched_release_image() -> None: + """Generated Loki uses the stable release with its fixed gRPC dependency.""" + definition = LokiServicePlugin().service_definition + + assert definition["image"] == "ghcr.io/phlohouse/phlo-loki:3.7.4-grpc1.82.1" + assert definition["build"] == {"context": ".", "dockerfile": "loki/Dockerfile"} + assert "LOKI_VERSION" not in definition["env_vars"] + + def test_loki_plugin_metadata(): """Validate Loki plugin metadata tags and name.""" diff --git a/packages/phlo-minio/pyproject.toml b/packages/phlo-minio/pyproject.toml index 7d84fe449c3..5c7d69487f8 100644 --- a/packages/phlo-minio/pyproject.toml +++ b/packages/phlo-minio/pyproject.toml @@ -48,6 +48,8 @@ include-package-data = true [tool.setuptools.package-data] phlo_minio = [ + "Dockerfile", + "mc.Dockerfile", "service.yaml", "minio-setup.yaml", ] diff --git a/packages/phlo-minio/src/phlo_minio/Dockerfile b/packages/phlo-minio/src/phlo_minio/Dockerfile new file mode 100644 index 00000000000..989148afbe4 --- /dev/null +++ b/packages/phlo-minio/src/phlo_minio/Dockerfile @@ -0,0 +1,50 @@ +FROM golang:1.26.5-alpine3.24@sha256:0178a641fbb4858c5f1b48e34bdaabe0350a330a1b1149aabd498d0699ff5fb2 AS build + +RUN apk add --no-cache git=2.54.0-r0 +WORKDIR /src +RUN git clone https://github.com/minio/minio.git . && \ + git checkout 7aac2a2c5b7c882e68c1ce017d8256be2feea27f && \ + go get \ + github.com/apache/thrift@v0.23.0 \ + github.com/buger/jsonparser@v1.1.2 \ + github.com/go-jose/go-jose/v4@v4.1.4 \ + github.com/prometheus/prometheus@v0.311.3 \ + go.opentelemetry.io/otel/sdk@v1.43.0 \ + golang.org/x/crypto@v0.52.0 \ + golang.org/x/net@v0.55.0 \ + google.golang.org/grpc@v1.82.1 && \ + go mod tidy -e +# hadolint ignore=DL3062 +RUN LDFLAGS="$(go run buildscripts/gen-ldflags.go)" && \ + CGO_ENABLED=0 go build -tags kqueue -trimpath -ldflags="$LDFLAGS -s -w" -o /out/minio . + +FROM golang:1.26.5-alpine3.24@sha256:0178a641fbb4858c5f1b48e34bdaabe0350a330a1b1149aabd498d0699ff5fb2 AS mc-build + +RUN apk add --no-cache git=2.54.0-r0 +WORKDIR /src +RUN git clone https://github.com/minio/mc.git . && \ + git checkout 77f82e18b5401a65958f1619df6ebb994634bd88 && \ + go get \ + github.com/prometheus/prometheus@v0.311.3 \ + golang.org/x/crypto@v0.52.0 \ + golang.org/x/net@v0.55.0 \ + google.golang.org/grpc@v1.82.1 && \ + go mod tidy +# hadolint ignore=DL3062 +RUN LDFLAGS="$(go run buildscripts/gen-ldflags.go)" && \ + CGO_ENABLED=0 go build -tags kqueue -trimpath -ldflags="$LDFLAGS -s -w" -o /out/mc . + +FROM alpine:3.22.5@sha256:14358309a308569c32bdc37e2e0e9694be33a9d99e68afb0f5ff33cc1f695dce + +RUN apk upgrade --no-cache && \ + apk add --no-cache ca-certificates=20260611-r0 curl=8.14.1-r3 && \ + adduser -D -h /home/minio -s /sbin/nologin minio && \ + mkdir /data && \ + chown minio:minio /data +COPY --from=build /out/minio /usr/bin/minio +COPY --from=mc-build /out/mc /usr/bin/mc + +ENV HOME=/home/minio +USER minio +EXPOSE 9000 9001 +ENTRYPOINT ["/usr/bin/minio"] diff --git a/packages/phlo-minio/src/phlo_minio/mc.Dockerfile b/packages/phlo-minio/src/phlo_minio/mc.Dockerfile new file mode 100644 index 00000000000..0e64c7c5223 --- /dev/null +++ b/packages/phlo-minio/src/phlo_minio/mc.Dockerfile @@ -0,0 +1,26 @@ +FROM golang:1.26.5-alpine3.24@sha256:0178a641fbb4858c5f1b48e34bdaabe0350a330a1b1149aabd498d0699ff5fb2 AS build + +RUN apk add --no-cache git=2.54.0-r0 +WORKDIR /src +RUN git clone https://github.com/minio/mc.git . && \ + git checkout 77f82e18b5401a65958f1619df6ebb994634bd88 && \ + go get \ + github.com/prometheus/prometheus@v0.311.3 \ + golang.org/x/crypto@v0.52.0 \ + golang.org/x/net@v0.55.0 \ + google.golang.org/grpc@v1.82.1 && \ + go mod tidy +# hadolint ignore=DL3062 +RUN LDFLAGS="$(go run buildscripts/gen-ldflags.go)" && \ + CGO_ENABLED=0 go build -tags kqueue -trimpath -ldflags="$LDFLAGS -s -w" -o /out/mc . + +FROM alpine:3.22.5@sha256:14358309a308569c32bdc37e2e0e9694be33a9d99e68afb0f5ff33cc1f695dce + +RUN apk upgrade --no-cache && \ + apk add --no-cache ca-certificates=20260611-r0 && \ + adduser -D -h /home/mc -s /sbin/nologin mc +COPY --from=build /out/mc /usr/bin/mc + +ENV HOME=/home/mc +USER mc +ENTRYPOINT ["/usr/bin/mc"] diff --git a/packages/phlo-minio/src/phlo_minio/minio-setup.yaml b/packages/phlo-minio/src/phlo_minio/minio-setup.yaml index 5ebe0df3f1f..0cbb420731d 100644 --- a/packages/phlo-minio/src/phlo_minio/minio-setup.yaml +++ b/packages/phlo-minio/src/phlo_minio/minio-setup.yaml @@ -4,7 +4,10 @@ description: Initialize MinIO buckets for data lake category: core default: true -image: minio/mc:RELEASE.2025-08-13T08-35-41Z +image: ghcr.io/phlohouse/phlo-minio-mc:77f82e18b540 +build: + context: . + dockerfile: minio-mc/Dockerfile depends_on: - minio @@ -23,3 +26,7 @@ compose: mc mb --ignore-existing myminio/lake/stage && echo 'Buckets created successfully' " + +files: + - source: mc.Dockerfile + dest: minio-mc/Dockerfile diff --git a/packages/phlo-minio/src/phlo_minio/service.yaml b/packages/phlo-minio/src/phlo_minio/service.yaml index 2273582d4b8..609812c650a 100644 --- a/packages/phlo-minio/src/phlo_minio/service.yaml +++ b/packages/phlo-minio/src/phlo_minio/service.yaml @@ -3,7 +3,10 @@ description: S3-compatible object storage for data lake category: core default: true -image: minio/minio:RELEASE.2025-09-07T16-13-09Z +image: ghcr.io/phlohouse/phlo-minio:7aac2a2c5b7c +build: + context: . + dockerfile: minio/Dockerfile compose: restart: unless-stopped @@ -59,6 +62,7 @@ env_vars: MINIO_ROOT_USER: default: minio description: MinIO root username + MINIO_ROOT_PASSWORD: default: minio123 description: MinIO root password @@ -118,3 +122,7 @@ env_vars: MINIO_AUDIT_ENDPOINT: default: "" description: "Webhook endpoint for audit logs" + +files: + - source: Dockerfile + dest: minio/Dockerfile diff --git a/packages/phlo-minio/tests/test_minio_plugin.py b/packages/phlo-minio/tests/test_minio_plugin.py index 74a027fb8aa..e51b561a652 100644 --- a/packages/phlo-minio/tests/test_minio_plugin.py +++ b/packages/phlo-minio/tests/test_minio_plugin.py @@ -1,6 +1,8 @@ """Tests for MinIO service plugin.""" -from phlo_minio.plugin import MinioResourceProvider, MinioServicePlugin +from importlib import resources + +from phlo_minio.plugin import MinioResourceProvider, MinioServicePlugin, MinioSetupServicePlugin def test_minio_service_definition(): @@ -23,6 +25,24 @@ def test_minio_service_uses_named_volume(): assert all("./volumes/minio" not in volume for volume in volumes) +def test_minio_services_build_pinned_phlo_images() -> None: + """Server and client should build hardened Phlo-owned images from pinned source.""" + server = MinioServicePlugin().service_definition + setup = MinioSetupServicePlugin().service_definition + + assert server["image"] == "ghcr.io/phlohouse/phlo-minio:7aac2a2c5b7c" + assert server["build"]["dockerfile"] == "minio/Dockerfile" + assert setup["image"] == "ghcr.io/phlohouse/phlo-minio-mc:77f82e18b540" + assert setup["build"]["dockerfile"] == "minio-mc/Dockerfile" + + +def test_minio_server_image_includes_the_public_cli_runtime_client() -> None: + """Release maintenance commands execute `mc` inside the MinIO server container.""" + dockerfile = resources.files("phlo_minio").joinpath("Dockerfile").read_text() + + assert "COPY --from=mc-build /out/mc /usr/bin/mc" in dockerfile + + def test_minio_resource_provider_exposes_object_store(monkeypatch) -> None: """MinIO should expose an object_store capability.""" monkeypatch.setattr( diff --git a/packages/phlo-nessie/README.md b/packages/phlo-nessie/README.md index bba9a36ec04..cfe3f2afabe 100644 --- a/packages/phlo-nessie/README.md +++ b/packages/phlo-nessie/README.md @@ -22,7 +22,6 @@ pip install 'phlo-nessie[trino]' | Variable | Default | Description | | ---------------------- | --------- | -------------------------- | | `NESSIE_PORT` | `10003` | Nessie API host port | -| `NESSIE_VERSION` | `0.107.2` | Nessie version | | `NESSIE_OIDC_ENABLED` | `false` | Enable OIDC authentication | | `NESSIE_AUTHZ_ENABLED` | `false` | Enable authorization | diff --git a/packages/phlo-nessie/pyproject.toml b/packages/phlo-nessie/pyproject.toml index 51a40d2c53c..a72761455ab 100644 --- a/packages/phlo-nessie/pyproject.toml +++ b/packages/phlo-nessie/pyproject.toml @@ -62,6 +62,8 @@ include-package-data = true [tool.setuptools.package-data] phlo_nessie = [ + "Dockerfile", + "libraries.sha256", "service.yaml", "observatory_assets/*", ] diff --git a/packages/phlo-nessie/src/phlo_nessie/Dockerfile b/packages/phlo-nessie/src/phlo_nessie/Dockerfile new file mode 100644 index 00000000000..680eb136060 --- /dev/null +++ b/packages/phlo-nessie/src/phlo_nessie/Dockerfile @@ -0,0 +1,44 @@ +FROM eclipse-temurin:21-jdk-alpine@sha256:1ff763083f2993d57d0bf374ab10bb3e2cb873af6c13a04458ebbd3e0337dc76 AS build + +RUN apk add --no-cache bash=5.3.3-r1 curl=8.20.0-r0 git=2.52.0-r0 +WORKDIR /src +RUN git clone https://github.com/projectnessie/nessie.git . && \ + git checkout b715f4374d69e68bcfac0e359bf2dd55875b3e25 +RUN ./gradlew --no-daemon :nessie-quarkus:quarkusBuild -x check +RUN mkdir /patches && \ + for artifact in \ + netty-buffer netty-codec netty-codec-dns netty-codec-haproxy \ + netty-codec-http netty-codec-http2 netty-codec-socks netty-common \ + netty-handler netty-handler-proxy netty-resolver netty-resolver-dns \ + netty-resolver-dns-classes-macos netty-transport \ + netty-transport-classes-epoll netty-transport-classes-kqueue \ + netty-transport-native-unix-common; do \ + curl --fail --silent --show-error --location \ + "https://repo1.maven.org/maven2/io/netty/${artifact}/4.1.136.Final/${artifact}-4.1.136.Final.jar" \ + --output "/patches/io.netty.${artifact}-4.1.136.Final.jar"; \ + done && \ + curl --fail --silent --show-error --location \ + "https://repo1.maven.org/maven2/com/fasterxml/jackson/core/jackson-core/2.22.1/jackson-core-2.22.1.jar" \ + --output /patches/com.fasterxml.jackson.core.jackson-core-2.22.1.jar +COPY nessie/libraries.sha256 /patches/libraries.sha256 +WORKDIR /patches +RUN sha256sum --check libraries.sha256 && \ + for artifact in io.netty.*-4.1.136.Final.jar; do \ + mv "${artifact}" "${artifact%-4.1.136.Final.jar}-4.1.135.Final.jar"; \ + done && \ + mv com.fasterxml.jackson.core.jackson-core-2.22.1.jar \ + com.fasterxml.jackson.core.jackson-core-2.22.0.jar + +FROM eclipse-temurin:21-jre-alpine@sha256:3f08b13888f595cc49edabea7250ba69499ba25602b267da591720769400e08c + +RUN apk upgrade --no-cache && \ + apk add --no-cache curl=8.20.0-r0 && \ + addgroup -g 10001 nessie && \ + adduser -D -u 10000 -G nessie -h /home/nessie -s /sbin/nologin nessie +COPY --from=build --chown=nessie:nessie /src/servers/quarkus-server/build/quarkus-app/ /deployments/ +COPY --from=build --chown=nessie:nessie /patches/*.jar /deployments/lib/main/ + +ENV HOME=/home/nessie +USER nessie +EXPOSE 19120 +ENTRYPOINT ["java", "-jar", "/deployments/quarkus-run.jar"] diff --git a/packages/phlo-nessie/src/phlo_nessie/libraries.sha256 b/packages/phlo-nessie/src/phlo_nessie/libraries.sha256 new file mode 100644 index 00000000000..3063af1ed84 --- /dev/null +++ b/packages/phlo-nessie/src/phlo_nessie/libraries.sha256 @@ -0,0 +1,18 @@ +941ff029bcdb93e83d209ce516c1a7fb8bbac07d0a2fa122f5bf194b2cd7b4f4 com.fasterxml.jackson.core.jackson-core-2.22.1.jar +c88f13fc41156fde5df918c7b240038dfbe97ac57576163f208630391789b5d5 io.netty.netty-buffer-4.1.136.Final.jar +2de4fc13005c7740b46427a47ee04265a19779c147d53e583f441889b1148159 io.netty.netty-codec-4.1.136.Final.jar +89bddc4ec42756c41a0195a50a1323a324836162b6712a843c3de5c29096c93b io.netty.netty-codec-dns-4.1.136.Final.jar +841be7fc6f1c929ecc63c70a909e0df085b4e80e44680acf46708f0e93e68037 io.netty.netty-codec-haproxy-4.1.136.Final.jar +ffd1e1b19a533bc6e47ef2cbc1290374ff4a7cb53a280defa3df392538214948 io.netty.netty-codec-http-4.1.136.Final.jar +14f67ae095c056b062aa0ee2c7b01f6b78004cf3620a6e9061bcb637f079387a io.netty.netty-codec-http2-4.1.136.Final.jar +341f47dbaac667a6e7e6cad4114218025f9755ced641bb0740fb69e34d29b421 io.netty.netty-codec-socks-4.1.136.Final.jar +e2f73be7ec359b46583ad875521137000eff520bafd320e730846ec9974f3be6 io.netty.netty-common-4.1.136.Final.jar +54bb1a59f46a3aefd117942e6acee09e555b756776bad731fc06159d89c2da28 io.netty.netty-handler-4.1.136.Final.jar +47400551f0444dea33629ee0c52d4e30856b2ddf00b12adf1881902957e74cf2 io.netty.netty-handler-proxy-4.1.136.Final.jar +e64972fec474f5b9bd086b738154d190a923e82c4923ac550aff4f5ea9e98600 io.netty.netty-resolver-4.1.136.Final.jar +9c05a9b18b5d54fcc1569c48b93958019cf3dde5ae7011f8a46451ee05e7daff io.netty.netty-resolver-dns-4.1.136.Final.jar +c8c3ad3b364d302cf2a3ed8702bfeae455536963382d0d9260b9fc47ea6f234a io.netty.netty-resolver-dns-classes-macos-4.1.136.Final.jar +b92881ea925721ed42fee5122a42b8bce4b84da737dbc3ca2d84dfaca52c28b6 io.netty.netty-transport-4.1.136.Final.jar +f6a0b631b98667f131daf4ca07a9cae1072d58e259dcbc0f8c6a053d843449c6 io.netty.netty-transport-classes-epoll-4.1.136.Final.jar +4ed7ee5cb8c611e879d0d6621e8a1b4255e7ba5a31e5f7828d635682ef244a20 io.netty.netty-transport-classes-kqueue-4.1.136.Final.jar +7e014c9b13defd9d254d4e5a5edd8ab6dde17533e1152f99699a6b28b2967a8a io.netty.netty-transport-native-unix-common-4.1.136.Final.jar diff --git a/packages/phlo-nessie/src/phlo_nessie/service.yaml b/packages/phlo-nessie/src/phlo_nessie/service.yaml index 3e39fc29f4c..838909859b1 100644 --- a/packages/phlo-nessie/src/phlo_nessie/service.yaml +++ b/packages/phlo-nessie/src/phlo_nessie/service.yaml @@ -3,7 +3,10 @@ description: Git-like catalog for Iceberg tables with branch/merge support category: core default: true -image: ghcr.io/projectnessie/nessie:${NESSIE_VERSION:-0.107.2} +image: ghcr.io/phlohouse/phlo-nessie:0.108.3-netty4.2.16 +build: + context: . + dockerfile: nessie/Dockerfile depends_on: - postgres @@ -50,9 +53,6 @@ compose: start_period: 30s env_vars: - NESSIE_VERSION: - default: "0.107.2" - description: Nessie version NESSIE_PORT: default: 10003 description: Nessie API host port @@ -94,3 +94,9 @@ hooks: - phlo_nessie.hooks - init-branches timeout_seconds: 60 + +files: + - source: Dockerfile + dest: nessie/Dockerfile + - source: libraries.sha256 + dest: nessie/libraries.sha256 diff --git a/packages/phlo-nessie/src/phlo_nessie/settings.py b/packages/phlo-nessie/src/phlo_nessie/settings.py index 23558c1cd05..d99a3da1291 100644 --- a/packages/phlo-nessie/src/phlo_nessie/settings.py +++ b/packages/phlo-nessie/src/phlo_nessie/settings.py @@ -27,7 +27,7 @@ class NessieSettings(BaseConfig): """Nessie catalog configuration.""" - nessie_version: str = Field(default="0.107.2", description="Nessie version") + nessie_version: str = Field(default="0.108.3", description="Nessie version") nessie_port: int = Field(default=19120, description="Nessie REST API port") nessie_host: str = Field(default="nessie", description="Nessie service hostname") nessie_api_version: str = Field(default="v1", description="Nessie API version") diff --git a/packages/phlo-nessie/tests/test_nessie_plugin.py b/packages/phlo-nessie/tests/test_nessie_plugin.py index e74949075e9..1b1e49acf07 100644 --- a/packages/phlo-nessie/tests/test_nessie_plugin.py +++ b/packages/phlo-nessie/tests/test_nessie_plugin.py @@ -1,5 +1,7 @@ """Tests for Nessie service plugin.""" +from importlib import resources + from phlo.capabilities import CapabilitySupport from phlo_nessie.plugin import NessieServicePlugin from phlo_nessie.resource import NessieResource @@ -16,6 +18,21 @@ def test_nessie_service_definition(): assert service_definition["category"] == "core" +def test_nessie_service_builds_the_patched_stable_image() -> None: + definition = NessieServicePlugin().service_definition + + assert definition["image"] == "ghcr.io/phlohouse/phlo-nessie:0.108.3-netty4.2.16" + assert definition["build"] == {"context": ".", "dockerfile": "nessie/Dockerfile"} + assert "NESSIE_VERSION" not in definition["env_vars"] + assert { + "source": "libraries.sha256", + "dest": "nessie/libraries.sha256", + } in definition["files"] + + dockerfile = resources.files("phlo_nessie").joinpath("Dockerfile").read_text() + assert "COPY nessie/libraries.sha256 /patches/libraries.sha256" in dockerfile + + def test_nessie_resource_provider_registers_catalog_capability() -> None: """Nessie should register as a versioned catalog capability.""" provider = NessieResourceProvider() diff --git a/packages/phlo-oauth2-proxy/pyproject.toml b/packages/phlo-oauth2-proxy/pyproject.toml index d9b7c039263..ee3e77832b7 100644 --- a/packages/phlo-oauth2-proxy/pyproject.toml +++ b/packages/phlo-oauth2-proxy/pyproject.toml @@ -40,7 +40,7 @@ target-version = "py311" include-package-data = true [tool.setuptools.package-data] -phlo_oauth2_proxy = ["service.yaml"] +phlo_oauth2_proxy = ["Dockerfile", "service.yaml"] [tool.setuptools.package-dir] "" = "src" diff --git a/packages/phlo-oauth2-proxy/src/phlo_oauth2_proxy/Dockerfile b/packages/phlo-oauth2-proxy/src/phlo_oauth2_proxy/Dockerfile new file mode 100644 index 00000000000..87a582c584b --- /dev/null +++ b/packages/phlo-oauth2-proxy/src/phlo_oauth2_proxy/Dockerfile @@ -0,0 +1,16 @@ +FROM golang:1.26.5-alpine3.24@sha256:0178a641fbb4858c5f1b48e34bdaabe0350a330a1b1149aabd498d0699ff5fb2 AS build + +RUN apk add --no-cache git=2.54.0-r0 +WORKDIR /src +RUN git clone https://github.com/oauth2-proxy/oauth2-proxy.git . && \ + git checkout 66b3a17db09f0b51a4bc3159d4c7fe3fbeca1288 && \ + go get google.golang.org/grpc@v1.82.1 +RUN CGO_ENABLED=0 go build -trimpath \ + -ldflags="-s -w -X github.com/oauth2-proxy/oauth2-proxy/v7/pkg/version.VERSION=v7.15.3" \ + -o /out/oauth2-proxy . + +FROM quay.io/oauth2-proxy/oauth2-proxy:v7.15.3@sha256:10a1165743a192e1940b4708fb9647027185ce11a681a1c5519b442ff7f1f561 + +COPY --from=build /out/oauth2-proxy /bin/oauth2-proxy + +USER 65532 diff --git a/packages/phlo-oauth2-proxy/src/phlo_oauth2_proxy/service.yaml b/packages/phlo-oauth2-proxy/src/phlo_oauth2_proxy/service.yaml index cfc9573a7eb..5aba02af099 100644 --- a/packages/phlo-oauth2-proxy/src/phlo_oauth2_proxy/service.yaml +++ b/packages/phlo-oauth2-proxy/src/phlo_oauth2_proxy/service.yaml @@ -4,7 +4,10 @@ category: auth default: false profile: proxy -image: quay.io/oauth2-proxy/oauth2-proxy:v7.7.1 +image: ghcr.io/phlohouse/phlo-oauth2-proxy:v7.15.3-grpc1.82.1 +build: + context: . + dockerfile: oauth2-proxy/Dockerfile compose: restart: unless-stopped @@ -61,3 +64,7 @@ env_vars: OAUTH2_PROXY_EMAIL_DOMAINS: default: "*" description: Comma-separated list of allowed email domains (use * for any) + +files: + - source: Dockerfile + dest: oauth2-proxy/Dockerfile diff --git a/packages/phlo-oauth2-proxy/tests/test_oauth2_proxy_plugin.py b/packages/phlo-oauth2-proxy/tests/test_oauth2_proxy_plugin.py index 51a4c501b58..9ec56e39da6 100644 --- a/packages/phlo-oauth2-proxy/tests/test_oauth2_proxy_plugin.py +++ b/packages/phlo-oauth2-proxy/tests/test_oauth2_proxy_plugin.py @@ -38,5 +38,5 @@ def test_oauth2_proxy_image_pinned(): plugin = Oauth2ProxyServicePlugin() defn = plugin.service_definition - assert "oauth2-proxy" in defn["image"] - assert ":" in defn["image"], "Image must be version-pinned" + assert defn["image"] == "ghcr.io/phlohouse/phlo-oauth2-proxy:v7.15.3-grpc1.82.1" + assert defn["build"]["dockerfile"] == "oauth2-proxy/Dockerfile" diff --git a/packages/phlo-observatory/src/phlo_observatory/Dockerfile b/packages/phlo-observatory/src/phlo_observatory/Dockerfile index b6b3c34cea9..dd07b1cde06 100644 --- a/packages/phlo-observatory/src/phlo_observatory/Dockerfile +++ b/packages/phlo-observatory/src/phlo_observatory/Dockerfile @@ -21,7 +21,8 @@ FROM node:24-alpine AS runner WORKDIR /app # Required for service discovery/status code paths that run `docker ps`. -RUN apk add --no-cache docker-cli +RUN apk upgrade --no-cache \ + && apk add --no-cache docker-cli=29.5.3-r0 # Install only the runtime dependency graph. The build-only Vite, Nitro, and # scaffolding packages remain in the builder stage. @@ -31,6 +32,9 @@ RUN npm ci --omit=dev --ignore-scripts # Copy built application COPY --from=builder /app/.output /app/.output +# npm is a build tool; its bundled CLI dependencies are not needed at runtime. +RUN rm -rf /usr/local/lib/node_modules/npm /usr/local/bin/npm /usr/local/bin/npx + # Set environment ENV NODE_ENV=production ENV HOST=0.0.0.0 diff --git a/packages/phlo-observatory/src/phlo_observatory/package-lock.json b/packages/phlo-observatory/src/phlo_observatory/package-lock.json index 3c6ff0f75ae..1089c071859 100644 --- a/packages/phlo-observatory/src/phlo_observatory/package-lock.json +++ b/packages/phlo-observatory/src/phlo_observatory/package-lock.json @@ -5022,45 +5022,6 @@ "path-browserify": "^1.0.1" } }, - "node_modules/@ts-morph/common/node_modules/balanced-match": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", - "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", - "dev": true, - "license": "MIT", - "engines": { - "node": "18 || 20 || >=22" - } - }, - "node_modules/@ts-morph/common/node_modules/brace-expansion": { - "version": "5.0.7", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.7.tgz", - "integrity": "sha512-7oFy703dxfY3/NLxC1fh2SUCQ0H9rmAY+5EpDVfXjUTTs+HEwR2nYaqLv+GWcTsumwxPfiz6CzCNkwXwBUwqCA==", - "dev": true, - "license": "MIT", - "dependencies": { - "balanced-match": "^4.0.2" - }, - "engines": { - "node": "18 || 20 || >=22" - } - }, - "node_modules/@ts-morph/common/node_modules/minimatch": { - "version": "10.2.5", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.5.tgz", - "integrity": "sha512-MULkVLfKGYDFYejP07QOurDLLQpcjk7Fw+7jXS2R2czRQzR56yHRveU5NDJEOviH+hETZKSkIk5c+T23GjFUMg==", - "dev": true, - "license": "BlueOak-1.0.0", - "dependencies": { - "brace-expansion": "^5.0.5" - }, - "engines": { - "node": "18 || 20 || >=22" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, "node_modules/@tybys/wasm-util": { "version": "0.10.3", "resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.3.tgz", @@ -5438,32 +5399,6 @@ "typescript": ">=4.8.4 <6.0.0" } }, - "node_modules/@typescript-eslint/typescript-estree/node_modules/brace-expansion": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.2.tgz", - "integrity": "sha512-w5JZcKgdhDOgOwm8H+KgbosopHMuGcl6qbulwjtz3SM7I7P3yW1eAjzMPLrIE+NQ9vjgANKHWeMHnrT0OXW1oA==", - "dev": true, - "license": "MIT", - "dependencies": { - "balanced-match": "^1.0.0" - } - }, - "node_modules/@typescript-eslint/typescript-estree/node_modules/minimatch": { - "version": "9.0.9", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.9.tgz", - "integrity": "sha512-OBwBN9AL4dqmETlpS2zasx+vTeWclWzkblfZk7KTA5j3jeOONz/tRCnZomUyvNg83wL5Zv9Ss6HMJXAgL8R2Yg==", - "dev": true, - "license": "ISC", - "dependencies": { - "brace-expansion": "^2.0.2" - }, - "engines": { - "node": ">=16 || 14 >=14.17" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, "node_modules/@typescript-eslint/typescript-estree/node_modules/semver": { "version": "7.7.3", "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.3.tgz", @@ -6189,11 +6124,14 @@ } }, "node_modules/balanced-match": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", - "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", + "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", "dev": true, - "license": "MIT" + "license": "MIT", + "engines": { + "node": "18 || 20 || >=22" + } }, "node_modules/baseline-browser-mapping": { "version": "2.9.4", @@ -6255,15 +6193,16 @@ } }, "node_modules/brace-expansion": { - "version": "1.1.16", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.16.tgz", - "integrity": "sha512-IDw48K2/2kRkg9LdJxurvq3lV3aBgq0REY89duEqFRthjlPdXHKMj7EnQOXVckxzgisinf3nHfrcE2FufFLXMw==", + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.8.tgz", + "integrity": "sha512-JZyDyq3D4AUifKTPOB7DELf6XsB3WdPuNxCtob1vFXPsSXhdAiHBWJ/tJ8HAc9aH84BK+5JFZLNkJKx3G9kzQg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { - "balanced-match": "^1.0.0", - "concat-map": "0.0.1" + "balanced-match": "^4.0.2" + }, + "engines": { + "node": "20 || >=22" } }, "node_modules/braces": { @@ -6567,14 +6506,6 @@ "node": ">= 12.0.0" } }, - "node_modules/concat-map": { - "version": "0.0.1", - "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", - "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", - "dev": true, - "license": "MIT", - "peer": true - }, "node_modules/consola": { "version": "3.4.2", "resolved": "https://registry.npmjs.org/consola/-/consola-3.4.2.tgz", @@ -7507,45 +7438,6 @@ } } }, - "node_modules/eslint-plugin-import-x/node_modules/balanced-match": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", - "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", - "dev": true, - "license": "MIT", - "engines": { - "node": "18 || 20 || >=22" - } - }, - "node_modules/eslint-plugin-import-x/node_modules/brace-expansion": { - "version": "5.0.7", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.7.tgz", - "integrity": "sha512-7oFy703dxfY3/NLxC1fh2SUCQ0H9rmAY+5EpDVfXjUTTs+HEwR2nYaqLv+GWcTsumwxPfiz6CzCNkwXwBUwqCA==", - "dev": true, - "license": "MIT", - "dependencies": { - "balanced-match": "^4.0.2" - }, - "engines": { - "node": "18 || 20 || >=22" - } - }, - "node_modules/eslint-plugin-import-x/node_modules/minimatch": { - "version": "10.2.5", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.5.tgz", - "integrity": "sha512-MULkVLfKGYDFYejP07QOurDLLQpcjk7Fw+7jXS2R2czRQzR56yHRveU5NDJEOviH+hETZKSkIk5c+T23GjFUMg==", - "dev": true, - "license": "BlueOak-1.0.0", - "dependencies": { - "brace-expansion": "^5.0.5" - }, - "engines": { - "node": "18 || 20 || >=22" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, "node_modules/eslint-plugin-import-x/node_modules/semver": { "version": "7.7.3", "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.3.tgz", @@ -9620,17 +9512,19 @@ } }, "node_modules/minimatch": { - "version": "3.1.5", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.5.tgz", - "integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==", + "version": "10.2.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.5.tgz", + "integrity": "sha512-MULkVLfKGYDFYejP07QOurDLLQpcjk7Fw+7jXS2R2czRQzR56yHRveU5NDJEOviH+hETZKSkIk5c+T23GjFUMg==", "dev": true, - "license": "ISC", - "peer": true, + "license": "BlueOak-1.0.0", "dependencies": { - "brace-expansion": "^1.1.7" + "brace-expansion": "^5.0.5" }, "engines": { - "node": "*" + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" } }, "node_modules/minimist": { diff --git a/packages/phlo-observatory/src/phlo_observatory/package.json b/packages/phlo-observatory/src/phlo_observatory/package.json index b3b2caa9452..a40fe4923b5 100644 --- a/packages/phlo-observatory/src/phlo_observatory/package.json +++ b/packages/phlo-observatory/src/phlo_observatory/package.json @@ -64,5 +64,8 @@ "vite": "7.3.6", "vite-tsconfig-paths": "^6.1.1", "vitest": "^4.1.4" + }, + "overrides": { + "minimatch": "10.2.5" } } diff --git a/packages/phlo-observatory/src/phlo_observatory/service.yaml b/packages/phlo-observatory/src/phlo_observatory/service.yaml index fb6025b40eb..ae662436ec4 100644 --- a/packages/phlo-observatory/src/phlo_observatory/service.yaml +++ b/packages/phlo-observatory/src/phlo_observatory/service.yaml @@ -4,6 +4,7 @@ category: orchestration default: false profile: api +image: ghcr.io/phlohouse/phlo-observatory:0.7.0 build: context: source dockerfile: Dockerfile diff --git a/packages/phlo-openmetadata/pyproject.toml b/packages/phlo-openmetadata/pyproject.toml index 12207cba5b4..5b4e5c00288 100644 --- a/packages/phlo-openmetadata/pyproject.toml +++ b/packages/phlo-openmetadata/pyproject.toml @@ -59,6 +59,10 @@ include-package-data = true [tool.setuptools.package-data] phlo_openmetadata = [ + "Dockerfile", + "db.Dockerfile", + "es.Dockerfile", + "es-libraries.sha256", "service.yaml", "openmetadata-setup.yaml", "openmetadata-mysql-setup.yaml", diff --git a/packages/phlo-openmetadata/src/phlo_openmetadata/Dockerfile b/packages/phlo-openmetadata/src/phlo_openmetadata/Dockerfile new file mode 100644 index 00000000000..ee41f4f0044 --- /dev/null +++ b/packages/phlo-openmetadata/src/phlo_openmetadata/Dockerfile @@ -0,0 +1,47 @@ +FROM alpine:3.24.1@sha256:28bd5fe8b56d1bd048e5babf5b10710ebe0bae67db86916198a6eec434943f8b AS patches + +SHELL ["/bin/ash", "-o", "pipefail", "-c"] +RUN apk add --no-cache curl=8.21.0-r0 +WORKDIR /patches +RUN set -eux; \ + for artifact in \ + netty-buffer \ + netty-codec \ + netty-codec-dns \ + netty-codec-http \ + netty-codec-http2 \ + netty-codec-socks \ + netty-common \ + netty-handler \ + netty-handler-proxy \ + netty-resolver \ + netty-resolver-dns \ + netty-resolver-dns-classes-macos \ + netty-transport \ + netty-transport-classes-epoll \ + netty-transport-classes-kqueue \ + netty-transport-native-unix-common; do \ + file="${artifact}-4.1.136.Final.jar"; \ + base="https://repo1.maven.org/maven2/io/netty/${artifact}/4.1.136.Final/${file}"; \ + curl --fail --location --silent --show-error "$base" --output "$file"; \ + curl --fail --location --silent --show-error "$base.sha256" --output "$file.sha256"; \ + echo "$(cat "$file.sha256") $file" | sha256sum -c -; \ + done +RUN curl --fail --location --silent --show-error \ + https://repo1.maven.org/maven2/org/eclipse/jetty/jetty-security/12.1.10/jetty-security-12.1.10.jar \ + --output jetty-security-12.1.10.jar && \ + echo "2f34b7895cec4e3547a1b52e12e7e92b3f3a110bf3c17637f8743ca3f4e42f0c jetty-security-12.1.10.jar" | sha256sum -c - +RUN curl --fail --location --silent --show-error \ + https://repo1.maven.org/maven2/org/postgresql/postgresql/42.7.13/postgresql-42.7.13.jar \ + --output postgresql-42.7.13.jar && \ + echo "6e0e4cc2d8cae902084f8a2b18728b073a6fd9d1f87c9d8bff8f298c18185b93 postgresql-42.7.13.jar" | sha256sum -c - + +FROM docker.io/openmetadata/server:1.13.1@sha256:eaa318584c52d4a492a2c56c95818b5564c6ea28b2e9695ac532c856b2c61bc9 + +USER root +RUN rm /opt/openmetadata/libs/netty-*-4.1.135.Final.jar \ + /opt/openmetadata/libs/jetty-security-12.1.1.jar \ + /opt/openmetadata/libs/postgresql-42.7.11.jar +COPY --from=patches --chown=openmetadata:openmetadata /patches/*.jar /opt/openmetadata/libs/ + +USER openmetadata diff --git a/packages/phlo-openmetadata/src/phlo_openmetadata/db.Dockerfile b/packages/phlo-openmetadata/src/phlo_openmetadata/db.Dockerfile new file mode 100644 index 00000000000..1c487771b9e --- /dev/null +++ b/packages/phlo-openmetadata/src/phlo_openmetadata/db.Dockerfile @@ -0,0 +1,15 @@ +FROM docker.io/openmetadata/db:1.13.1@sha256:6659446dba183f1e9364602839dd999c06a83f7d2e905d1c3fb22a74f3e27288 + +RUN microdnf update -y --disablerepo=mysql-tools-community && \ + python3.9 -m pip install --no-cache-dir --upgrade \ + --target=/usr/lib/mysqlsh/lib/python3.9/site-packages \ + certifi==2026.7.22 \ + cryptography==49.0.0 \ + pyOpenSSL==26.3.0 && \ + rm -rf \ + /usr/lib/mysqlsh/lib/python3.9/site-packages/certifi-2022.9.24.dist-info \ + /usr/lib/mysqlsh/lib/python3.9/site-packages/cryptography-37.0.2.dist-info \ + /usr/lib/mysqlsh/lib/python3.9/site-packages/pyOpenSSL-22.0.0.dist-info && \ + microdnf clean all + +USER mysql diff --git a/packages/phlo-openmetadata/src/phlo_openmetadata/es-libraries.sha256 b/packages/phlo-openmetadata/src/phlo_openmetadata/es-libraries.sha256 new file mode 100644 index 00000000000..43ff1e1bcf1 --- /dev/null +++ b/packages/phlo-openmetadata/src/phlo_openmetadata/es-libraries.sha256 @@ -0,0 +1,23 @@ +b21889d334ff04b8a3efe54473613ec79e7eed080a28e98abbc3997b46901a54 jackson-annotations-2.18.8.jar +ab865e39f01403598748b090a3bf528616e701913a71afc37a227daa0fabb4ae jackson-core-2.18.8.jar +06ea5950905263fac3e1730de6a21a023151626af6bb3cb099f88644a0fa04f0 jackson-databind-2.18.8.jar +af5cdf1b3e6ab98e8178942834e1f903107ab0c5b21a0da4c7e1f49ea3a5a3f5 jackson-dataformat-cbor-2.18.8.jar +e2670aac4e3159e289710f547962ad8f2f68489bf7ccb9fffcac714f2a88bc05 jackson-dataformat-xml-2.18.8.jar +299010ea6dd1496abc4e51659904cdf51bb15f1f8ee6d2169b9348f74d24e2de jackson-datatype-jsr310-2.18.8.jar +f690fbd1e9a7758c9e896fe95c3ec22f2e0addf94ad83b0e18b652af12afc6ea jackson-module-jaxb-annotations-2.18.8.jar +e35d1df7232568ed4b81d9a9ed94a0159b3ff6cf1915cc21eef5cb55d7b157a2 commons-io-2.14.0.jar +73bca52833f7ad9571f9c054964007e54f0abcc60e0703a003d1cfa2080336bb nimbus-jose-jwt-9.37.2.jar +8540247fad9e06baefa8fb45eb313802d019f485f14300e0f9d6b556ed88e753 protobuf-java-3.25.5.jar +c88f13fc41156fde5df918c7b240038dfbe97ac57576163f208630391789b5d5 netty-buffer-4.1.136.Final.jar +2de4fc13005c7740b46427a47ee04265a19779c147d53e583f441889b1148159 netty-codec-4.1.136.Final.jar +89bddc4ec42756c41a0195a50a1323a324836162b6712a843c3de5c29096c93b netty-codec-dns-4.1.136.Final.jar +ffd1e1b19a533bc6e47ef2cbc1290374ff4a7cb53a280defa3df392538214948 netty-codec-http-4.1.136.Final.jar +14f67ae095c056b062aa0ee2c7b01f6b78004cf3620a6e9061bcb637f079387a netty-codec-http2-4.1.136.Final.jar +341f47dbaac667a6e7e6cad4114218025f9755ced641bb0740fb69e34d29b421 netty-codec-socks-4.1.136.Final.jar +e2f73be7ec359b46583ad875521137000eff520bafd320e730846ec9974f3be6 netty-common-4.1.136.Final.jar +54bb1a59f46a3aefd117942e6acee09e555b756776bad731fc06159d89c2da28 netty-handler-4.1.136.Final.jar +47400551f0444dea33629ee0c52d4e30856b2ddf00b12adf1881902957e74cf2 netty-handler-proxy-4.1.136.Final.jar +e64972fec474f5b9bd086b738154d190a923e82c4923ac550aff4f5ea9e98600 netty-resolver-4.1.136.Final.jar +9c05a9b18b5d54fcc1569c48b93958019cf3dde5ae7011f8a46451ee05e7daff netty-resolver-dns-4.1.136.Final.jar +b92881ea925721ed42fee5122a42b8bce4b84da737dbc3ca2d84dfaca52c28b6 netty-transport-4.1.136.Final.jar +7e014c9b13defd9d254d4e5a5edd8ab6dde17533e1152f99699a6b28b2967a8a netty-transport-native-unix-common-4.1.136.Final.jar diff --git a/packages/phlo-openmetadata/src/phlo_openmetadata/es.Dockerfile b/packages/phlo-openmetadata/src/phlo_openmetadata/es.Dockerfile new file mode 100644 index 00000000000..e3a2165a6ff --- /dev/null +++ b/packages/phlo-openmetadata/src/phlo_openmetadata/es.Dockerfile @@ -0,0 +1,112 @@ +FROM alpine:3.23@sha256:fd791d74b68913cbb027c6546007b3f0d3bc45125f797758156952bc2d6daf40 AS patches + +RUN apk add --no-cache curl=8.20.0-r0 +WORKDIR /patches +RUN for artifact in \ + netty-buffer netty-codec netty-codec-dns netty-codec-http \ + netty-codec-http2 netty-codec-socks netty-common netty-handler \ + netty-handler-proxy netty-resolver netty-resolver-dns netty-transport \ + netty-transport-native-unix-common; do \ + curl --fail --silent --show-error --location \ + "https://repo1.maven.org/maven2/io/netty/${artifact}/4.1.136.Final/${artifact}-4.1.136.Final.jar" \ + --output "${artifact}-4.1.136.Final.jar"; \ + done && \ + curl --fail --silent --show-error --location \ + https://repo1.maven.org/maven2/com/fasterxml/jackson/core/jackson-annotations/2.18.8/jackson-annotations-2.18.8.jar \ + --output jackson-annotations-2.18.8.jar && \ + curl --fail --silent --show-error --location \ + https://repo1.maven.org/maven2/com/fasterxml/jackson/core/jackson-core/2.18.8/jackson-core-2.18.8.jar \ + --output jackson-core-2.18.8.jar && \ + curl --fail --silent --show-error --location \ + https://repo1.maven.org/maven2/com/fasterxml/jackson/core/jackson-databind/2.18.8/jackson-databind-2.18.8.jar \ + --output jackson-databind-2.18.8.jar && \ + curl --fail --silent --show-error --location \ + https://repo1.maven.org/maven2/com/fasterxml/jackson/dataformat/jackson-dataformat-cbor/2.18.8/jackson-dataformat-cbor-2.18.8.jar \ + --output jackson-dataformat-cbor-2.18.8.jar && \ + curl --fail --silent --show-error --location \ + https://repo1.maven.org/maven2/com/fasterxml/jackson/dataformat/jackson-dataformat-xml/2.18.8/jackson-dataformat-xml-2.18.8.jar \ + --output jackson-dataformat-xml-2.18.8.jar && \ + curl --fail --silent --show-error --location \ + https://repo1.maven.org/maven2/com/fasterxml/jackson/datatype/jackson-datatype-jsr310/2.18.8/jackson-datatype-jsr310-2.18.8.jar \ + --output jackson-datatype-jsr310-2.18.8.jar && \ + curl --fail --silent --show-error --location \ + https://repo1.maven.org/maven2/com/fasterxml/jackson/module/jackson-module-jaxb-annotations/2.18.8/jackson-module-jaxb-annotations-2.18.8.jar \ + --output jackson-module-jaxb-annotations-2.18.8.jar && \ + curl --fail --silent --show-error --location \ + https://repo1.maven.org/maven2/com/google/protobuf/protobuf-java/3.25.5/protobuf-java-3.25.5.jar \ + --output protobuf-java-3.25.5.jar && \ + curl --fail --silent --show-error --location \ + https://repo1.maven.org/maven2/com/nimbusds/nimbus-jose-jwt/9.37.2/nimbus-jose-jwt-9.37.2.jar \ + --output nimbus-jose-jwt-9.37.2.jar && \ + curl --fail --silent --show-error --location \ + https://repo1.maven.org/maven2/commons-io/commons-io/2.14.0/commons-io-2.14.0.jar \ + --output commons-io-2.14.0.jar +COPY openmetadata-elasticsearch/libraries.sha256 libraries.sha256 +RUN sha256sum -c libraries.sha256 + +FROM docker.elastic.co/elasticsearch/elasticsearch:8.11.4@sha256:8425bc28027fd667d9a29cde58bed4050a64a854d973d8d1ad4152ecec52bfdb + +COPY --from=patches /patches/commons-io-2.14.0.jar /usr/share/elasticsearch/modules/ingest-attachment/commons-io-2.11.0.jar +COPY --from=patches /patches/protobuf-java-3.25.5.jar /usr/share/elasticsearch/modules/repository-gcs/protobuf-java-3.21.9.jar +COPY --from=patches /patches/protobuf-java-3.25.5.jar /usr/share/elasticsearch/modules/vector-tile/protobuf-java-3.21.9.jar +COPY --from=patches /patches/nimbus-jose-jwt-9.37.2.jar /usr/share/elasticsearch/modules/x-pack-security/nimbus-jose-jwt-9.23.jar + +COPY --from=patches /patches/jackson-annotations-2.18.8.jar /usr/share/elasticsearch/modules/ingest-geoip/jackson-annotations-2.15.0.jar +COPY --from=patches /patches/jackson-core-2.18.8.jar /usr/share/elasticsearch/modules/ingest-geoip/jackson-core-2.15.0.jar +COPY --from=patches /patches/jackson-databind-2.18.8.jar /usr/share/elasticsearch/modules/ingest-geoip/jackson-databind-2.15.0.jar +COPY --from=patches /patches/jackson-core-2.18.8.jar /usr/share/elasticsearch/modules/legacy-geo/jackson-core-2.15.0.jar +COPY --from=patches /patches/jackson-annotations-2.18.8.jar /usr/share/elasticsearch/modules/repository-azure/jackson-annotations-2.13.4.jar +COPY --from=patches /patches/jackson-core-2.18.8.jar /usr/share/elasticsearch/modules/repository-azure/jackson-core-2.13.4.jar +COPY --from=patches /patches/jackson-databind-2.18.8.jar /usr/share/elasticsearch/modules/repository-azure/jackson-databind-2.13.4.2.jar +COPY --from=patches /patches/jackson-dataformat-xml-2.18.8.jar /usr/share/elasticsearch/modules/repository-azure/jackson-dataformat-xml-2.13.4.jar +COPY --from=patches /patches/jackson-datatype-jsr310-2.18.8.jar /usr/share/elasticsearch/modules/repository-azure/jackson-datatype-jsr310-2.13.4.jar +COPY --from=patches /patches/jackson-module-jaxb-annotations-2.18.8.jar /usr/share/elasticsearch/modules/repository-azure/jackson-module-jaxb-annotations-2.13.4.jar +COPY --from=patches /patches/jackson-core-2.18.8.jar /usr/share/elasticsearch/modules/repository-gcs/jackson-core-2.15.0.jar +COPY --from=patches /patches/jackson-annotations-2.18.8.jar /usr/share/elasticsearch/modules/repository-s3/jackson-annotations-2.15.0.jar +COPY --from=patches /patches/jackson-core-2.18.8.jar /usr/share/elasticsearch/modules/repository-s3/jackson-core-2.15.0.jar +COPY --from=patches /patches/jackson-databind-2.18.8.jar /usr/share/elasticsearch/modules/repository-s3/jackson-databind-2.15.0.jar +COPY --from=patches /patches/jackson-dataformat-cbor-2.18.8.jar /usr/share/elasticsearch/modules/repository-s3/jackson-dataformat-cbor-2.15.0.jar +COPY --from=patches /patches/jackson-annotations-2.18.8.jar /usr/share/elasticsearch/modules/x-pack-ent-search/jackson-annotations-2.15.0.jar +COPY --from=patches /patches/jackson-core-2.18.8.jar /usr/share/elasticsearch/modules/x-pack-ent-search/jackson-core-2.15.0.jar +COPY --from=patches /patches/jackson-databind-2.18.8.jar /usr/share/elasticsearch/modules/x-pack-ent-search/jackson-databind-2.15.0.jar +COPY --from=patches /patches/jackson-core-2.18.8.jar /usr/share/elasticsearch/modules/x-pack-monitoring/jackson-core-2.15.0.jar +COPY --from=patches /patches/jackson-core-2.18.8.jar /usr/share/elasticsearch/modules/x-pack-sql/jackson-core-2.15.0.jar +COPY --from=patches /patches/jackson-dataformat-cbor-2.18.8.jar /usr/share/elasticsearch/modules/x-pack-sql/jackson-dataformat-cbor-2.15.0.jar + +COPY --from=patches /patches/netty-buffer-4.1.136.Final.jar /usr/share/elasticsearch/modules/repository-azure/netty-buffer-4.1.94.Final.jar +COPY --from=patches /patches/netty-codec-4.1.136.Final.jar /usr/share/elasticsearch/modules/repository-azure/netty-codec-4.1.94.Final.jar +COPY --from=patches /patches/netty-codec-dns-4.1.136.Final.jar /usr/share/elasticsearch/modules/repository-azure/netty-codec-dns-4.1.94.Final.jar +COPY --from=patches /patches/netty-codec-http-4.1.136.Final.jar /usr/share/elasticsearch/modules/repository-azure/netty-codec-http-4.1.94.Final.jar +COPY --from=patches /patches/netty-codec-http2-4.1.136.Final.jar /usr/share/elasticsearch/modules/repository-azure/netty-codec-http2-4.1.94.Final.jar +COPY --from=patches /patches/netty-codec-socks-4.1.136.Final.jar /usr/share/elasticsearch/modules/repository-azure/netty-codec-socks-4.1.94.Final.jar +COPY --from=patches /patches/netty-common-4.1.136.Final.jar /usr/share/elasticsearch/modules/repository-azure/netty-common-4.1.94.Final.jar +COPY --from=patches /patches/netty-handler-4.1.136.Final.jar /usr/share/elasticsearch/modules/repository-azure/netty-handler-4.1.94.Final.jar +COPY --from=patches /patches/netty-handler-proxy-4.1.136.Final.jar /usr/share/elasticsearch/modules/repository-azure/netty-handler-proxy-4.1.94.Final.jar +COPY --from=patches /patches/netty-resolver-4.1.136.Final.jar /usr/share/elasticsearch/modules/repository-azure/netty-resolver-4.1.94.Final.jar +COPY --from=patches /patches/netty-resolver-dns-4.1.136.Final.jar /usr/share/elasticsearch/modules/repository-azure/netty-resolver-dns-4.1.94.Final.jar +COPY --from=patches /patches/netty-transport-4.1.136.Final.jar /usr/share/elasticsearch/modules/repository-azure/netty-transport-4.1.94.Final.jar +COPY --from=patches /patches/netty-transport-native-unix-common-4.1.136.Final.jar /usr/share/elasticsearch/modules/repository-azure/netty-transport-native-unix-common-4.1.94.Final.jar + +COPY --from=patches /patches/netty-buffer-4.1.136.Final.jar /usr/share/elasticsearch/modules/transport-netty4/netty-buffer-4.1.94.Final.jar +COPY --from=patches /patches/netty-codec-4.1.136.Final.jar /usr/share/elasticsearch/modules/transport-netty4/netty-codec-4.1.94.Final.jar +COPY --from=patches /patches/netty-codec-http-4.1.136.Final.jar /usr/share/elasticsearch/modules/transport-netty4/netty-codec-http-4.1.94.Final.jar +COPY --from=patches /patches/netty-common-4.1.136.Final.jar /usr/share/elasticsearch/modules/transport-netty4/netty-common-4.1.94.Final.jar +COPY --from=patches /patches/netty-handler-4.1.136.Final.jar /usr/share/elasticsearch/modules/transport-netty4/netty-handler-4.1.94.Final.jar +COPY --from=patches /patches/netty-resolver-4.1.136.Final.jar /usr/share/elasticsearch/modules/transport-netty4/netty-resolver-4.1.94.Final.jar +COPY --from=patches /patches/netty-transport-4.1.136.Final.jar /usr/share/elasticsearch/modules/transport-netty4/netty-transport-4.1.94.Final.jar +COPY --from=patches /patches/netty-transport-native-unix-common-4.1.136.Final.jar /usr/share/elasticsearch/modules/transport-netty4/netty-transport-native-unix-common-4.1.94.Final.jar + +COPY --from=patches /patches/netty-buffer-4.1.136.Final.jar /usr/share/elasticsearch/modules/x-pack-security/netty-buffer-4.1.94.Final.jar +COPY --from=patches /patches/netty-codec-4.1.136.Final.jar /usr/share/elasticsearch/modules/x-pack-security/netty-codec-4.1.94.Final.jar +COPY --from=patches /patches/netty-codec-http-4.1.136.Final.jar /usr/share/elasticsearch/modules/x-pack-security/netty-codec-http-4.1.94.Final.jar +COPY --from=patches /patches/netty-common-4.1.136.Final.jar /usr/share/elasticsearch/modules/x-pack-security/netty-common-4.1.94.Final.jar +COPY --from=patches /patches/netty-handler-4.1.136.Final.jar /usr/share/elasticsearch/modules/x-pack-security/netty-handler-4.1.94.Final.jar +COPY --from=patches /patches/netty-resolver-4.1.136.Final.jar /usr/share/elasticsearch/modules/x-pack-security/netty-resolver-4.1.94.Final.jar +COPY --from=patches /patches/netty-transport-4.1.136.Final.jar /usr/share/elasticsearch/modules/x-pack-security/netty-transport-4.1.94.Final.jar +COPY --from=patches /patches/netty-transport-native-unix-common-4.1.136.Final.jar /usr/share/elasticsearch/modules/x-pack-security/netty-transport-native-unix-common-4.1.94.Final.jar + +USER "0" +RUN find /usr/share/elasticsearch/modules -name 'jackson-core-2.15.0.jar' \ + -exec sh -c 'for path do mv "$path" "${path%/*}/jackson-core-2.18.8.jar"; done' sh {} + + +USER "1000" diff --git a/packages/phlo-openmetadata/src/phlo_openmetadata/openmetadata-elasticsearch-setup.yaml b/packages/phlo-openmetadata/src/phlo_openmetadata/openmetadata-elasticsearch-setup.yaml index adc52fc5c51..4c6eef791d3 100644 --- a/packages/phlo-openmetadata/src/phlo_openmetadata/openmetadata-elasticsearch-setup.yaml +++ b/packages/phlo-openmetadata/src/phlo_openmetadata/openmetadata-elasticsearch-setup.yaml @@ -4,7 +4,11 @@ category: catalog default: false profile: openmetadata -image: docker.elastic.co/elasticsearch/elasticsearch:8.11.4 +image: ghcr.io/phlohouse/phlo-openmetadata-elasticsearch:8.11.4-java-patches + +build: + context: . + dockerfile: openmetadata-elasticsearch/Dockerfile compose: restart: unless-stopped @@ -29,3 +33,9 @@ env_vars: OPENMETADATA_ES_JAVA_OPTS: default: "-Xms256m -Xmx256m" description: JVM heap settings for the bundled OpenMetadata Elasticsearch service + +files: + - source: es.Dockerfile + dest: openmetadata-elasticsearch/Dockerfile + - source: es-libraries.sha256 + dest: openmetadata-elasticsearch/libraries.sha256 diff --git a/packages/phlo-openmetadata/src/phlo_openmetadata/openmetadata-mysql-setup.yaml b/packages/phlo-openmetadata/src/phlo_openmetadata/openmetadata-mysql-setup.yaml index 04cb9ed82e3..32531c97549 100644 --- a/packages/phlo-openmetadata/src/phlo_openmetadata/openmetadata-mysql-setup.yaml +++ b/packages/phlo-openmetadata/src/phlo_openmetadata/openmetadata-mysql-setup.yaml @@ -4,7 +4,10 @@ category: catalog default: false profile: openmetadata -image: docker.getcollate.io/openmetadata/db:${OPENMETADATA_VERSION:-1.9.7} +image: ghcr.io/phlohouse/phlo-openmetadata-db:1.13.1-ol8.10 +build: + context: . + dockerfile: openmetadata-mysql/Dockerfile compose: restart: unless-stopped @@ -25,3 +28,7 @@ compose: interval: 15s timeout: 10s retries: 10 + +files: + - source: db.Dockerfile + dest: openmetadata-mysql/Dockerfile diff --git a/packages/phlo-openmetadata/src/phlo_openmetadata/openmetadata-setup.yaml b/packages/phlo-openmetadata/src/phlo_openmetadata/openmetadata-setup.yaml index bac66646696..509bab5f3f0 100644 --- a/packages/phlo-openmetadata/src/phlo_openmetadata/openmetadata-setup.yaml +++ b/packages/phlo-openmetadata/src/phlo_openmetadata/openmetadata-setup.yaml @@ -8,7 +8,13 @@ depends_on: - openmetadata-mysql - openmetadata-elasticsearch -image: docker.getcollate.io/openmetadata/server:${OPENMETADATA_VERSION:-1.9.7} +image: ghcr.io/phlohouse/phlo-openmetadata:1.13.1-java-patches +build: + context: . + dockerfile: openmetadata/Dockerfile +files: + - source: Dockerfile + dest: openmetadata/Dockerfile compose: restart: "no" diff --git a/packages/phlo-openmetadata/src/phlo_openmetadata/service.yaml b/packages/phlo-openmetadata/src/phlo_openmetadata/service.yaml index 802d889e2e2..8db44c02682 100644 --- a/packages/phlo-openmetadata/src/phlo_openmetadata/service.yaml +++ b/packages/phlo-openmetadata/src/phlo_openmetadata/service.yaml @@ -9,7 +9,10 @@ depends_on: - openmetadata-elasticsearch - openmetadata-setup -image: docker.getcollate.io/openmetadata/server:${OPENMETADATA_VERSION:-1.9.7} +image: ghcr.io/phlohouse/phlo-openmetadata:1.13.1-java-patches +build: + context: . + dockerfile: openmetadata/Dockerfile compose: restart: unless-stopped @@ -50,9 +53,6 @@ compose: retries: 10 env_vars: - OPENMETADATA_VERSION: - default: "1.9.7" - description: OpenMetadata version OPENMETADATA_PORT: default: 8585 description: OpenMetadata API port @@ -71,3 +71,7 @@ env_vars: OPENMETADATA_PASSWORD: default: admin description: OpenMetadata admin password + +files: + - source: Dockerfile + dest: openmetadata/Dockerfile diff --git a/packages/phlo-openmetadata/tests/test_service_plugin.py b/packages/phlo-openmetadata/tests/test_service_plugin.py new file mode 100644 index 00000000000..3167b8b6fb8 --- /dev/null +++ b/packages/phlo-openmetadata/tests/test_service_plugin.py @@ -0,0 +1,70 @@ +"""OpenMetadata generated service contracts.""" + +from importlib import resources + +import yaml + +from phlo_openmetadata.plugin import OpenMetadataServicePlugin + + +def test_openmetadata_builds_a_patched_stable_server_image() -> None: + definition = OpenMetadataServicePlugin().service_definition + dockerfile = resources.files("phlo_openmetadata").joinpath("Dockerfile").read_text() + + assert definition["image"] == "ghcr.io/phlohouse/phlo-openmetadata:1.13.1-java-patches" + assert definition["build"] == { + "context": ".", + "dockerfile": "openmetadata/Dockerfile", + } + assert "FROM docker.io/openmetadata/server:1.13.1@sha256:eaa3185" in dockerfile + assert "docker.getcollate.io" not in dockerfile + assert "OPENMETADATA_VERSION" not in definition["env_vars"] + + +def test_openmetadata_mysql_builds_the_updated_stable_database_image() -> None: + raw = resources.files("phlo_openmetadata").joinpath("openmetadata-mysql-setup.yaml").read_text() + definition = yaml.safe_load(raw) + dockerfile = resources.files("phlo_openmetadata").joinpath("db.Dockerfile").read_text() + + assert definition["image"] == "ghcr.io/phlohouse/phlo-openmetadata-db:1.13.1-ol8.10" + assert definition["build"] == { + "context": ".", + "dockerfile": "openmetadata-mysql/Dockerfile", + } + assert "FROM docker.io/openmetadata/db:1.13.1@sha256:6659446" in dockerfile + assert "docker.getcollate.io" not in dockerfile + + +def test_openmetadata_setup_uses_the_patched_server_image() -> None: + raw = resources.files("phlo_openmetadata").joinpath("openmetadata-setup.yaml").read_text() + definition = yaml.safe_load(raw) + + assert definition["image"] == "ghcr.io/phlohouse/phlo-openmetadata:1.13.1-java-patches" + assert definition["build"] == { + "context": ".", + "dockerfile": "openmetadata/Dockerfile", + } + + +def test_openmetadata_elasticsearch_builds_the_patched_compatible_image() -> None: + raw = ( + resources.files("phlo_openmetadata") + .joinpath("openmetadata-elasticsearch-setup.yaml") + .read_text() + ) + definition = yaml.safe_load(raw) + + assert definition["image"] == ( + "ghcr.io/phlohouse/phlo-openmetadata-elasticsearch:8.11.4-java-patches" + ) + assert definition["build"] == { + "context": ".", + "dockerfile": "openmetadata-elasticsearch/Dockerfile", + } + assert { + "source": "es-libraries.sha256", + "dest": "openmetadata-elasticsearch/libraries.sha256", + } in definition["files"] + + dockerfile = resources.files("phlo_openmetadata").joinpath("es.Dockerfile").read_text() + assert "COPY openmetadata-elasticsearch/libraries.sha256 libraries.sha256" in dockerfile diff --git a/packages/phlo-pgweb/pyproject.toml b/packages/phlo-pgweb/pyproject.toml index e2953a036c4..5e19f28b725 100644 --- a/packages/phlo-pgweb/pyproject.toml +++ b/packages/phlo-pgweb/pyproject.toml @@ -42,7 +42,7 @@ target-version = "py311" include-package-data = true [tool.setuptools.package-data] -phlo_pgweb = ["service.yaml"] +phlo_pgweb = ["service.yaml", "Dockerfile"] [tool.setuptools.package-dir] "" = "src" diff --git a/packages/phlo-pgweb/src/phlo_pgweb/Dockerfile b/packages/phlo-pgweb/src/phlo_pgweb/Dockerfile new file mode 100644 index 00000000000..c22256a5672 --- /dev/null +++ b/packages/phlo-pgweb/src/phlo_pgweb/Dockerfile @@ -0,0 +1,23 @@ +FROM golang:1.26.5-alpine3.24@sha256:0178a641fbb4858c5f1b48e34bdaabe0350a330a1b1149aabd498d0699ff5fb2 AS build + +RUN apk add --no-cache git=2.54.0-r0 +WORKDIR /src +RUN git clone --depth 1 --branch v0.17.0 https://github.com/sosedoff/pgweb.git . && \ + git checkout 6b0b0244d1aefd6971999b03481eeeaa4ec7cf55 && \ + go get github.com/sirupsen/logrus@v1.9.3 golang.org/x/crypto@v0.52.0 golang.org/x/net@v0.55.0 && \ + go mod tidy +RUN CGO_ENABLED=0 go build -trimpath -ldflags="-s -w" -o /out/pgweb . + +FROM alpine:3.22.5@sha256:14358309a308569c32bdc37e2e0e9694be33a9d99e68afb0f5ff33cc1f695dce + +RUN apk add --no-cache \ + ca-certificates=20260611-r0 \ + curl=8.14.1-r3 \ + netcat-openbsd=1.229.1-r0 \ + postgresql17-client=17.10-r0 && \ + adduser -D -H -s /sbin/nologin pgweb +COPY --from=build /out/pgweb /usr/bin/pgweb + +USER pgweb +EXPOSE 8081 +ENTRYPOINT ["/usr/bin/pgweb", "--bind=0.0.0.0", "--listen=8081"] diff --git a/packages/phlo-pgweb/src/phlo_pgweb/service.yaml b/packages/phlo-pgweb/src/phlo_pgweb/service.yaml index ab1ec835b63..4c255177861 100644 --- a/packages/phlo-pgweb/src/phlo_pgweb/service.yaml +++ b/packages/phlo-pgweb/src/phlo_pgweb/service.yaml @@ -3,7 +3,10 @@ description: Web-based PostgreSQL database browser category: admin default: true -image: sosedoff/pgweb +image: ghcr.io/phlohouse/phlo-pgweb:0.17.0-security-patches +build: + context: . + dockerfile: pgweb/Dockerfile depends_on: - postgres @@ -21,3 +24,7 @@ env_vars: PGWEB_PORT: default: 8081 description: pgweb UI port + +files: + - source: Dockerfile + dest: pgweb/Dockerfile diff --git a/packages/phlo-pgweb/tests/test_pgweb_plugin.py b/packages/phlo-pgweb/tests/test_pgweb_plugin.py index 527c0efb729..9bb39feb891 100644 --- a/packages/phlo-pgweb/tests/test_pgweb_plugin.py +++ b/packages/phlo-pgweb/tests/test_pgweb_plugin.py @@ -1,5 +1,7 @@ """Tests for Pgweb service plugin.""" +from importlib import resources + from phlo_pgweb.plugin import PgwebServicePlugin @@ -9,6 +11,8 @@ def test_pgweb_service_definition(): defn = plugin.service_definition assert defn["name"] == "pgweb" + assert defn["build"] == {"context": ".", "dockerfile": "pgweb/Dockerfile"} + assert defn["files"] == [{"source": "Dockerfile", "dest": "pgweb/Dockerfile"}] def test_pgweb_plugin_metadata(): @@ -18,3 +22,9 @@ def test_pgweb_plugin_metadata(): assert meta.name == "pgweb" assert "postgres" in meta.tags + + +def test_pgweb_runtime_packages_are_reproducible() -> None: + dockerfile = resources.files("phlo_pgweb").joinpath("Dockerfile").read_text() + + assert "apk upgrade" not in dockerfile diff --git a/packages/phlo-postgres/pyproject.toml b/packages/phlo-postgres/pyproject.toml index 47930683118..505c0d84b68 100644 --- a/packages/phlo-postgres/pyproject.toml +++ b/packages/phlo-postgres/pyproject.toml @@ -52,6 +52,8 @@ include-package-data = true [tool.setuptools.package-data] phlo_postgres = [ + "Dockerfile", + "exporter.Dockerfile", "service.yaml", "exporter_service.yaml", "volume_setup.yaml", diff --git a/packages/phlo-postgres/src/phlo_postgres/Dockerfile b/packages/phlo-postgres/src/phlo_postgres/Dockerfile new file mode 100644 index 00000000000..448e5ad5044 --- /dev/null +++ b/packages/phlo-postgres/src/phlo_postgres/Dockerfile @@ -0,0 +1,14 @@ +FROM golang:1.26.5-alpine3.24@sha256:0178a641fbb4858c5f1b48e34bdaabe0350a330a1b1149aabd498d0699ff5fb2 AS gosu-build + +RUN apk add --no-cache git=2.54.0-r0 +WORKDIR /src +RUN git clone https://github.com/tianon/gosu.git . && \ + git checkout 6456aaa0f3c854d199d0f037f068eb97515b7513 +RUN CGO_ENABLED=0 go build -trimpath -ldflags="-s -w" -o /out/gosu . + +FROM postgres:18.4-alpine3.24@sha256:9a8afca54e7861fd90fab5fdf4c42477a6b1cb7d293595148e674e0a3181de15 + +RUN apk upgrade --no-cache +COPY --from=gosu-build /out/gosu /usr/local/bin/gosu + +USER postgres diff --git a/packages/phlo-postgres/src/phlo_postgres/exporter.Dockerfile b/packages/phlo-postgres/src/phlo_postgres/exporter.Dockerfile new file mode 100644 index 00000000000..1f1ba317e62 --- /dev/null +++ b/packages/phlo-postgres/src/phlo_postgres/exporter.Dockerfile @@ -0,0 +1,15 @@ +FROM golang:1.26.5-alpine3.24@sha256:0178a641fbb4858c5f1b48e34bdaabe0350a330a1b1149aabd498d0699ff5fb2 AS build + +RUN apk add --no-cache git=2.54.0-r0 +WORKDIR /src +RUN git clone https://github.com/prometheus-community/postgres_exporter.git . && \ + git checkout 867fbcac31cd18c143e244190ea9168cca069827 +RUN CGO_ENABLED=0 go build -trimpath \ + -ldflags="-s -w -X github.com/prometheus/common/version.Version=0.20.1 -X github.com/prometheus/common/version.Revision=867fbcac31cd18c143e244190ea9168cca069827" \ + -o /out/postgres_exporter ./cmd/postgres_exporter + +FROM quay.io/prometheuscommunity/postgres-exporter:v0.20.1@sha256:ac5ec343104fae0e2d84a27bb8d69b38430a11910c5382cad85d478d2bab713e + +COPY --from=build /out/postgres_exporter /bin/postgres_exporter + +USER nobody diff --git a/packages/phlo-postgres/src/phlo_postgres/exporter_service.yaml b/packages/phlo-postgres/src/phlo_postgres/exporter_service.yaml index 80bb18ddc14..e56d68ad8e2 100644 --- a/packages/phlo-postgres/src/phlo_postgres/exporter_service.yaml +++ b/packages/phlo-postgres/src/phlo_postgres/exporter_service.yaml @@ -4,7 +4,10 @@ category: observability default: false profile: observability -image: quay.io/prometheuscommunity/postgres-exporter:v0.15.0 +image: ghcr.io/phlohouse/phlo-postgres-exporter:v0.20.1-go1.26.5 +build: + context: . + dockerfile: postgres-exporter/Dockerfile depends_on: - postgres @@ -37,3 +40,7 @@ env_vars: POSTGRES_EXPORTER_PORT: default: 9187 description: Postgres exporter metrics port + +files: + - source: exporter.Dockerfile + dest: postgres-exporter/Dockerfile diff --git a/packages/phlo-postgres/src/phlo_postgres/service.yaml b/packages/phlo-postgres/src/phlo_postgres/service.yaml index 2d72ebe7d56..3519a99a9a5 100644 --- a/packages/phlo-postgres/src/phlo_postgres/service.yaml +++ b/packages/phlo-postgres/src/phlo_postgres/service.yaml @@ -6,7 +6,10 @@ default: true depends_on: - postgres-volume-setup -image: postgres:16-alpine +image: ghcr.io/phlohouse/phlo-postgres:18.4-alpine3.24-gosu1.19 +build: + context: . + dockerfile: postgres/Dockerfile compose: restart: unless-stopped @@ -26,7 +29,7 @@ compose: ports: - "${POSTGRES_PORT:-10000}:5432" volumes: - - postgres-data:/var/lib/postgresql/data + - postgres-data:/var/lib/postgresql healthcheck: test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-phlo}"] interval: 10s @@ -47,6 +50,7 @@ env_vars: POSTGRES_PORT: default: 10000 description: PostgreSQL host port + # SSL/TLS POSTGRES_SSL_MODE: default: "prefer" @@ -60,3 +64,7 @@ env_vars: POSTGRES_SSL_CA_FILE: default: "" description: "Path to SSL CA certificate file" + +files: + - source: Dockerfile + dest: postgres/Dockerfile diff --git a/packages/phlo-postgres/src/phlo_postgres/volume_setup.yaml b/packages/phlo-postgres/src/phlo_postgres/volume_setup.yaml index a0626a0cc84..a9f1004e0e0 100644 --- a/packages/phlo-postgres/src/phlo_postgres/volume_setup.yaml +++ b/packages/phlo-postgres/src/phlo_postgres/volume_setup.yaml @@ -3,7 +3,7 @@ description: Initialize PostgreSQL data volume permissions category: core default: false -image: alpine:3.20 +image: alpine:3.24.1 compose: restart: "no" @@ -11,10 +11,14 @@ compose: entrypoint: /bin/sh command: > -c " - mkdir -p /var/lib/postgresql/data && - chown -R 70:70 /var/lib/postgresql/data && - chmod 700 /var/lib/postgresql/data && + if [ -f /var/lib/postgresql/PG_VERSION ]; then + echo 'PostgreSQL 16 data volume detected. Back it up with PostgreSQL 16, then restore it into a new PostgreSQL 18 volume before starting Phlo.' >&2; + exit 1; + fi && + mkdir -p /var/lib/postgresql && + chown -R 70:70 /var/lib/postgresql && + chmod 700 /var/lib/postgresql && echo 'Postgres data volume ownership initialized' " volumes: - - postgres-data:/var/lib/postgresql/data + - postgres-data:/var/lib/postgresql diff --git a/packages/phlo-postgres/tests/test_postgres_plugin.py b/packages/phlo-postgres/tests/test_postgres_plugin.py index 27fc45223cc..eecf2d8868e 100644 --- a/packages/phlo-postgres/tests/test_postgres_plugin.py +++ b/packages/phlo-postgres/tests/test_postgres_plugin.py @@ -1,7 +1,14 @@ """Tests for Postgres service and resource plugins.""" +from importlib import resources + from phlo.capabilities import PublishTargetSpec -from phlo_postgres.plugin import PostgresResourceProvider, PostgresServicePlugin +from phlo_postgres.plugin import ( + PostgresExporterServicePlugin, + PostgresResourceProvider, + PostgresServicePlugin, + PostgresVolumeSetupServicePlugin, +) from phlo_postgres.publish_target import PostgresPublishTarget @@ -14,6 +21,39 @@ def test_postgres_service_definition(): assert service_definition["category"] == "core" +def test_postgres_service_builds_pinned_hardened_image() -> None: + service_definition = PostgresServicePlugin().service_definition + + assert service_definition["image"] == ( + "ghcr.io/phlohouse/phlo-postgres:18.4-alpine3.24-gosu1.19" + ) + assert service_definition["build"]["dockerfile"] == "postgres/Dockerfile" + + +def test_postgres_image_runs_as_postgres_after_volume_setup() -> None: + """The setup service owns the data volume before the database starts non-root.""" + dockerfile = resources.files("phlo_postgres").joinpath("Dockerfile").read_text() + + assert dockerfile.rstrip().endswith("USER postgres") + + +def test_postgres_volume_setup_rejects_pre_18_data_layout() -> None: + command = PostgresVolumeSetupServicePlugin().service_definition["compose"]["command"] + + assert "/var/lib/postgresql/PG_VERSION" in command + assert "PostgreSQL 16 data volume detected" in command + assert "exit 1" in command + + +def test_postgres_exporter_builds_pinned_hardened_image() -> None: + service_definition = PostgresExporterServicePlugin().service_definition + + assert service_definition["image"] == ( + "ghcr.io/phlohouse/phlo-postgres-exporter:v0.20.1-go1.26.5" + ) + assert service_definition["build"]["dockerfile"] == "postgres-exporter/Dockerfile" + + def test_postgres_resource_provider(): """Validate Postgres resource provider output.""" provider = PostgresResourceProvider() diff --git a/packages/phlo-postgrest/README.md b/packages/phlo-postgrest/README.md index c7b59b5e2aa..45de730e73a 100644 --- a/packages/phlo-postgrest/README.md +++ b/packages/phlo-postgrest/README.md @@ -23,7 +23,7 @@ Part of the `api` profile. | Variable | Default | Description | | ------------------- | --------- | ------------------ | | `POSTGREST_PORT` | `3002` | PostgREST API port | -| `POSTGREST_VERSION` | `v12.2.3` | PostgREST version | +| `POSTGREST_VERSION` | `v14.15` | PostgREST version | | `POSTGRES_USER` | `phlo` | Database user | | `POSTGRES_PASSWORD` | `phlo` | Database password | | `POSTGRES_DB` | `phlo` | Database name | diff --git a/packages/phlo-postgrest/pyproject.toml b/packages/phlo-postgrest/pyproject.toml index 416f433f2aa..0ec8946baa2 100644 --- a/packages/phlo-postgrest/pyproject.toml +++ b/packages/phlo-postgrest/pyproject.toml @@ -49,6 +49,7 @@ include-package-data = true [tool.setuptools.package-data] phlo_postgrest = [ + "Dockerfile", "service.yaml", "sql/*.sql", "conf/*.conf", diff --git a/packages/phlo-postgrest/src/phlo_postgrest/Dockerfile b/packages/phlo-postgrest/src/phlo_postgrest/Dockerfile new file mode 100644 index 00000000000..ae68cbfa585 --- /dev/null +++ b/packages/phlo-postgrest/src/phlo_postgrest/Dockerfile @@ -0,0 +1,21 @@ +FROM postgrest/postgrest:v14.15@sha256:2f8e7b656f09db697a8875177694b417b35cb76c21370de07fc54e711e902326 AS upstream + +FROM ubuntu:26.04@sha256:3131b4cc82a783df6c9df078f86e01819a13594b865c2cad47bd1bca2b7063bb + +# Install the repository's current security revisions instead of freezing stale package versions. +# hadolint ignore=DL3008 +RUN apt-get update && \ + apt-get upgrade --yes && \ + apt-get install --yes --no-install-recommends \ + ca-certificates \ + libgmp10 \ + libpq5 \ + procps && \ + rm -f /usr/bin/pebble && \ + rm -rf /var/lib/apt/lists/* + +COPY --from=upstream /bin/postgrest /usr/bin/postgrest + +USER "1000" +EXPOSE 3000 +CMD ["postgrest"] diff --git a/packages/phlo-postgrest/src/phlo_postgrest/service.yaml b/packages/phlo-postgrest/src/phlo_postgrest/service.yaml index 00b553018e0..74c0955a431 100644 --- a/packages/phlo-postgrest/src/phlo_postgrest/service.yaml +++ b/packages/phlo-postgrest/src/phlo_postgrest/service.yaml @@ -4,7 +4,10 @@ category: api default: false profile: api -image: postgrest/postgrest:${POSTGREST_VERSION:-v12.2.3} +image: ghcr.io/phlohouse/phlo-postgrest:14.15-security-patches +build: + context: . + dockerfile: postgrest/Dockerfile depends_on: - postgres @@ -25,9 +28,6 @@ compose: retries: 5 env_vars: - POSTGREST_VERSION: - default: v12.2.3 - description: PostgREST version POSTGREST_PORT: default: 3002 description: PostgREST API port @@ -36,6 +36,8 @@ env_vars: description: Comma-separated list of schemas to expose (auto-configured by hook) files: + - source: Dockerfile + dest: postgrest/Dockerfile - source: conf dest: postgrest/conf diff --git a/packages/phlo-postgrest/tests/test_postgrest_plugin.py b/packages/phlo-postgrest/tests/test_postgrest_plugin.py new file mode 100644 index 00000000000..944be36fc30 --- /dev/null +++ b/packages/phlo-postgrest/tests/test_postgrest_plugin.py @@ -0,0 +1,23 @@ +"""PostgREST generated service contracts.""" + +from importlib import resources + +from phlo_postgrest.plugin import PostgrestServicePlugin + + +def test_postgrest_builds_a_minimal_stable_runtime_image() -> None: + definition = PostgrestServicePlugin().service_definition + + assert definition["image"] == "ghcr.io/phlohouse/phlo-postgrest:14.15-security-patches" + assert definition["build"] == { + "context": ".", + "dockerfile": "postgrest/Dockerfile", + } + assert "POSTGREST_VERSION" not in definition["env_vars"] + assert {"source": "Dockerfile", "dest": "postgrest/Dockerfile"} in definition["files"] + + dockerfile = resources.files("phlo_postgrest").joinpath("Dockerfile").read_text() + assert "postgrest/postgrest:v14.15@sha256:" in dockerfile + assert "COPY --from=upstream /bin/postgrest /usr/bin/postgrest" in dockerfile + assert "rm -f /usr/bin/pebble" in dockerfile + assert dockerfile.rstrip().endswith('CMD ["postgrest"]') diff --git a/packages/phlo-prometheus/README.md b/packages/phlo-prometheus/README.md index dedb85d9d23..55b9faabb04 100644 --- a/packages/phlo-prometheus/README.md +++ b/packages/phlo-prometheus/README.md @@ -23,7 +23,7 @@ Part of the `observability` profile. | Variable | Default | Description | | -------------------- | -------- | ---------------------- | | `PROMETHEUS_PORT` | `9090` | Prometheus web UI port | -| `PROMETHEUS_VERSION` | `v3.1.0` | Prometheus version | +| `PROMETHEUS_VERSION` | `v3.13.1` | Prometheus version | ## Auto-Configuration diff --git a/packages/phlo-prometheus/pyproject.toml b/packages/phlo-prometheus/pyproject.toml index 6c29326cf1b..7d8741ec44d 100644 --- a/packages/phlo-prometheus/pyproject.toml +++ b/packages/phlo-prometheus/pyproject.toml @@ -41,6 +41,7 @@ include-package-data = true [tool.setuptools.package-data] phlo_prometheus = [ + "Dockerfile", "service.yaml", "prometheus.yml", ] diff --git a/packages/phlo-prometheus/src/phlo_prometheus/Dockerfile b/packages/phlo-prometheus/src/phlo_prometheus/Dockerfile new file mode 100644 index 00000000000..cd21f317a3c --- /dev/null +++ b/packages/phlo-prometheus/src/phlo_prometheus/Dockerfile @@ -0,0 +1,17 @@ +FROM golang:1.26.5-alpine3.24@sha256:0178a641fbb4858c5f1b48e34bdaabe0350a330a1b1149aabd498d0699ff5fb2 AS build + +RUN apk add --no-cache git=2.54.0-r0 +WORKDIR /src +RUN git clone https://github.com/prometheus/prometheus.git . && \ + git checkout 73ff57ce2b8161059ac7fe5188f03f1c3d22b29a && \ + go get google.golang.org/grpc@v1.82.1 +RUN LDFLAGS="-s -w -X github.com/prometheus/common/version.Version=3.13.1 -X github.com/prometheus/common/version.Revision=73ff57ce2b8161059ac7fe5188f03f1c3d22b29a" && \ + CGO_ENABLED=0 go build -trimpath -ldflags="$LDFLAGS" -o /out/prometheus ./cmd/prometheus && \ + CGO_ENABLED=0 go build -trimpath -ldflags="$LDFLAGS" -o /out/promtool ./cmd/promtool + +FROM prom/prometheus:v3.13.1@sha256:3c42b892cf723fa54d2f262c37a0e1f80aa8c8ddb1da7b9b0df9455a35a7f893 + +COPY --from=build /out/prometheus /bin/prometheus +COPY --from=build /out/promtool /bin/promtool + +USER nobody diff --git a/packages/phlo-prometheus/src/phlo_prometheus/service.yaml b/packages/phlo-prometheus/src/phlo_prometheus/service.yaml index 3b8dcfce55a..57f848f8cd8 100644 --- a/packages/phlo-prometheus/src/phlo_prometheus/service.yaml +++ b/packages/phlo-prometheus/src/phlo_prometheus/service.yaml @@ -4,7 +4,10 @@ category: observability default: false profile: observability -image: prom/prometheus:${PROMETHEUS_VERSION:-v3.1.0} +image: ${PROMETHEUS_IMAGE:-ghcr.io/phlohouse/phlo-prometheus:v3.13.1-grpc1.82.1} +build: + context: . + dockerfile: prometheus/Dockerfile compose: restart: unless-stopped @@ -38,8 +41,11 @@ compose: retries: 5 env_vars: + PROMETHEUS_IMAGE: + default: ghcr.io/phlohouse/phlo-prometheus:v3.13.1-grpc1.82.1 + description: Pinned Prometheus image to build or run PROMETHEUS_VERSION: - default: v3.1.0 + default: v3.13.1 description: Prometheus version PROMETHEUS_PORT: default: 9090 @@ -52,5 +58,7 @@ env_vars: description: Path used to build Prometheus query links files: + - source: Dockerfile + dest: prometheus/Dockerfile - source: prometheus.yml dest: prometheus/prometheus.yml diff --git a/packages/phlo-prometheus/tests/test_prometheus_plugin.py b/packages/phlo-prometheus/tests/test_prometheus_plugin.py index 0b8e9abb104..d57ca26d420 100644 --- a/packages/phlo-prometheus/tests/test_prometheus_plugin.py +++ b/packages/phlo-prometheus/tests/test_prometheus_plugin.py @@ -12,6 +12,10 @@ def test_prometheus_service_definition(): assert defn["profile"] == "observability" assert "prometheus-data:/prometheus" in defn["compose"]["volumes"] assert "./volumes/prometheus:/prometheus" not in defn["compose"]["volumes"] + assert defn["image"] == ( + "${PROMETHEUS_IMAGE:-ghcr.io/phlohouse/phlo-prometheus:v3.13.1-grpc1.82.1}" + ) + assert defn["build"]["dockerfile"] == "prometheus/Dockerfile" def test_prometheus_plugin_metadata(): diff --git a/packages/phlo-rustfs/src/phlo_rustfs/rustfs-setup.yaml b/packages/phlo-rustfs/src/phlo_rustfs/rustfs-setup.yaml index 92971462c10..ff205e21708 100644 --- a/packages/phlo-rustfs/src/phlo_rustfs/rustfs-setup.yaml +++ b/packages/phlo-rustfs/src/phlo_rustfs/rustfs-setup.yaml @@ -4,7 +4,7 @@ description: Initialize RustFS buckets for data lake category: core default: false -image: amazon/aws-cli:2.15.40 +image: amazon/aws-cli:2.36.8 depends_on: - rustfs diff --git a/packages/phlo-rustfs/src/phlo_rustfs/rustfs-volume-setup.yaml b/packages/phlo-rustfs/src/phlo_rustfs/rustfs-volume-setup.yaml index 53098cacb5d..ea3bfaa224e 100644 --- a/packages/phlo-rustfs/src/phlo_rustfs/rustfs-volume-setup.yaml +++ b/packages/phlo-rustfs/src/phlo_rustfs/rustfs-volume-setup.yaml @@ -4,7 +4,7 @@ description: Initialize RustFS data volume permissions category: core default: false -image: alpine:3.20 +image: alpine:3.24.1 compose: restart: "no" diff --git a/packages/phlo-rustfs/src/phlo_rustfs/service.yaml b/packages/phlo-rustfs/src/phlo_rustfs/service.yaml index a39cbc87bfe..aa311c5ccc0 100644 --- a/packages/phlo-rustfs/src/phlo_rustfs/service.yaml +++ b/packages/phlo-rustfs/src/phlo_rustfs/service.yaml @@ -6,7 +6,7 @@ default: false depends_on: - rustfs-volume-setup -image: rustfs/rustfs:latest +image: rustfs/rustfs:1.0.0-beta.11 compose: restart: unless-stopped diff --git a/packages/phlo-superset/README.md b/packages/phlo-superset/README.md index a32c260a854..13d0dd1d7d0 100644 --- a/packages/phlo-superset/README.md +++ b/packages/phlo-superset/README.md @@ -19,7 +19,7 @@ phlo plugin install superset | Variable | Default | Description | | ------------------------- | ------------------- | ---------------------- | | `SUPERSET_PORT` | `8088` | Superset web UI port | -| `SUPERSET_VERSION` | `4.0.0` | Superset version | +| `SUPERSET_VERSION` | `6.1.0` | Superset version | | `SUPERSET_SECRET_KEY` | auto-generated | Session encryption key | | `SUPERSET_ADMIN_USER` | `admin` | Admin username | | `SUPERSET_ADMIN_PASSWORD` | `admin` | Admin password | diff --git a/packages/phlo-superset/pyproject.toml b/packages/phlo-superset/pyproject.toml index 645a37ce124..69f2cf90202 100644 --- a/packages/phlo-superset/pyproject.toml +++ b/packages/phlo-superset/pyproject.toml @@ -44,6 +44,7 @@ include-package-data = true [tool.setuptools.package-data] phlo_superset = [ + "Dockerfile", "service.yaml", "dashboards/**", ] diff --git a/packages/phlo-superset/src/phlo_superset/Dockerfile b/packages/phlo-superset/src/phlo_superset/Dockerfile new file mode 100644 index 00000000000..d2e213fdc87 --- /dev/null +++ b/packages/phlo-superset/src/phlo_superset/Dockerfile @@ -0,0 +1,38 @@ +FROM apache/superset:6.1.0@sha256:fb3464528ec7076f91195f0ff7835755aa023e281f1bb78a84782ce7a36b3705 + +USER "0" + +RUN apt-get update \ + && apt-get upgrade --yes \ + && apt-get purge --yes \ + libc-dev-bin \ + libc6-dev \ + libcrypt-dev \ + libecpg-dev \ + libldap-dev \ + libldap2-dev \ + libnsl-dev \ + libpq-dev \ + libsasl2-dev \ + libssl-dev \ + libtirpc-dev \ + linux-libc-dev \ + && apt-get autoremove --yes \ + && rm -rf /var/lib/apt/lists/* + +RUN uv pip install --python /app/.venv/bin/python --upgrade \ + 'Mako>=1.3.12' \ + 'PyJWT>=2.13.0' \ + 'cryptography==46.0.5' \ + 'pillow>=12.3.0' \ + 'pyOpenSSL>=26.0.0' \ + 'pyasn1>=0.6.4' \ + 'urllib3>=2.7.0' \ + 'setuptools>=80.9.0' \ + && uv pip install --python /usr/local/bin/python --upgrade \ + 'setuptools>=80.9.0' \ + 'wheel>=0.46.2' \ + && uv pip check --python /app/.venv/bin/python \ + && rm -f /usr/local/bin/uv /usr/local/bin/uvx + +USER "1000" diff --git a/packages/phlo-superset/src/phlo_superset/service.yaml b/packages/phlo-superset/src/phlo_superset/service.yaml index 6d189aedb7b..4e7b3881177 100644 --- a/packages/phlo-superset/src/phlo_superset/service.yaml +++ b/packages/phlo-superset/src/phlo_superset/service.yaml @@ -3,14 +3,20 @@ description: Apache Superset for business intelligence and data visualization category: bi default: true -image: apache/superset:${SUPERSET_VERSION:-4.0.0} +image: ghcr.io/phlohouse/phlo-superset:6.1.0-security-patches +build: + context: . + dockerfile: superset/Dockerfile +files: + - source: Dockerfile + dest: superset/Dockerfile depends_on: - postgres - trino compose: - user: "0" + user: "1000:1000" restart: unless-stopped labels: phlo.metrics.enabled: "false" # Superset doesn't expose native Prometheus metrics @@ -30,14 +36,10 @@ compose: SUPERSET_ADMIN_EMAIL: ${SUPERSET_ADMIN_EMAIL:-admin@example.com} command: > /bin/sh -c " - mkdir -p /app/superset_home && - chown -R superset:superset /app/superset_home && - su superset -s /bin/sh -c ' superset db upgrade && superset fab create-admin --username $${SUPERSET_ADMIN_USER} --firstname Admin --lastname User --email $${SUPERSET_ADMIN_EMAIL} --password $${SUPERSET_ADMIN_PASSWORD} || true && superset init && superset run -h 0.0.0.0 -p 8088 --with-threads - ' " ports: - "${SUPERSET_PORT:-8088}:8088" @@ -45,9 +47,6 @@ compose: - superset-home:/app/superset_home env_vars: - SUPERSET_VERSION: - default: "4.0.0" - description: Apache Superset version SUPERSET_PORT: default: 8088 description: Superset web UI port diff --git a/packages/phlo-superset/tests/test_superset_plugin.py b/packages/phlo-superset/tests/test_superset_plugin.py index cdee81009bb..882118c55e3 100644 --- a/packages/phlo-superset/tests/test_superset_plugin.py +++ b/packages/phlo-superset/tests/test_superset_plugin.py @@ -1,5 +1,7 @@ """Tests for Superset service plugin.""" +from importlib import resources + from phlo_superset.plugin import SupersetServicePlugin from phlo_superset.settings import SupersetSettings @@ -12,6 +14,17 @@ def test_superset_service_definition(): assert defn["name"] == "superset" +def test_superset_builds_a_patched_runtime_image(): + """The generated service builds the audited runtime derivative.""" + definition = SupersetServicePlugin().service_definition + dockerfile = resources.files("phlo_superset").joinpath("Dockerfile").read_text() + + assert definition["image"] == "ghcr.io/phlohouse/phlo-superset:6.1.0-security-patches" + assert definition["build"] == {"context": ".", "dockerfile": "superset/Dockerfile"} + assert definition["compose"]["user"] == "1000:1000" + assert 'USER "1000"' in dockerfile + + def test_superset_plugin_metadata(): """Verify Superset plugin metadata includes expected identity and tags.""" plugin = SupersetServicePlugin() diff --git a/packages/phlo-traefik/src/phlo_traefik/service.yaml b/packages/phlo-traefik/src/phlo_traefik/service.yaml index 85aeee5a258..5081c440c18 100644 --- a/packages/phlo-traefik/src/phlo_traefik/service.yaml +++ b/packages/phlo-traefik/src/phlo_traefik/service.yaml @@ -4,7 +4,7 @@ category: networking default: false profile: proxy -image: traefik:v3.3 +image: traefik:v3.7.9 compose: restart: unless-stopped diff --git a/packages/phlo-trino/pyproject.toml b/packages/phlo-trino/pyproject.toml index 1b5f468cfec..696abcc7c5a 100644 --- a/packages/phlo-trino/pyproject.toml +++ b/packages/phlo-trino/pyproject.toml @@ -57,6 +57,7 @@ include-package-data = true [tool.setuptools.package-data] phlo_trino = [ + "Dockerfile", "service.yaml", "jvm.config", "config.properties", diff --git a/packages/phlo-trino/src/phlo_trino/Dockerfile b/packages/phlo-trino/src/phlo_trino/Dockerfile new file mode 100644 index 00000000000..a7b140b63fe --- /dev/null +++ b/packages/phlo-trino/src/phlo_trino/Dockerfile @@ -0,0 +1,17 @@ +FROM golang:1.26.5-alpine3.23@sha256:622e56dbc11a8cfe87cafa2331e9a201877271cbff918af53d3be315f3da88cc AS build + +RUN apk add --no-cache git=2.52.0-r0 +WORKDIR /src +RUN git clone https://github.com/airlift/launcher.git . && \ + git checkout e0e239b162b4dd45f60aacd5c58394d2832d0d13 +WORKDIR /src/src/main/go +RUN CGO_ENABLED=0 go build -buildvcs=false -trimpath \ + -ldflags="-s -w -extldflags=-static -X launcher/args.Version=318" \ + -o /out/launcher . + +FROM trinodb/trino:483@sha256:db58cc93e593a2706553745f276bb119c9810e69918be56ecde088ba7ccb0534 + +ARG TARGETARCH +COPY --from=build --chown=trino:trino /out/launcher /usr/lib/trino/bin/linux-${TARGETARCH}/launcher + +USER trino diff --git a/packages/phlo-trino/src/phlo_trino/service.yaml b/packages/phlo-trino/src/phlo_trino/service.yaml index 04a59885e1d..03572411529 100644 --- a/packages/phlo-trino/src/phlo_trino/service.yaml +++ b/packages/phlo-trino/src/phlo_trino/service.yaml @@ -3,7 +3,11 @@ description: Distributed SQL query engine for the data lake category: core default: true -image: trinodb/trino:${TRINO_VERSION:-467} +image: ghcr.io/phlohouse/phlo-trino:483-launcher318-go1.26.5 + +build: + context: ./trino + dockerfile: Dockerfile depends_on: - nessie @@ -41,7 +45,7 @@ compose: env_vars: TRINO_VERSION: - default: "467" + default: "483" description: Trino version TRINO_PORT: default: 10005 @@ -96,6 +100,8 @@ env_vars: description: Time-to-live for cached query results files: + - source: Dockerfile + dest: trino/Dockerfile - source: jvm.config dest: trino/jvm.config - source: config.properties diff --git a/packages/phlo-trino/tests/test_trino_plugin.py b/packages/phlo-trino/tests/test_trino_plugin.py index ba6d18884c2..b83674e8e1e 100644 --- a/packages/phlo-trino/tests/test_trino_plugin.py +++ b/packages/phlo-trino/tests/test_trino_plugin.py @@ -17,6 +17,17 @@ def test_trino_service_definition(): assert service_definition["category"] == "core" +def test_trino_rebuilds_the_launcher_with_current_go(): + service_definition = TrinoServicePlugin().service_definition + + assert service_definition["image"] == "ghcr.io/phlohouse/phlo-trino:483-launcher318-go1.26.5" + assert service_definition["build"] == { + "context": "./trino", + "dockerfile": "Dockerfile", + } + assert {"source": "Dockerfile", "dest": "trino/Dockerfile"} in service_definition["files"] + + def test_trino_runtime_files_are_included_in_package_data(): """Every file copied into .phlo/trino must be present in installed wheels.""" diff --git a/registry/support/v1.json b/registry/support/v1.json index f73b0f0d802..dc4ffef5d9f 100644 --- a/registry/support/v1.json +++ b/registry/support/v1.json @@ -26,16 +26,16 @@ {"name": "phlo-trino", "version": "0.5.0"} ], "services": [ - {"name": "dagster", "image_reference": "build:context=.;dockerfile=dagster/Dockerfile"}, - {"name": "dagster-daemon", "image_reference": "build:context=.;dockerfile=dagster/Dockerfile"}, - {"name": "postgres", "image_reference": "postgres:16-alpine"}, - {"name": "postgres-volume-setup", "image_reference": "alpine:3.20"}, - {"name": "minio", "image_reference": "minio/minio:RELEASE.2025-09-07T16-13-09Z"}, - {"name": "minio-setup", "image_reference": "minio/mc:RELEASE.2025-08-13T08-35-41Z"}, - {"name": "nessie", "image_reference": "ghcr.io/projectnessie/nessie:0.107.2"}, - {"name": "trino", "image_reference": "trinodb/trino:467"}, - {"name": "phlo-api", "image_reference": "build:context=.;dockerfile=phlo-api/Dockerfile"}, - {"name": "observatory", "image_reference": "build:context=source;dockerfile=Dockerfile"} + {"name": "dagster", "image_reference": "ghcr.io/phlohouse/phlo-dagster:0.6.0"}, + {"name": "dagster-daemon", "image_reference": "ghcr.io/phlohouse/phlo-dagster:0.6.0"}, + {"name": "postgres", "image_reference": "ghcr.io/phlohouse/phlo-postgres:18.4-alpine3.24-gosu1.19"}, + {"name": "postgres-volume-setup", "image_reference": "alpine:3.24.1"}, + {"name": "minio", "image_reference": "ghcr.io/phlohouse/phlo-minio:7aac2a2c5b7c"}, + {"name": "minio-setup", "image_reference": "ghcr.io/phlohouse/phlo-minio-mc:77f82e18b540"}, + {"name": "nessie", "image_reference": "ghcr.io/phlohouse/phlo-nessie:0.108.3-netty4.2.16"}, + {"name": "trino", "image_reference": "ghcr.io/phlohouse/phlo-trino:483-launcher318-go1.26.5"}, + {"name": "phlo-api", "image_reference": "ghcr.io/phlohouse/phlo-api:0.7.0"}, + {"name": "observatory", "image_reference": "ghcr.io/phlohouse/phlo-observatory:0.7.0"} ], "schemas": { "configuration": { diff --git a/scripts/generated_image_matrix.py b/scripts/generated_image_matrix.py new file mode 100644 index 00000000000..5bfa8a644db --- /dev/null +++ b/scripts/generated_image_matrix.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +"""Build a unique GHCR publication matrix from rendered Compose JSON.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from typing import Any + + +def publication_matrix( + compose: dict[str, Any], + project_root: Path, + source_root: Path | None = None, + selected_services: set[str] | None = None, +) -> dict[str, Any]: + """Return one build target per unique published image.""" + services = compose.get("services") + if not isinstance(services, dict): + raise ValueError("Compose JSON has no services object") + published: dict[str, dict[str, Any]] = {} + for service_name, service in services.items(): + if not isinstance(service, dict) or not service.get("build"): + continue + image = service.get("image") + if not isinstance(image, str) or not image.startswith("ghcr.io/phlohouse/phlo-"): + raise ValueError(f"built service {service_name!r} has no Phlo GHCR image") + build = service["build"] + if not isinstance(build, dict): + raise ValueError(f"built service {service_name!r} has invalid build configuration") + context = Path(str(build.get("context", ""))).resolve() + dockerfile = Path(str(build.get("dockerfile", "Dockerfile"))) + dockerfile = dockerfile if dockerfile.is_absolute() else context / dockerfile + roots = (("generated", project_root), ("source", source_root)) + resolved_paths: tuple[str, Path, Path] | None = None + for root_name, root in roots: + if root is None: + continue + try: + resolved_paths = ( + root_name, + context.relative_to(root.resolve()), + dockerfile.resolve().relative_to(root.resolve()), + ) + break + except ValueError: + continue + if resolved_paths is None: + raise ValueError(f"built service {service_name!r} escapes publication roots") + context_root, context_relative, dockerfile_relative = resolved_paths + tag = image.split("@", 1)[0] + target = { + "service": service_name, + "services": [service_name], + "image": tag, + "root": context_root, + "context": str(context_relative), + "dockerfile": str(dockerfile_relative), + "build_args": build.get("args") or {}, + } + existing = published.get(tag) + if existing is None: + published[tag] = target + continue + comparable_keys = ("root", "context", "dockerfile", "build_args") + if any(existing[key] != target[key] for key in comparable_keys): + raise ValueError(f"published image {tag!r} has conflicting build definitions") + existing["services"].append(service_name) + if not published: + raise ValueError("Compose JSON has no published build services") + targets = list(published.values()) + if selected_services: + known_services = {service for target in targets for service in target["services"]} + unknown_services = selected_services - known_services + if unknown_services: + unknown = ", ".join(sorted(unknown_services)) + raise ValueError(f"selected services are not published build services: {unknown}") + targets = [ + target for target in targets if selected_services.intersection(target["services"]) + ] + return {"include": targets} + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("compose_json", type=Path) + parser.add_argument("project_root", type=Path) + parser.add_argument("source_root", type=Path) + parser.add_argument("--services", default="") + args = parser.parse_args() + compose = json.loads(args.compose_json.read_text(encoding="utf-8")) + print( + json.dumps( + publication_matrix( + compose, + args.project_root, + args.source_root, + {service.strip() for service in args.services.split(",") if service.strip()}, + ), + separators=(",", ":"), + ) + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/recovery_drill.py b/scripts/recovery_drill.py index ecc5e7f05e1..74528df94d6 100644 --- a/scripts/recovery_drill.py +++ b/scripts/recovery_drill.py @@ -23,10 +23,10 @@ from typing import Any from uuid import uuid4 -POSTGRES_IMAGE = "postgres:16-alpine" +POSTGRES_IMAGE = "postgres:18-alpine" MINIO_IMAGE = "minio/minio:RELEASE.2025-09-07T16-13-09Z" -NESSIE_IMAGE = "ghcr.io/projectnessie/nessie:0.107.2" -NESSIE_ADMIN_IMAGE = "ghcr.io/projectnessie/nessie-server-admin@sha256:94e67d471380e6da17680e166e3111819058a64c6717c96beafd1f3df6ec5876" +NESSIE_IMAGE = "ghcr.io/projectnessie/nessie:0.108.3" +NESSIE_ADMIN_IMAGE = "ghcr.io/projectnessie/nessie-server-admin@sha256:ffccc83adc048ae9c069205b2b7c79c8c72604574558f915b730f2266262c159" MC_IMAGE = "minio/mc@sha256:a7fe349ef4bd8521fb8497f55c6042871b2ae640607cf99d9bede5e9bdf11727" HELPER_IMAGE = "python@sha256:db3ff2e1800a8581e2c48a27c3995339d47bdf046da21c7627accd3d51053a93" OWNER_MARKER = ".phlo-recovery-drill-owner.json" @@ -77,7 +77,7 @@ def compose_yaml(stack: Stack) -> str: POSTGRES_PASSWORD: phlo POSTGRES_DB: phlo ports: [\"127.0.0.1::5432\"] - volumes: [postgres-data:/var/lib/postgresql/data] + volumes: [postgres-data:/var/lib/postgresql] healthcheck: test: [\"CMD-SHELL\", \"pg_isready -U phlo\"] interval: 2s diff --git a/scripts/release_golden_path.py b/scripts/release_golden_path.py index 1b531326b60..3271c3f18a4 100644 --- a/scripts/release_golden_path.py +++ b/scripts/release_golden_path.py @@ -740,6 +740,19 @@ def verify_run_report(config: RunConfig, wap_run: WapRun) -> None: def verify_rejected_wap_report(config: RunConfig, wap_run: WapRun) -> None: """Require durable rejected-quality evidence and prove that run was not promoted.""" payload = fetch_run_report(config, wap_run, config.rejection_report_token) + deadline = time.monotonic() + RUN_REPORT_TIMEOUT_SECONDS + while True: + catalog_changes = payload.get("catalog_changes") + if isinstance(catalog_changes, list) and any( + isinstance(change, dict) and change.get("merge_outcome") == "rejected_quality" + for change in catalog_changes + ): + break + if time.monotonic() >= deadline: + break + time.sleep(1) + payload = fetch_run_report(config, wap_run, config.rejection_report_token) + quality = payload.get("quality") if not isinstance(quality, list) or not any( isinstance(result, dict) @@ -750,7 +763,6 @@ def verify_rejected_wap_report(config: RunConfig, wap_run: WapRun) -> None: raise RuntimeError( f"rejected WAP report lacks a blocking failed quality result: {payload!r}" ) - catalog_changes = payload.get("catalog_changes") if not isinstance(catalog_changes, list) or not any( isinstance(change, dict) and change.get("merge_outcome") == "rejected_quality" for change in catalog_changes diff --git a/src/phlo/cli/commands/plugin/check.py b/src/phlo/cli/commands/plugin/check.py index 796e842c037..d14a4dff0b2 100644 --- a/src/phlo/cli/commands/plugin/check.py +++ b/src/phlo/cli/commands/plugin/check.py @@ -2,17 +2,1094 @@ from __future__ import annotations +import hashlib +import importlib.metadata import json +import os +import re +import shutil +import subprocess import sys +import tempfile +from collections.abc import Callable +from dataclasses import dataclass +from pathlib import Path +from typing import Any import click +from rich.text import Text from phlo.cli.commands.plugin.utils import console from phlo.logging import get_logger from phlo.plugins import discover_plugins, validate_plugins +from phlo.plugins.discovery import ServiceDiscovery +from phlo.plugins.discovery._service_loading import resolve_plugin_source_path logger = get_logger(__name__) +HADOLINT_IMAGE = ( + "hadolint/hadolint@sha256:27086352fd5e1907ea2b934eb1023f217c5ae087992eb59fde121dce9c9ff21e" +) +TRIVY_IMAGE = ( + "aquasec/trivy@sha256:cffe3f5161a47a6823fbd23d985795b3ed72a4c806da4c4df16266c02accdd6f" +) +MAX_TOOL_OUTPUT_CHARS = 64 * 1024 +MAX_TRIVY_JSON_CHARS = 8 * 1024 * 1024 +MAX_COMPOSE_CONFIG_CHARS = 4 * 1024 * 1024 +_REAL_SUBPROCESS_RUN = subprocess.run + + +class ContainerCheckError(RuntimeError): + """Raised when generated container checks cannot complete.""" + + +@dataclass(frozen=True) +class VulnerabilityWaiver: + """A waiver bound to one exact normalized Trivy finding set.""" + + evidence_sha256: str + reason: str + + +def _plugin_package(plugin: Any) -> str: + """Resolve the installed distribution owning a discovered service plugin.""" + top_level = plugin.__class__.__module__.split(".", 1)[0] + distributions = importlib.metadata.packages_distributions().get(top_level, []) + return sorted(distributions)[0] if distributions else plugin.metadata.name + + +def _service_inventory() -> tuple[dict[str, str], list[str]]: + """Return generated service-file owners and all currently installed service names.""" + owners: dict[str, str] = {} + service_names: list[str] = [] + package_roots: dict[Path, str] = {} + discovered = discover_plugins(plugin_type="service", auto_register=True) + for plugin in discovered.get("service", []): + package = _plugin_package(plugin) + source_path = resolve_plugin_source_path(plugin) + if source_path: + package_roots[source_path.resolve()] = package + service_definition = plugin.service_definition + service_name = service_definition.get("name") + if service_name: + service_names.append(service_name) + for file_spec in plugin.get_files(): + destination = file_spec.get("dest") + if destination: + owners[destination] = package + for service_name, definition in ServiceDiscovery().discover().items(): + source_path = definition.source_path + if not source_path: + continue + resolved_source = source_path.resolve() + matching_roots = [ + (len(root.parts), package) + for root, package in package_roots.items() + if resolved_source == root or root in resolved_source.parents + ] + if matching_roots: + package = max(matching_roots)[1] + owners.setdefault(f"@service:{service_name}", package) + for file_spec in definition.files: + destination = file_spec.get("dest") + if destination: + owners.setdefault(destination, package) + return owners, list(dict.fromkeys(service_names)) + + +def _run_command( + command: list[str], + *, + cwd: Path, + runner: Callable[..., Any], + label: str, +) -> str | None: + """Run one external command and return a failure detail, if any.""" + try: + result = _run_with_capture(command, cwd=cwd, runner=runner) + except OSError as exc: + return f"{label} could not start: {exc}" + if result.returncode: + return _run_command_result_failure(result, label) + return None + + +def _run_output_command( + command: list[str], + *, + cwd: Path, + runner: Callable[..., Any], + label: str, + max_output_chars: int = MAX_TOOL_OUTPUT_CHARS, +) -> tuple[str, str]: + """Run a command and return stdout, raising with both output streams on failure.""" + try: + result = _run_with_capture( + command, + cwd=cwd, + runner=runner, + max_output_chars=max_output_chars, + ) + except OSError as exc: + raise ContainerCheckError(f"{label} could not start: {exc}") from exc + if result.returncode: + failure = _run_command_result_failure(result, label) + raise ContainerCheckError(failure) + return result.stdout or "", result.stderr or "" + + +def _run_command_result_failure(result: Any, label: str) -> str: + """Format a failed command result without losing either output stream.""" + output = [] + if result.stdout: + output.append(f"stdout: {_limit_tool_output(result.stdout).strip()}") + if result.stderr: + output.append(f"stderr: {_limit_tool_output(result.stderr).strip()}") + detail = "\n".join(output) or "no output" + return f"{label} failed with exit code {result.returncode}: {detail}" + + +def _join_failure_details(*details: str | None) -> str: + """Keep the context from multiple failed steps in the final report.""" + return "\n".join(detail for detail in details if detail) or "no output" + + +def _parse_vulnerability_waivers( + values: tuple[str, ...], +) -> dict[tuple[str, str], VulnerabilityWaiver]: + """Parse waivers bound to an exact service, image, and finding fingerprint.""" + waivers: dict[tuple[str, str], VulnerabilityWaiver] = {} + for value in values: + parts = [part.strip() for part in value.split("=", 3)] + if len(parts) != 4 or not all(parts): + raise click.BadParameter( + "expected SERVICE=IMAGE=EVIDENCE_SHA256=REASON", + param_hint="--allow-vulnerable-image", + ) + service, image, evidence_sha256, reason = parts + if len(evidence_sha256) != 64 or any( + character not in "0123456789abcdef" for character in evidence_sha256.lower() + ): + raise click.BadParameter( + "EVIDENCE_SHA256 must be a 64-character hexadecimal digest", + param_hint="--allow-vulnerable-image", + ) + key = (service, image) + if key in waivers: + raise click.BadParameter( + f"duplicate waiver for {service} using {image}", + param_hint="--allow-vulnerable-image", + ) + waivers[key] = VulnerabilityWaiver(evidence_sha256.lower(), reason) + return waivers + + +def _run_with_capture( + command: list[str], + *, + cwd: Path, + runner: Callable[..., Any], + max_output_chars: int = MAX_TOOL_OUTPUT_CHARS, +) -> Any: + """Run a real command without retaining an unbounded scanner transcript.""" + if runner is _REAL_SUBPROCESS_RUN: + return _run_bounded_subprocess( + command, + cwd=cwd, + max_output_chars=max_output_chars, + ) + return runner(command, cwd=cwd, capture_output=True, text=True, check=False) + + +def _run_bounded_subprocess( + command: list[str], + *, + cwd: Path, + max_output_chars: int = MAX_TOOL_OUTPUT_CHARS, +) -> subprocess.CompletedProcess[str]: + """Run a command through spooled files and retain useful output at bounded size.""" + with tempfile.TemporaryFile() as stdout_file, tempfile.TemporaryFile() as stderr_file: + completed = _REAL_SUBPROCESS_RUN( + command, + cwd=cwd, + stdout=stdout_file, + stderr=stderr_file, + check=False, + ) + return subprocess.CompletedProcess( + command, + completed.returncode, + stdout=_read_bounded_file(stdout_file, max_output_chars=max_output_chars), + stderr=_read_bounded_file(stderr_file, max_output_chars=max_output_chars), + ) + + +def _read_bounded_file(file_handle: Any, *, max_output_chars: int = MAX_TOOL_OUTPUT_CHARS) -> str: + """Read a file's head and tail without loading a large tool report.""" + file_handle.seek(0, 2) + size = file_handle.tell() + if size <= max_output_chars: + file_handle.seek(0) + return file_handle.read().decode("utf-8", errors="replace") + + half_limit = max_output_chars // 2 + file_handle.seek(0) + head = file_handle.read(half_limit) + file_handle.seek(-half_limit, 2) + tail = file_handle.read(half_limit) + return ( + head.decode("utf-8", errors="replace") + + f"\n... [output truncated; {size} bytes total] ...\n" + + tail.decode("utf-8", errors="replace") + ) + + +def _limit_tool_output(value: str) -> str: + """Bound output from injected runners as well as real subprocesses.""" + if len(value) <= MAX_TOOL_OUTPUT_CHARS: + return value + half_limit = MAX_TOOL_OUTPUT_CHARS // 2 + return ( + value[:half_limit] + + f"\n... [output truncated; {len(value)} characters total] ...\n" + + value[-half_limit:] + ) + + +def _trivy_vulnerability_evidence(stdout: str) -> dict[str, Any] | None: + """Extract exact HIGH/CRITICAL findings from Trivy JSON output.""" + if not stdout.strip(): + return None + try: + report = json.loads(stdout) + except json.JSONDecodeError: + return None + + vulnerable_components: list[dict[str, str]] = [] + for result in report.get("Results") or []: + for finding in result.get("Vulnerabilities") or []: + severity = str(finding.get("Severity") or "").upper() + if severity not in {"HIGH", "CRITICAL"}: + continue + vulnerable_components.append( + { + "target": str(result.get("Target") or ""), + "class": str(result.get("Class") or ""), + "type": str(result.get("Type") or ""), + "component": str(finding.get("PkgName") or ""), + "installed_version": str(finding.get("InstalledVersion") or ""), + "fixed_version": str(finding.get("FixedVersion") or ""), + "vulnerability_id": str(finding.get("VulnerabilityID") or ""), + "severity": severity, + } + ) + return { + "high_count": sum(finding["severity"] == "HIGH" for finding in vulnerable_components), + "critical_count": sum( + finding["severity"] == "CRITICAL" for finding in vulnerable_components + ), + "vulnerable_components": vulnerable_components, + } + + +def _vulnerability_evidence_sha256(evidence: dict[str, Any]) -> str: + """Fingerprint the exact vulnerability IDs and components, including duplicates.""" + normalized = sorted( + [finding["vulnerability_id"], finding["component"]] + for finding in evidence["vulnerable_components"] + ) + payload = json.dumps(normalized, separators=(",", ":")).encode() + return hashlib.sha256(payload).hexdigest() + + +def _run_trivy_image_scan( + command: list[str], + *, + cwd: Path, + runner: Callable[..., Any], + label: str, +) -> tuple[str | None, dict[str, Any] | None, bool]: + """Run Trivy once, preserving failure streams and returning structured findings.""" + try: + result = _run_with_capture( + command, + cwd=cwd, + runner=runner, + max_output_chars=MAX_TRIVY_JSON_CHARS, + ) + except OSError as exc: + return f"{label} could not start: {exc}", None, False + + evidence = _trivy_vulnerability_evidence(result.stdout or "") + if result.returncode: + waivable = bool( + result.returncode == 1 + and evidence + and evidence["vulnerable_components"] + and not (result.stderr or "").strip() + ) + return _run_command_result_failure(result, label), evidence, waivable + return ( + None, + evidence + or { + "high_count": 0, + "critical_count": 0, + "vulnerable_components": [], + }, + False, + ) + + +def _run_checked_command( + command: list[str], + *, + cwd: Path, + runner: Callable[..., Any], + label: str, +) -> None: + """Run one required setup command and raise on failure.""" + failure = _run_command(command, cwd=cwd, runner=runner, label=label) + if failure: + raise ContainerCheckError(failure) + + +def _existing_image_id( + docker: str, + image: str, + *, + cwd: Path, + runner: Callable[..., Any], +) -> str | None: + """Return a local image ID without treating an absent tag as a check failure.""" + try: + result = _run_with_capture( + [docker, "image", "inspect", "--format", "{{.Id}}", image], + cwd=cwd, + runner=runner, + ) + except OSError as exc: + raise ContainerCheckError(f"docker image inspect {image} could not start: {exc}") from exc + if result.returncode: + detail = f"{result.stdout or ''}\n{result.stderr or ''}".lower() + if "no such image" in detail: + return None + raise ContainerCheckError( + _run_command_result_failure(result, f"docker image inspect {image}") + ) + return (result.stdout or "").strip() or None + + +def _remote_image_reference( + docker: str, + image: str, + *, + cwd: Path, + runner: Callable[..., Any], +) -> str: + """Resolve a registry image tag to an immutable manifest digest without pulling it.""" + if "@sha256:" in image: + return image + stdout, _ = _run_output_command( + [ + docker, + "buildx", + "imagetools", + "inspect", + "--format", + "{{json .Manifest.Digest}}", + image, + ], + cwd=cwd, + runner=runner, + label=f"docker buildx imagetools inspect {image}", + ) + try: + digest = json.loads(stdout) + except json.JSONDecodeError as exc: + raise ContainerCheckError(f"registry returned an invalid digest for {image}") from exc + if not isinstance(digest, str) or not re.fullmatch(r"sha256:[0-9a-f]{64}", digest): + raise ContainerCheckError(f"registry returned an invalid digest for {image}: {digest!r}") + registry_path, separator, image_name = image.rpartition("/") + repository_name = image_name.rsplit(":", 1)[0] + repository = f"{registry_path}/{repository_name}" if separator else repository_name + return f"{repository}@{digest}" + + +def _check_remote_service_images( + *, + compose_services: dict[str, Any], + service_owners: dict[str, str], + docker: str, + project: Path, + trivy_cache: Path, + vulnerability_waivers: dict[tuple[str, str], VulnerabilityWaiver], + runner: Callable[..., Any], +) -> list[dict[str, Any]]: + """Resolve and scan every rendered image remotely without building or pulling it.""" + service_results: list[dict[str, Any]] = [] + scan_results: dict[str, tuple[str | None, dict[str, Any] | None, bool]] = {} + docker_config = Path.home() / ".docker" / "config.json" + auth_mount = ( + ["-v", f"{docker_config.resolve()}:/root/.docker/config.json:ro"] + if docker_config.is_file() + else [] + ) + for service_name, service in compose_services.items(): + package = service_owners.get(service_name) + if not package: + raise ContainerCheckError( + f"generated Compose service '{service_name}' has no package owner" + ) + image = service.get("image") + if not image: + service_results.append( + { + "service": service_name, + "package": package, + "image": "", + "status": "failed", + "image_scan": "unavailable", + "detail": "remote image scan requires an explicit published image", + } + ) + continue + try: + image_id = _remote_image_reference( + docker, + image, + cwd=project, + runner=runner, + ) + except ContainerCheckError as exc: + service_results.append( + { + "service": service_name, + "package": package, + "image": image, + "status": "failed", + "image_scan": "unavailable", + "detail": str(exc), + } + ) + continue + + result: dict[str, Any] = { + "service": service_name, + "package": package, + "image": image, + "image_id": image_id, + "status": "pending", + "image_scan": "pending", + } + service_results.append(result) + if image_id not in scan_results: + scan_results[image_id] = _run_trivy_image_scan( + [ + docker, + "run", + "--rm", + *auth_mount, + "-v", + f"{trivy_cache.resolve()}:/root/.cache/trivy", + TRIVY_IMAGE, + "image", + "--image-src", + "remote", + "--quiet", + "--timeout", + "15m", + "--exit-code", + "1", + "--scanners", + "vuln", + "--severity", + "HIGH,CRITICAL", + "--format", + "json", + image_id, + ], + cwd=project, + runner=runner, + label=f"trivy image {image_id}", + ) + trivy_failure, evidence, waiver_eligible = scan_results[image_id] + if evidence is not None: + result.update(evidence) + if evidence["vulnerable_components"]: + result["vulnerability_evidence_sha256"] = _vulnerability_evidence_sha256(evidence) + waiver = vulnerability_waivers.get((service_name, image)) + waiver_matches = bool( + waiver and result.get("vulnerability_evidence_sha256") == waiver.evidence_sha256 + ) + if trivy_failure and waiver and waiver_eligible and waiver_matches: + result["status"] = "waived" + result["image_scan"] = "waived" + result["vulnerability_waiver"] = waiver.reason + result["detail"] = trivy_failure + else: + result["status"] = "failed" if trivy_failure else "passed" + result["image_scan"] = "failed" if trivy_failure else "passed" + if trivy_failure: + result["detail"] = trivy_failure + if waiver and waiver_eligible and not waiver_matches: + result["detail"] = _join_failure_details( + result["detail"], + "vulnerability waiver evidence does not match: " + f"expected {waiver.evidence_sha256}, observed " + f"{result.get('vulnerability_evidence_sha256', '')}", + ) + return service_results + + +def check_generated_containers( + *, + project_parent: Path | None = None, + service_files: dict[str, str] | None = None, + service_names: list[str] | None = None, + vulnerability_waivers: dict[tuple[str, str], VulnerabilityWaiver] | None = None, + command_runner: Callable[..., Any] | None = None, + remote_images: bool = False, +) -> dict[str, Any]: + """Generate a disposable user project and check only its generated files.""" + command_runner = command_runner or subprocess.run + vulnerability_waivers = vulnerability_waivers or {} + phlo = shutil.which("phlo") + docker = shutil.which("docker") + if not phlo: + raise ContainerCheckError("required installed CLI 'phlo' is not installed or not on PATH") + if not docker: + raise ContainerCheckError("required tool 'docker' is not installed or not on PATH") + + if service_files is None: + owners, discovered_service_names = _service_inventory() + else: + owners = service_files + discovered_service_names = service_names or [] + with tempfile.TemporaryDirectory(prefix="phlo-container-check-", dir=project_parent) as raw: + project = Path(raw) + project.mkdir(exist_ok=True) + configured_trivy_cache = os.environ.get("PHLO_TRIVY_CACHE_DIR") + trivy_cache = ( + Path(configured_trivy_cache).expanduser().resolve() + if configured_trivy_cache + else project / ".trivy-cache" + ) + trivy_cache.mkdir(parents=True, exist_ok=True) + init_command = [phlo, "services", "init", "--no-dev", "--force"] + _run_checked_command( + init_command, + cwd=project, + runner=command_runner, + label="phlo services init", + ) + if discovered_service_names: + add_command = [phlo, "services", "add"] + for service_name in discovered_service_names: + add_command.extend(["--service", service_name]) + add_command.append("--no-start") + _run_checked_command( + add_command, + cwd=project, + runner=command_runner, + label="phlo services add", + ) + + generated_root = project / ".phlo" + dockerfiles = ( + sorted(path for path in generated_root.rglob("Dockerfile") if path.is_file()) + if generated_root.exists() + else [] + ) + relative_dockerfiles = [str(path.relative_to(generated_root)) for path in dockerfiles] + unowned = [relative for relative in relative_dockerfiles if relative not in owners] + if unowned: + raise ContainerCheckError( + "generated Dockerfile(s) have no package owner: " + ", ".join(unowned) + ) + dockerfile_owners = { + relative: owners[relative] for relative in relative_dockerfiles if relative in owners + } + + failures: list[dict[str, str]] = [] + if dockerfiles: + for dockerfile in dockerfiles: + relative = str(dockerfile.relative_to(generated_root)) + failure = _run_command( + [ + docker, + "run", + "--rm", + "-v", + f"{project.resolve()}:/workspace:ro", + HADOLINT_IMAGE, + "/bin/hadolint", + f"/workspace/.phlo/{relative}", + ], + cwd=project, + runner=command_runner, + label=f"hadolint {relative}", + ) + if failure: + failures.append( + { + "tool": "hadolint", + "package": dockerfile_owners[relative], + "target": relative, + "detail": failure, + } + ) + + compose_file = generated_root / "docker-compose.yml" + compose_command = [ + docker, + "compose", + "--profile", + "*", + "-f", + str(compose_file), + "--project-directory", + str(generated_root), + "config", + "--format", + "json", + ] + compose_stdout, _ = _run_output_command( + compose_command, + cwd=project, + runner=command_runner, + label="docker compose config", + max_output_chars=MAX_COMPOSE_CONFIG_CHARS, + ) + try: + compose_config = json.loads(compose_stdout) + compose_services = compose_config["services"] + compose_project = compose_config["name"] + except (TypeError, KeyError, json.JSONDecodeError) as exc: + raise ContainerCheckError( + "docker compose config returned invalid service JSON" + ) from exc + + service_owners = { + name.removeprefix("@service:"): package + for name, package in owners.items() + if name.startswith("@service:") + } + service_results: list[dict[str, Any]] = [] + resolved_image_ids: dict[str, str] = {} + previous_image_ids: dict[str, str | None] = {} + image_scan_results: dict[str, tuple[str | None, dict[str, Any] | None, bool]] = {} + builder_name = f"phlo-check-{project.name.rsplit('-', 1)[-1]}" + uses_local_builds = not remote_images and any( + service.get("build") for service in compose_services.values() + ) + builder_created = False + if uses_local_builds: + _run_checked_command( + [ + docker, + "buildx", + "create", + "--driver", + "docker-container", + "--name", + builder_name, + ], + cwd=project, + runner=command_runner, + label="docker buildx create", + ) + builder_created = True + + builder_cleanup_failure: str | None = None + builder_cache_owner: tuple[str, str] | None = None + + def prune_builder_cache() -> None: + nonlocal builder_cache_owner + if builder_cache_owner is None: + return + package, image = builder_cache_owner + cache_cleanup_failure = _run_command( + [ + docker, + "buildx", + "prune", + "--builder", + builder_name, + "--force", + ], + cwd=project, + runner=command_runner, + label="docker buildx prune", + ) + if cache_cleanup_failure: + failures.append( + { + "tool": "docker cleanup", + "package": package, + "target": image, + "detail": cache_cleanup_failure, + } + ) + builder_cache_owner = None + + if remote_images: + service_results = _check_remote_service_images( + compose_services=compose_services, + service_owners=service_owners, + docker=docker, + project=project, + trivy_cache=trivy_cache, + vulnerability_waivers=vulnerability_waivers, + runner=command_runner, + ) + try: + services_to_build = () if remote_images else compose_services.items() + for service_name, service in services_to_build: + prune_builder_cache() + package = service_owners.get(service_name) + if not package: + raise ContainerCheckError( + f"generated Compose service '{service_name}' has no package owner" + ) + image = service.get("image") + locally_built = bool(service.get("build")) + build_failure: str | None = None + if locally_built and not image: + image = f"{compose_project}-{service_name}" + if not image: + service_results.append( + { + "service": service_name, + "package": package, + "image": "", + "status": "failed", + "image_scan": "unavailable", + "detail": "generated service has neither image nor build", + } + ) + continue + + image_id = resolved_image_ids.get(image) + first_resolution = image_id is None + if first_resolution: + try: + previous_image_ids[image] = _existing_image_id( + docker, + image, + cwd=project, + runner=command_runner, + ) + except ContainerCheckError as exc: + service_results.append( + { + "service": service_name, + "package": package, + "image": image, + "status": "failed", + "image_scan": "unavailable", + "detail": str(exc), + } + ) + continue + if locally_built and first_resolution: + build_failure = _run_command( + [ + docker, + "compose", + "--profile", + "*", + "-f", + str(compose_file), + "--project-directory", + str(generated_root), + "build", + "--builder", + builder_name, + "--quiet", + service_name, + ], + cwd=project, + runner=command_runner, + label=f"docker compose build {service_name}", + ) + builder_cache_owner = (package, image) + if first_resolution: + pull_failure = None + if not locally_built or build_failure: + pull_failure = _run_command( + [docker, "pull", image], + cwd=project, + runner=command_runner, + label=f"docker pull {service_name}", + ) + if pull_failure: + service_results.append( + { + "service": service_name, + "package": package, + "image": image, + "status": "failed", + "image_scan": "unavailable", + "detail": _join_failure_details(build_failure, pull_failure), + } + ) + continue + try: + inspect_stdout, _ = _run_output_command( + [docker, "image", "inspect", "--format", "{{.Id}}", image], + cwd=project, + runner=command_runner, + label=f"docker image inspect {service_name}", + ) + except ContainerCheckError as exc: + service_results.append( + { + "service": service_name, + "package": package, + "image": image, + "status": "failed", + "image_scan": "unavailable", + "detail": _join_failure_details(build_failure, str(exc)), + } + ) + continue + image_id = inspect_stdout.strip() + if not image_id: + service_results.append( + { + "service": service_name, + "package": package, + "image": image, + "status": "failed", + "image_scan": "unavailable", + "detail": _join_failure_details( + build_failure, "docker image inspect returned no image ID" + ), + } + ) + continue + + resolved_image_ids[image] = image_id + result: dict[str, Any] = { + "service": service_name, + "package": package, + "image": image, + "image_id": image_id, + "status": "failed" if build_failure else "pending", + "image_scan": "pending", + **({"detail": build_failure} if build_failure else {}), + } + service_results.append(result) + if image_id not in image_scan_results: + image_scan_results[image_id] = _run_trivy_image_scan( + [ + docker, + "run", + "--rm", + "-v", + "/var/run/docker.sock:/var/run/docker.sock", + "-v", + f"{trivy_cache.resolve()}:/root/.cache/trivy", + TRIVY_IMAGE, + "image", + "--quiet", + "--timeout", + "15m", + "--exit-code", + "1", + "--scanners", + "vuln", + "--severity", + "HIGH,CRITICAL", + "--format", + "json", + image_id, + ], + cwd=project, + runner=command_runner, + label=f"trivy image {image_id}", + ) + trivy_image_failure, vulnerability_evidence, waiver_eligible = image_scan_results[ + image_id + ] + if vulnerability_evidence is not None: + result.update(vulnerability_evidence) + if vulnerability_evidence["vulnerable_components"]: + result["vulnerability_evidence_sha256"] = _vulnerability_evidence_sha256( + vulnerability_evidence + ) + waiver = vulnerability_waivers.get((service_name, image)) + build_failed = result["status"] == "failed" + waiver_matches = bool( + waiver and result.get("vulnerability_evidence_sha256") == waiver.evidence_sha256 + ) + if trivy_image_failure and waiver and waiver_eligible and waiver_matches: + result["image_scan"] = "waived" + result["vulnerability_waiver"] = waiver.reason + result["detail"] = _join_failure_details( + result.get("detail"), trivy_image_failure + ) + result["status"] = "failed" if build_failed else "waived" + else: + result["status"] = "failed" if trivy_image_failure or build_failed else "passed" + result["image_scan"] = "failed" if trivy_image_failure else "passed" + if trivy_image_failure: + result["detail"] = _join_failure_details( + result.get("detail"), trivy_image_failure + ) + if waiver and waiver_eligible and not waiver_matches: + result["detail"] = _join_failure_details( + result.get("detail"), + "vulnerability waiver evidence does not match: " + f"expected {waiver.evidence_sha256}, observed " + f"{result.get('vulnerability_evidence_sha256', '')}", + ) + + if first_resolution: + previous_image_id = previous_image_ids[image] + cleanup_commands: list[tuple[list[str], str]] = [] + if previous_image_id and previous_image_id != image_id: + cleanup_commands.extend( + [ + ( + [docker, "image", "tag", previous_image_id, image], + f"docker image restore {service_name}", + ), + ( + [docker, "image", "rm", image_id], + f"docker image rm {service_name}", + ), + ] + ) + elif previous_image_id is None: + cleanup_commands.append( + ([docker, "image", "rm", image], f"docker image rm {service_name}") + ) + for cleanup_command, cleanup_label in cleanup_commands: + image_cleanup_failure = _run_command( + cleanup_command, + cwd=project, + runner=command_runner, + label=cleanup_label, + ) + if not image_cleanup_failure: + continue + failures.append( + { + "tool": "docker cleanup", + "package": package, + "target": image, + "detail": image_cleanup_failure, + } + ) + break + prune_builder_cache() + finally: + if builder_created: + builder_cleanup_failure = _run_command( + [docker, "buildx", "rm", "--force", builder_name], + cwd=project, + runner=command_runner, + label="docker buildx rm", + ) + if builder_cleanup_failure: + failures.append( + { + "tool": "docker cleanup", + "package": "project", + "target": builder_name, + "detail": builder_cleanup_failure, + } + ) + + trivy_failure = _run_command( + [ + docker, + "run", + "--rm", + "-v", + f"{project.resolve()}:/workspace:ro", + "-v", + f"{trivy_cache.resolve()}:/root/.cache/trivy", + TRIVY_IMAGE, + "config", + "--exit-code", + "1", + "--severity", + "HIGH,CRITICAL", + "/workspace/.phlo", + ], + cwd=project, + runner=command_runner, + label="trivy config", + ) + if trivy_failure: + failures.append( + { + "tool": "trivy", + "package": "project", + "target": ".phlo", + "detail": trivy_failure, + } + ) + reported_image_failures: set[str] = set() + for result in service_results: + if result["status"] in {"passed", "waived"}: + continue + failure_key = result.get("image_id", result["service"]) + if failure_key in reported_image_failures: + continue + reported_image_failures.add(failure_key) + failures.append( + { + "tool": "trivy image", + "package": result["package"], + "target": result["service"], + "detail": result.get("detail", "image scan failed"), + } + ) + missing_results = [ + result["service"] + for result in service_results + if result.get("image_scan") not in {"passed", "failed", "unavailable", "waived"} + ] + if missing_results: + failures.append( + { + "tool": "trivy image", + "package": "unknown", + "target": ", ".join(missing_results), + "detail": "generated service has no image-scan result", + } + ) + if failures: + lines = ["Generated container checks failed:"] + lines.extend( + f"- service [{result['package']}] {result['service']}: " + f"{result['image'] or ''} -> {result['status']} " + f"(image scan: {result.get('image_scan', 'missing')})" + for result in service_results + ) + lines.extend( + f"- {failure['tool']} [{failure['package']}] {failure['target']}: " + f"{failure['detail']}" + for failure in failures + ) + raise ContainerCheckError("\n".join(lines)) + + return { + "dockerfiles": relative_dockerfiles, + "owners": dockerfile_owners, + "hadolint": "passed" if dockerfiles else "skipped (no generated Dockerfiles)", + "trivy": ( + "passed with explicit vulnerability waiver(s)" + if any(result.get("image_scan") == "waived" for result in service_results) + else "passed" + ), + "services": service_results, + } + @click.command(name="check") @click.option( @@ -22,7 +1099,29 @@ default=False, help="Output as JSON", ) -def check_cmd(output_json: bool): +@click.option( + "--containers", + is_flag=True, + help="Generate a temporary user project and check its generated container files.", +) +@click.option( + "--remote-images", + is_flag=True, + help="Resolve and scan rendered registry images without building or pulling them.", +) +@click.option( + "--allow-vulnerable-image", + "vulnerability_waiver_values", + multiple=True, + metavar="SERVICE=IMAGE=EVIDENCE_SHA256=REASON", + help="Waive one exact HIGH/CRITICAL finding set for one generated service image.", +) +def check_cmd( + output_json: bool, + containers: bool, + remote_images: bool, + vulnerability_waiver_values: tuple[str, ...], +): """Validate installed plugins. Checks that all plugins comply with their interface requirements @@ -31,8 +1130,13 @@ def check_cmd(output_json: bool): Examples: phlo plugin check # Check all plugins phlo plugin check --json # Output as JSON + phlo plugin check --containers # Check generated container files + phlo plugin check --containers \\ + --allow-vulnerable-image SERVICE=IMAGE=EVIDENCE_SHA256=REASON """ try: + if remote_images and not containers: + raise click.UsageError("--remote-images requires --containers") if not output_json: console.print("Validating plugins...") @@ -42,6 +1146,12 @@ def check_cmd(output_json: bool): # Then validate validation_results = validate_plugins() + if containers: + validation_results["containers"] = check_generated_containers( + vulnerability_waivers=_parse_vulnerability_waivers(vulnerability_waiver_values), + remote_images=remote_images, + ) + if output_json: click.echo(json.dumps(validation_results, indent=2)) return @@ -69,10 +1179,31 @@ def check_cmd(output_json: bool): sys.exit(1) else: console.print("\n[green]All plugins are valid![/green]") + if containers: + checked = validation_results["containers"] + console.print( + f"\n[green]Generated container checks passed:[/green] " + f"{len(checked['dockerfiles'])} Dockerfile(s)" + ) + for service in checked["services"]: + if service["status"] == "waived": + waiver_line = Text(" ") + waiver_line.append("⚠ WAIVED", style="yellow") + waiver_line.append( + f" {service['package']} / {service['service']} → " + f"{service['image']}: {service['vulnerability_waiver']}" + ) + console.print(waiver_line) + console.print(Text(f" {service['detail']}", style="yellow")) + continue + console.print( + f" [green]✓[/green] {service['package']} / {service['service']} " + f"→ {service['image']} ({service['status']})" + ) except SystemExit: raise except Exception as e: logger.exception("plugin_check_failed", output_json=output_json) - console.print(f"[red]Error validating plugins: {e}[/red]") + console.print(f"Error validating plugins: {e}", style="red", markup=False) sys.exit(1) diff --git a/src/phlo/plugins/compose/generator.py b/src/phlo/plugins/compose/generator.py index 1acb5bd8230..956e19058c9 100644 --- a/src/phlo/plugins/compose/generator.py +++ b/src/phlo/plugins/compose/generator.py @@ -227,7 +227,7 @@ def _build_service_config( # Image or build if service.image: config["image"] = service.image - elif service.build: + if service.build: # Determine build context context = service.build.get("context", ".") if context == "source" and service.source_path: diff --git a/tests/cli/test_cli_plugin.py b/tests/cli/test_cli_plugin.py index ba549eb15e4..d832f986779 100644 --- a/tests/cli/test_cli_plugin.py +++ b/tests/cli/test_cli_plugin.py @@ -3,6 +3,7 @@ import json import sys +import click import pytest from click.testing import CliRunner @@ -176,6 +177,1132 @@ def test_plugin_check_json_emits_only_json(setup_registry): assert "invalid" in data +def test_plugin_check_containers_checks_generated_project(monkeypatch, setup_registry, tmp_path): + """Container checks run tools against files generated in an external project.""" + from phlo.cli.commands.plugin import check as check_module + + calls = [] + + def fake_run(command, **kwargs): + calls.append((command, kwargs)) + if command[0] == "/bin/phlo": + project = kwargs["cwd"] + dockerfile = project / ".phlo" / "dagster" / "Dockerfile" + dockerfile.parent.mkdir(parents=True, exist_ok=True) + dockerfile.write_text("FROM python:3.11\n") + if command[:3] == ["/bin/docker", "compose", "--profile"] and command[-2:] == [ + "--format", + "json", + ]: + return type( + "Result", + (), + { + "returncode": 0, + "stdout": json.dumps( + { + "name": "test-project", + "services": { + "dagster": {"image": "example/dagster:1"}, + "observatory": {"image": "example/observatory:1"}, + }, + } + ), + "stderr": "", + }, + )() + if command[:3] == ["/bin/docker", "image", "inspect"]: + return type("Result", (), {"returncode": 0, "stdout": "sha256:test\n", "stderr": ""})() + return type("Result", (), {"returncode": 0, "stdout": "", "stderr": ""})() + + monkeypatch.setattr(check_module.shutil, "which", lambda name: f"/bin/{name}") + monkeypatch.setattr(check_module.subprocess, "run", fake_run) + monkeypatch.setattr(check_module, "discover_plugins", lambda **_: {"service": []}) + monkeypatch.setattr(check_module, "validate_plugins", lambda: {"valid": [], "invalid": []}) + + result = check_module.check_generated_containers( + project_parent=tmp_path, + service_files={ + "dagster/Dockerfile": "phlo-dagster", + "@service:dagster": "phlo-dagster", + "@service:observatory": "phlo-observatory", + }, + service_names=["phlo-api", "observatory"], + ) + + assert result["dockerfiles"] == ["dagster/Dockerfile"] + assert result["owners"] == {"dagster/Dockerfile": "phlo-dagster"} + assert calls[0][0] == ["/bin/phlo", "services", "init", "--no-dev", "--force"] + assert calls[1][0] == [ + "/bin/phlo", + "services", + "add", + "--service", + "phlo-api", + "--service", + "observatory", + "--no-start", + ] + project_mount = f"{calls[0][1]['cwd'].resolve()}:/workspace:ro" + assert calls[2][0] == [ + "/bin/docker", + "run", + "--rm", + "-v", + project_mount, + check_module.HADOLINT_IMAGE, + "/bin/hadolint", + "/workspace/.phlo/dagster/Dockerfile", + ] + assert any( + check_module.TRIVY_IMAGE in command + and "image" in command + and "/var/run/docker.sock:/var/run/docker.sock" in command + for command, _ in calls + ) + assert calls[-1][0][-1] == "/workspace/.phlo" + assert result["services"] == [ + { + "service": "dagster", + "package": "phlo-dagster", + "image": "example/dagster:1", + "image_id": "sha256:test", + "status": "passed", + "image_scan": "passed", + "high_count": 0, + "critical_count": 0, + "vulnerable_components": [], + }, + { + "service": "observatory", + "package": "phlo-observatory", + "image": "example/observatory:1", + "image_id": "sha256:test", + "status": "passed", + "image_scan": "passed", + "high_count": 0, + "critical_count": 0, + "vulnerable_components": [], + }, + ] + + +def test_service_inventory_attributes_companion_service_files(monkeypatch, tmp_path): + """Files declared by companion service YAMLs retain package ownership.""" + from phlo.cli.commands.plugin import check as check_module + + package_root = tmp_path / "phlo_openmetadata" + package_root.mkdir() + plugin = type("Plugin", (), {"service_definition": {"name": "openmetadata"}})() + plugin.get_files = list + companion = type( + "Definition", + (), + { + "source_path": package_root / "openmetadata-elasticsearch-setup.yaml", + "files": [ + { + "source": "es.Dockerfile", + "dest": "openmetadata-elasticsearch/Dockerfile", + } + ], + }, + )() + + monkeypatch.setattr(check_module, "discover_plugins", lambda **_: {"service": [plugin]}) + monkeypatch.setattr(check_module, "_plugin_package", lambda _: "phlo-openmetadata") + monkeypatch.setattr(check_module, "resolve_plugin_source_path", lambda _: package_root) + monkeypatch.setattr( + check_module, + "ServiceDiscovery", + lambda: type( + "Discovery", + (), + {"discover": lambda self: {"openmetadata-elasticsearch": companion}}, + )(), + ) + + owners, _ = check_module._service_inventory() + + assert owners["openmetadata-elasticsearch/Dockerfile"] == "phlo-openmetadata" + + +def test_plugin_check_containers_reports_tool_failure(monkeypatch, tmp_path): + """A generated-container tool failure is reported as a CLI failure.""" + from phlo.cli.commands.plugin import check as check_module + + def fake_run(command, **kwargs): + return type("Result", (), {"returncode": 1, "stdout": "bad", "stderr": "failure"})() + + monkeypatch.setattr(check_module.shutil, "which", lambda name: f"/bin/{name}") + monkeypatch.setattr(check_module.subprocess, "run", fake_run) + + with pytest.raises(check_module.ContainerCheckError, match="phlo services init failed"): + check_module.check_generated_containers( + project_parent=tmp_path, + service_files={"dagster/Dockerfile": "phlo-dagster"}, + ) + + with pytest.raises(check_module.ContainerCheckError) as exc_info: + check_module.check_generated_containers( + project_parent=tmp_path, + service_files={"dagster/Dockerfile": "phlo-dagster"}, + ) + assert "stdout: bad" in str(exc_info.value) + assert "stderr: failure" in str(exc_info.value) + + +def test_plugin_check_containers_bounds_large_tool_failure_output(monkeypatch, tmp_path): + """Large scanner reports retain both streams without exhausting memory.""" + from phlo.cli.commands.plugin import check as check_module + + stdout = "stdout-start\n" + ("x" * (check_module.MAX_TOOL_OUTPUT_CHARS + 100)) + "\nstdout-end" + stderr = "stderr-start\n" + ("y" * (check_module.MAX_TOOL_OUTPUT_CHARS + 100)) + "\nstderr-end" + + def fake_run(command, **kwargs): + return type("Result", (), {"returncode": 1, "stdout": stdout, "stderr": stderr})() + + monkeypatch.setattr(check_module.shutil, "which", lambda name: f"/bin/{name}") + monkeypatch.setattr(check_module.subprocess, "run", fake_run) + + with pytest.raises(check_module.ContainerCheckError) as exc_info: + check_module.check_generated_containers( + project_parent=tmp_path, + service_files={"dagster/Dockerfile": "phlo-dagster"}, + ) + + message = str(exc_info.value) + assert "stdout-start" in message + assert "stdout-end" in message + assert "stderr-start" in message + assert "stderr-end" in message + assert "output truncated" in message + assert len(message) < check_module.MAX_TOOL_OUTPUT_CHARS * 3 + + +def test_trivy_image_scan_retains_a_parseable_bounded_json_report(monkeypatch, tmp_path): + """Trivy JSON gets a larger bounded capture than human-readable tool failures.""" + from phlo.cli.commands.plugin import check as check_module + + capture: dict[str, int] = {} + + def fake_capture(command, **kwargs): + capture["max_output_chars"] = kwargs["max_output_chars"] + return type( + "Result", + (), + {"returncode": 1, "stdout": '{"Results": []}', "stderr": "findings"}, + )() + + monkeypatch.setattr(check_module, "_run_with_capture", fake_capture) + + failure, evidence, waivable = check_module._run_trivy_image_scan( + ["docker", "run", "trivy", "image"], + cwd=tmp_path, + runner=object(), + label="trivy image test", + ) + + assert capture["max_output_chars"] == check_module.MAX_TRIVY_JSON_CHARS + assert capture["max_output_chars"] > check_module.MAX_TOOL_OUTPUT_CHARS + assert failure is not None + assert evidence == {"high_count": 0, "critical_count": 0, "vulnerable_components": []} + assert waivable is False + + +def test_plugin_check_containers_keeps_large_compose_config_parseable(monkeypatch, tmp_path): + """The generated service inventory is bounded separately from tool transcripts.""" + from phlo.cli.commands.plugin import check as check_module + + compose_config = { + "name": "test-project", + "services": { + "one": { + "image": "example/one:1", + "labels": {"padding": "x" * check_module.MAX_TOOL_OUTPUT_CHARS}, + } + }, + } + + def fake_run(command, **kwargs): + if command[:3] == ["/bin/docker", "compose", "--profile"]: + return type( + "Result", + (), + {"returncode": 0, "stdout": json.dumps(compose_config), "stderr": ""}, + )() + if command[:3] == ["/bin/docker", "image", "inspect"]: + return type("Result", (), {"returncode": 0, "stdout": "sha256:test\n", "stderr": ""})() + return type("Result", (), {"returncode": 0, "stdout": "", "stderr": ""})() + + monkeypatch.setattr(check_module.shutil, "which", lambda name: f"/bin/{name}") + + result = check_module.check_generated_containers( + project_parent=tmp_path, + service_files={"@service:one": "package-one"}, + command_runner=fake_run, + ) + + assert result["services"][0]["status"] == "passed" + + +def test_plugin_check_containers_builds_a_shared_exact_image_once(monkeypatch, tmp_path): + """Services sharing one exact build tag must retain the image ID that gets scanned.""" + from phlo.cli.commands.plugin import check as check_module + + calls: list[list[str]] = [] + + def fake_run(command, **kwargs): + calls.append(command) + if command[:3] == ["/bin/docker", "compose", "--profile"] and command[-2:] == [ + "--format", + "json", + ]: + return type( + "Result", + (), + { + "returncode": 0, + "stdout": json.dumps( + { + "name": "test-project", + "services": { + "server": { + "image": "example/server:1", + "build": {"context": "."}, + }, + "setup": { + "image": "example/server:1", + "build": {"context": "."}, + }, + }, + } + ), + "stderr": "", + }, + )() + if command[:3] == ["/bin/docker", "image", "inspect"]: + return type( + "Result", (), {"returncode": 0, "stdout": "sha256:shared\n", "stderr": ""} + )() + return type("Result", (), {"returncode": 0, "stdout": "", "stderr": ""})() + + monkeypatch.setattr(check_module.shutil, "which", lambda name: f"/bin/{name}") + + result = check_module.check_generated_containers( + project_parent=tmp_path, + service_files={ + "@service:server": "package-server", + "@service:setup": "package-server", + }, + command_runner=fake_run, + ) + + build_calls = [command for command in calls if "build" in command] + assert len(build_calls) == 1 + assert [service["status"] for service in result["services"]] == ["passed", "passed"] + + +def test_plugin_check_containers_scans_each_build_before_starting_the_next(monkeypatch, tmp_path): + """Generated builds use disposable cache and get scanned before disk use accumulates.""" + from phlo.cli.commands.plugin import check as check_module + + calls: list[list[str]] = [] + inspect_counts: dict[str, int] = {} + + def fake_run(command, **kwargs): + calls.append(command) + if command[:3] == ["/bin/docker", "compose", "--profile"] and command[-2:] == [ + "--format", + "json", + ]: + return type( + "Result", + (), + { + "returncode": 0, + "stdout": json.dumps( + { + "name": "test-project", + "services": { + "one": { + "image": "example/one:1", + "build": {"context": "one"}, + }, + "two": { + "image": "example/two:1", + "build": {"context": "two"}, + }, + }, + } + ), + "stderr": "", + }, + )() + if command[:3] == ["/bin/docker", "image", "inspect"]: + image = command[-1] + inspect_counts[image] = inspect_counts.get(image, 0) + 1 + if inspect_counts[image] == 1: + return type( + "Result", (), {"returncode": 1, "stdout": "", "stderr": "No such image"} + )() + return type( + "Result", + (), + {"returncode": 0, "stdout": f"sha256:{image.split('/')[1][:-2]}\n", "stderr": ""}, + )() + return type("Result", (), {"returncode": 0, "stdout": "", "stderr": ""})() + + monkeypatch.setattr(check_module.shutil, "which", lambda name: f"/bin/{name}") + + check_module.check_generated_containers( + project_parent=tmp_path, + service_files={ + "@service:one": "package-one", + "@service:two": "package-two", + }, + command_runner=fake_run, + ) + + build_calls = [command for command in calls if "compose" in command and "build" in command] + assert len(build_calls) == 2 + assert all("--builder" in command for command in build_calls) + first_scan = next( + index + for index, command in enumerate(calls) + if check_module.TRIVY_IMAGE in command and "sha256:one" in command + ) + second_build = calls.index(build_calls[1]) + assert first_scan < second_build + create_call = next(command for command in calls if command[1:3] == ["buildx", "create"]) + prune_calls = [command for command in calls if command[1:3] == ["buildx", "prune"]] + remove_call = next(command for command in calls if command[1:3] == ["buildx", "rm"]) + assert len(prune_calls) == 2 + assert first_scan < calls.index(prune_calls[0]) < second_build + assert all( + command + == [ + "/bin/docker", + "buildx", + "prune", + "--builder", + create_call[create_call.index("--name") + 1], + "--force", + ] + for command in prune_calls + ) + assert create_call[create_call.index("--name") + 1] == remove_call[-1] + + +def test_plugin_check_containers_remote_mode_never_builds_or_pulls(monkeypatch, tmp_path) -> None: + """Published images resolve to immutable digests and scan without local image storage.""" + from phlo.cli.commands.plugin import check as check_module + + calls: list[list[str]] = [] + + def fake_run(command, **kwargs): + calls.append(command) + if command[:3] == ["/bin/docker", "compose", "--profile"]: + return type( + "Result", + (), + { + "returncode": 0, + "stdout": json.dumps( + { + "name": "test-project", + "services": { + "one": { + "image": "ghcr.io/phlohouse/phlo-one:1.2.3", + "build": {"context": "one"}, + } + }, + } + ), + "stderr": "", + }, + )() + if command[1:4] == ["buildx", "imagetools", "inspect"]: + return type( + "Result", + (), + {"returncode": 0, "stdout": f'"sha256:{"a" * 64}"\n', "stderr": ""}, + )() + return type("Result", (), {"returncode": 0, "stdout": "", "stderr": ""})() + + monkeypatch.setattr(check_module.shutil, "which", lambda name: f"/bin/{name}") + + result = check_module.check_generated_containers( + project_parent=tmp_path, + service_files={"@service:one": "package-one"}, + command_runner=fake_run, + remote_images=True, + ) + + assert not any("build" in command for command in calls if "compose" in command) + assert not any(command[1:2] == ["pull"] for command in calls) + assert not any(command[1:3] == ["image", "inspect"] for command in calls) + remote_scan = next( + command + for command in calls + if check_module.TRIVY_IMAGE in command and "--image-src" in command + ) + assert remote_scan[remote_scan.index("--image-src") + 1] == "remote" + assert remote_scan[-1] == f"ghcr.io/phlohouse/phlo-one@sha256:{'a' * 64}" + assert result["services"][0]["image_id"] == remote_scan[-1] + + +def test_plugin_check_containers_removes_only_images_created_for_the_check(monkeypatch, tmp_path): + """Temporary validation images are removed without touching pre-existing tags.""" + from phlo.cli.commands.plugin import check as check_module + + calls: list[list[str]] = [] + new_image_inspects = 0 + + def fake_run(command, **kwargs): + nonlocal new_image_inspects + calls.append(command) + if command[:3] == ["/bin/docker", "compose", "--profile"] and command[-2:] == [ + "--format", + "json", + ]: + return type( + "Result", + (), + { + "returncode": 0, + "stdout": json.dumps( + { + "name": "test-project", + "services": { + "new": { + "image": "example/new:1", + "build": {"context": "."}, + }, + "existing": {"image": "example/existing:1"}, + }, + } + ), + "stderr": "", + }, + )() + if command[:3] == ["/bin/docker", "image", "inspect"]: + image = command[-1] + if image == "example/new:1": + new_image_inspects += 1 + if new_image_inspects == 1: + return type( + "Result", (), {"returncode": 1, "stdout": "", "stderr": "No such image"} + )() + image_id = "sha256:new" + else: + image_id = "sha256:existing" + return type("Result", (), {"returncode": 0, "stdout": f"{image_id}\n", "stderr": ""})() + return type("Result", (), {"returncode": 0, "stdout": "", "stderr": ""})() + + monkeypatch.setattr(check_module.shutil, "which", lambda name: f"/bin/{name}") + + check_module.check_generated_containers( + project_parent=tmp_path, + service_files={ + "@service:new": "package-new", + "@service:existing": "package-existing", + }, + command_runner=fake_run, + ) + + remove_calls = [command for command in calls if command[1:3] == ["image", "rm"]] + assert ["/bin/docker", "image", "rm", "example/new:1"] in remove_calls + assert ["/bin/docker", "image", "rm", "example/existing:1"] not in remove_calls + + +def test_plugin_check_containers_restores_preexisting_local_image_tag(monkeypatch, tmp_path): + """A validation build cannot replace an image tag that the operator already had.""" + from phlo.cli.commands.plugin import check as check_module + + calls: list[list[str]] = [] + inspect_count = 0 + + def fake_run(command, **kwargs): + nonlocal inspect_count + calls.append(command) + if command[:3] == ["/bin/docker", "compose", "--profile"] and command[-2:] == [ + "--format", + "json", + ]: + return type( + "Result", + (), + { + "returncode": 0, + "stdout": json.dumps( + { + "name": "test-project", + "services": { + "existing": { + "image": "example/existing:1", + "build": {"context": "."}, + } + }, + } + ), + "stderr": "", + }, + )() + if command[:3] == ["/bin/docker", "image", "inspect"]: + inspect_count += 1 + image_id = "sha256:original" if inspect_count == 1 else "sha256:validation" + return type("Result", (), {"returncode": 0, "stdout": f"{image_id}\n", "stderr": ""})() + return type("Result", (), {"returncode": 0, "stdout": "", "stderr": ""})() + + monkeypatch.setattr(check_module.shutil, "which", lambda name: f"/bin/{name}") + + check_module.check_generated_containers( + project_parent=tmp_path, + service_files={"@service:existing": "package-existing"}, + command_runner=fake_run, + ) + + assert [ + "/bin/docker", + "image", + "tag", + "sha256:original", + "example/existing:1", + ] in calls + assert ["/bin/docker", "image", "rm", "sha256:validation"] in calls + + +def test_plugin_check_containers_restores_preexisting_pulled_image_tag(monkeypatch, tmp_path): + """A validation pull cannot refresh an image tag that the operator already had.""" + from phlo.cli.commands.plugin import check as check_module + + calls: list[list[str]] = [] + inspect_count = 0 + + def fake_run(command, **kwargs): + nonlocal inspect_count + calls.append(command) + if command[:3] == ["/bin/docker", "compose", "--profile"] and command[-2:] == [ + "--format", + "json", + ]: + return type( + "Result", + (), + { + "returncode": 0, + "stdout": json.dumps( + { + "name": "test-project", + "services": {"remote": {"image": "example/remote:latest"}}, + } + ), + "stderr": "", + }, + )() + if command[:3] == ["/bin/docker", "image", "inspect"]: + inspect_count += 1 + image_id = "sha256:original" if inspect_count == 1 else "sha256:pulled" + return type("Result", (), {"returncode": 0, "stdout": f"{image_id}\n", "stderr": ""})() + return type("Result", (), {"returncode": 0, "stdout": "", "stderr": ""})() + + monkeypatch.setattr(check_module.shutil, "which", lambda name: f"/bin/{name}") + + check_module.check_generated_containers( + project_parent=tmp_path, + service_files={"@service:remote": "package-remote"}, + command_runner=fake_run, + ) + + assert [ + "/bin/docker", + "image", + "tag", + "sha256:original", + "example/remote:latest", + ] in calls + assert ["/bin/docker", "image", "rm", "sha256:pulled"] in calls + + +def test_existing_image_lookup_fails_closed_on_inspect_error(tmp_path) -> None: + """A Docker inspect outage cannot be mistaken for an absent operator image.""" + from phlo.cli.commands.plugin import check as check_module + + def fake_run(command, **kwargs): + return type( + "Result", + (), + {"returncode": 1, "stdout": "", "stderr": "daemon unavailable"}, + )() + + with pytest.raises(check_module.ContainerCheckError, match="daemon unavailable"): + check_module._existing_image_id( + "/bin/docker", + "example/operator:1", + cwd=tmp_path, + runner=fake_run, + ) + + +def test_plugin_check_containers_reuses_configured_trivy_cache(monkeypatch, tmp_path): + """A configured cache survives the generated project cleanup for reuse.""" + from phlo.cli.commands.plugin import check as check_module + + cache_dir = tmp_path / "trivy-cache" + calls = [] + + def fake_run(command, **kwargs): + calls.append(command) + if command[:3] == ["/bin/docker", "compose", "--profile"]: + return type( + "Result", + (), + { + "returncode": 0, + "stdout": json.dumps( + {"name": "test-project", "services": {"one": {"image": "example/one:1"}}} + ), + "stderr": "", + }, + )() + if command[:3] == ["/bin/docker", "image", "inspect"]: + return type("Result", (), {"returncode": 0, "stdout": "sha256:test\n", "stderr": ""})() + return type("Result", (), {"returncode": 0, "stdout": "", "stderr": ""})() + + monkeypatch.setenv("PHLO_TRIVY_CACHE_DIR", str(cache_dir)) + monkeypatch.setattr(check_module.shutil, "which", lambda name: f"/bin/{name}") + + check_module.check_generated_containers( + project_parent=tmp_path, + service_files={"@service:one": "package-one"}, + command_runner=fake_run, + ) + + assert cache_dir.is_dir() + assert any( + f"{cache_dir.resolve()}:/root/.cache/trivy" in command + for command in calls + if command[0] == "/bin/docker" + ) + trivy_image_command = next( + command for command in calls if check_module.TRIVY_IMAGE in command and "image" in command + ) + assert trivy_image_command[ + trivy_image_command.index("--timeout") : trivy_image_command.index("--timeout") + 2 + ] == ["--timeout", "15m"] + + +def test_plugin_check_containers_requires_installed_cli(monkeypatch, tmp_path): + from phlo.cli.commands.plugin import check as check_module + + monkeypatch.setattr( + check_module.shutil, "which", lambda name: None if name == "phlo" else f"/bin/{name}" + ) + + with pytest.raises(check_module.ContainerCheckError, match="installed CLI 'phlo'"): + check_module.check_generated_containers(project_parent=tmp_path, service_files={}) + + +def test_plugin_check_containers_rejects_unowned_dockerfile(monkeypatch, tmp_path): + """Generated Dockerfiles without discovered package ownership fail closed.""" + from phlo.cli.commands.plugin import check as check_module + + def fake_run(command, **kwargs): + dockerfile = kwargs["cwd"] / ".phlo" / "unknown" / "Dockerfile" + dockerfile.parent.mkdir(parents=True, exist_ok=True) + dockerfile.write_text("FROM python:3.11\n") + return type("Result", (), {"returncode": 0, "stdout": "", "stderr": ""})() + + monkeypatch.setattr(check_module.shutil, "which", lambda name: f"/bin/{name}") + + with pytest.raises(check_module.ContainerCheckError, match="no package owner"): + check_module.check_generated_containers( + project_parent=tmp_path, + service_files={}, + command_runner=fake_run, + ) + + +def test_plugin_check_containers_reports_all_package_failures(monkeypatch, tmp_path): + """All generated package failures are reported after every scanner runs.""" + from phlo.cli.commands.plugin import check as check_module + + calls = [] + + def fake_run(command, **kwargs): + calls.append(command) + if command[0] == "/bin/phlo": + for name in ("one", "two"): + dockerfile = kwargs["cwd"] / ".phlo" / name / "Dockerfile" + dockerfile.parent.mkdir(parents=True, exist_ok=True) + dockerfile.write_text("FROM python:3.11\n") + return type("Result", (), {"returncode": 0, "stdout": "", "stderr": ""})() + if command[:3] == ["/bin/docker", "compose", "--profile"] and command[-2:] == [ + "--format", + "json", + ]: + return type( + "Result", + (), + { + "returncode": 0, + "stdout": json.dumps( + { + "name": "test-project", + "services": { + "one": {"image": "example/one:1"}, + "two": {"image": "example/two:1"}, + }, + } + ), + "stderr": "", + }, + )() + return type("Result", (), {"returncode": 1, "stdout": "", "stderr": "failed"})() + + monkeypatch.setattr(check_module.shutil, "which", lambda name: f"/bin/{name}") + + with pytest.raises(check_module.ContainerCheckError) as exc_info: + check_module.check_generated_containers( + project_parent=tmp_path, + service_files={ + "one/Dockerfile": "package-one", + "two/Dockerfile": "package-two", + "@service:one": "package-one", + "@service:two": "package-two", + }, + command_runner=fake_run, + ) + + message = str(exc_info.value) + assert "package-one" in message + assert "package-two" in message + assert "trivy [project]" in message + assert "stdout:" not in message + assert [command[0] for command in calls].count("/bin/phlo") == 1 + assert [command[0] for command in calls].count("/bin/docker") >= 5 + + +def test_plugin_check_containers_scans_original_after_wrapper_build_failure(monkeypatch, tmp_path): + """A wrapper failure still gets an attributed scan of the resolved base image.""" + from phlo.cli.commands.plugin import check as check_module + + def fake_run(command, **kwargs): + if command[0] == "/bin/phlo": + dockerfile = kwargs["cwd"] / ".phlo" / "one" / "Dockerfile" + dockerfile.parent.mkdir(parents=True, exist_ok=True) + dockerfile.write_text("FROM python:3.11\n") + return type("Result", (), {"returncode": 0, "stdout": "", "stderr": ""})() + if command[:3] == ["/bin/docker", "compose", "--profile"] and command[-2:] == [ + "--format", + "json", + ]: + return type( + "Result", + (), + { + "returncode": 0, + "stdout": json.dumps( + { + "name": "test-project", + "services": { + "one": { + "image": "example/one:1", + "build": {"context": "."}, + } + }, + } + ), + "stderr": "", + }, + )() + if "build" in command and command[-1] == "one": + return type( + "Result", (), {"returncode": 1, "stdout": "build output", "stderr": "build failed"} + )() + if command[:2] == ["/bin/docker", "pull"]: + return type("Result", (), {"returncode": 0, "stdout": "", "stderr": ""})() + if command[:3] == ["/bin/docker", "image", "inspect"]: + return type("Result", (), {"returncode": 0, "stdout": "sha256:base\n", "stderr": ""})() + if "image" in command: + return type( + "Result", (), {"returncode": 1, "stdout": "trivy output", "stderr": "trivy error"} + )() + return type("Result", (), {"returncode": 0, "stdout": "", "stderr": ""})() + + monkeypatch.setattr(check_module.shutil, "which", lambda name: f"/bin/{name}") + + with pytest.raises(check_module.ContainerCheckError) as exc_info: + check_module.check_generated_containers( + project_parent=tmp_path, + service_files={ + "one/Dockerfile": "package-one", + "@service:one": "package-one", + }, + command_runner=fake_run, + ) + + message = str(exc_info.value) + assert "service [package-one] one: example/one:1 -> failed (image scan: failed)" in message + assert "build output" in message + assert "trivy error" in message + assert "no image-scan result" not in message + + +def test_plugin_check_containers_reports_exact_image_vulnerability_waiver(monkeypatch, tmp_path): + """An exact service/image waiver is visible and does not hide other failures.""" + from phlo.cli.commands.plugin import check as check_module + + def fake_run(command, **kwargs): + if command[0] == "/bin/phlo": + (kwargs["cwd"] / ".phlo").mkdir(parents=True, exist_ok=True) + return type("Result", (), {"returncode": 0, "stdout": "", "stderr": ""})() + if command[:3] == ["/bin/docker", "compose", "--profile"] and command[-2:] == [ + "--format", + "json", + ]: + return type( + "Result", + (), + { + "returncode": 0, + "stdout": json.dumps( + { + "name": "test-project", + "services": {"one": {"image": "example/one:1"}}, + } + ), + "stderr": "", + }, + )() + if command[:2] == ["/bin/docker", "pull"]: + return type("Result", (), {"returncode": 0, "stdout": "", "stderr": ""})() + if command[:3] == ["/bin/docker", "image", "inspect"]: + return type("Result", (), {"returncode": 0, "stdout": "sha256:one\n", "stderr": ""})() + if "image" in command: + return type( + "Result", + (), + { + "returncode": 1, + "stdout": json.dumps( + { + "Results": [ + { + "Target": "one-binary", + "Class": "lang-pkgs", + "Type": "gobinary", + "Vulnerabilities": [ + { + "VulnerabilityID": "CVE-TEST", + "PkgName": "example/component", + "InstalledVersion": "1.0.0", + "FixedVersion": "1.0.1", + "Severity": "HIGH", + } + ], + } + ] + } + ), + "stderr": "", + }, + )() + return type("Result", (), {"returncode": 0, "stdout": "", "stderr": ""})() + + monkeypatch.setattr(check_module.shutil, "which", lambda name: f"/bin/{name}") + + result = check_module.check_generated_containers( + project_parent=tmp_path, + service_files={"@service:one": "package-one"}, + vulnerability_waivers={ + ("one", "example/one:1"): check_module.VulnerabilityWaiver( + evidence_sha256="8f315d5e0ddd6c0d0c830665f6b519de3c1ace3cc7386651ebb0bee566fcad61", + reason="No patched upstream release is available", + ) + }, + command_runner=fake_run, + ) + + assert result["trivy"] == "passed with explicit vulnerability waiver(s)" + assert result["services"] == [ + { + "service": "one", + "package": "package-one", + "image": "example/one:1", + "image_id": "sha256:one", + "status": "waived", + "image_scan": "waived", + "high_count": 1, + "critical_count": 0, + "vulnerable_components": [ + { + "target": "one-binary", + "class": "lang-pkgs", + "type": "gobinary", + "component": "example/component", + "installed_version": "1.0.0", + "fixed_version": "1.0.1", + "vulnerability_id": "CVE-TEST", + "severity": "HIGH", + } + ], + "vulnerability_waiver": "No patched upstream release is available", + "vulnerability_evidence_sha256": ( + "8f315d5e0ddd6c0d0c830665f6b519de3c1ace3cc7386651ebb0bee566fcad61" + ), + "detail": ( + "trivy image sha256:one failed with exit code 1: " + 'stdout: {"Results": [{"Target": "one-binary", "Class": "lang-pkgs", ' + '"Type": "gobinary", "Vulnerabilities": [{"VulnerabilityID": "CVE-TEST", ' + '"PkgName": "example/component", "InstalledVersion": "1.0.0", ' + '"FixedVersion": "1.0.1", "Severity": "HIGH"}]}]}' + ), + } + ] + + +def test_plugin_check_rejects_ambiguous_vulnerability_waiver() -> None: + """Waivers must bind a service and image to exact vulnerability evidence.""" + from phlo.cli.commands.plugin import check as check_module + + with pytest.raises(click.BadParameter, match="SERVICE=IMAGE=EVIDENCE_SHA256=REASON"): + check_module._parse_vulnerability_waivers(("one=example/one:1",)) + + +def test_plugin_check_containers_is_available_at_public_cli_seam(monkeypatch, setup_registry): + """The public check command exposes generated-container results as JSON.""" + from phlo.cli.commands.plugin import check as check_module + + monkeypatch.setattr( + check_module, + "check_generated_containers", + lambda **_: { + "dockerfiles": ["dagster/Dockerfile"], + "owners": {"dagster/Dockerfile": "phlo-dagster"}, + }, + ) + + result = CliRunner().invoke(plugin_group, ["check", "--containers", "--json"]) + + assert result.exit_code == 0 + assert json.loads(result.output)["containers"]["owners"] == { + "dagster/Dockerfile": "phlo-dagster" + } + + +def test_plugin_check_containers_forwards_exact_vulnerability_waiver( + monkeypatch, setup_registry +) -> None: + """The public CLI must pass an exact image waiver to the container checker.""" + from phlo.cli.commands.plugin import check as check_module + + received: dict[str, object] = {} + + def fake_check_generated_containers(**kwargs): + received.update(kwargs) + return {"dockerfiles": [], "owners": {}, "services": []} + + monkeypatch.setattr(check_module, "check_generated_containers", fake_check_generated_containers) + + result = CliRunner().invoke( + plugin_group, + [ + "check", + "--containers", + "--json", + "--allow-vulnerable-image", + "alloy=phlo/alloy:v1.18.0-go1.26.5=" + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa=" + "no compatible upstream fix", + ], + ) + + assert result.exit_code == 0 + assert received["vulnerability_waivers"] == { + ("alloy", "phlo/alloy:v1.18.0-go1.26.5"): check_module.VulnerabilityWaiver( + evidence_sha256="a" * 64, + reason="no compatible upstream fix", + ) + } + + +def test_plugin_check_containers_forwards_remote_image_mode(monkeypatch, setup_registry) -> None: + """CI can scan published image digests without rebuilding generated services.""" + from phlo.cli.commands.plugin import check as check_module + + received: dict[str, object] = {} + + def fake_check_generated_containers(**kwargs): + received.update(kwargs) + return {"dockerfiles": [], "owners": {}, "services": []} + + monkeypatch.setattr(check_module, "check_generated_containers", fake_check_generated_containers) + + result = CliRunner().invoke( + plugin_group, + ["check", "--containers", "--remote-images", "--json"], + ) + + assert result.exit_code == 0 + assert received["remote_images"] is True + + +def test_plugin_check_containers_preserves_package_owner_in_failure_output( + monkeypatch, setup_registry +) -> None: + """Rich output must not consume bracketed package ownership as markup.""" + from phlo.cli.commands.plugin import check as check_module + + monkeypatch.setattr( + check_module, + "check_generated_containers", + lambda **_: (_ for _ in ()).throw( + check_module.ContainerCheckError( + "Generated container checks failed:\n" + "- service [package-one] one: example/one:1 -> failed" + ) + ), + ) + + result = CliRunner().invoke(plugin_group, ["check", "--containers"]) + + assert result.exit_code == 1 + assert "[package-one]" in result.output + + +def test_plugin_check_containers_prints_waived_scanner_detail_without_markup( + monkeypatch, setup_registry +) -> None: + """Scanner paths that resemble Rich closing tags must print literally.""" + from phlo.cli.commands.plugin import check as check_module + + monkeypatch.setattr( + check_module, + "check_generated_containers", + lambda **_: { + "dockerfiles": [], + "owners": {}, + "services": [ + { + "status": "waived", + "package": "phlo-clickstack", + "service": "clickstack", + "image": "example/clickstack:1", + "vulnerability_waiver": "No compatible upstream fix", + "detail": "scanner target [/var/lib/clickhouse]", + } + ], + }, + ) + + result = CliRunner().invoke(plugin_group, ["check", "--containers"]) + + assert result.exit_code == 0 + assert "scanner target [/var/lib/clickhouse]" in result.output + + def test_plugin_list_all_json(setup_registry, monkeypatch): """List command includes registry plugins when --all is set.""" registry_plugins = [ diff --git a/tests/cli/test_cli_services_init.py b/tests/cli/test_cli_services_init.py index 98f3c5dbaa6..e9d4b306685 100644 --- a/tests/cli/test_cli_services_init.py +++ b/tests/cli/test_cli_services_init.py @@ -2,6 +2,7 @@ import os import re +from pathlib import Path from typing import cast import click @@ -18,6 +19,24 @@ from tests.helpers import FakeDiscovery, _service +def test_bundled_service_images_have_explicit_default_versions() -> None: + """Generated service defaults must never rely on an implicit or moving latest tag.""" + repository = Path(__file__).parents[2] + unpinned: list[str] = [] + + for service_file in sorted(repository.glob("packages/phlo-*/src/**/*.yaml")): + image = yaml.safe_load(service_file.read_text()).get("image") + if not image: + continue + default_image = re.sub(r"^\$\{[^:}]+:-([^}]+)\}$", r"\1", image) + if ( + ":" not in default_image.rsplit("/", 1)[-1] and "@sha256:" not in default_image + ) or default_image.endswith(":latest"): + unpinned.append(f"{service_file.relative_to(repository)}: {image}") + + assert unpinned == [] + + def test_production_credentials_reject_defaults_and_require_safe_usernames() -> None: from phlo.cli.commands.services.init import _validate_production_credentials @@ -97,6 +116,20 @@ def test_conditional_environment_creates_an_absent_environment(tmp_path) -> None assert data["services"]["conditional"]["environment"] == {"CONDITIONAL_VALUE": "enabled"} +def test_image_services_use_exact_upstream_image_without_fake_wrapper(tmp_path) -> None: + service = _service("postgres", default=True) + service.image = "postgres:18-alpine" + generator = ComposeGenerator(cast(ServiceDiscovery, FakeDiscovery({service.name: service}))) + + data = yaml.safe_load(generator.generate_compose([service], output_dir=tmp_path)) + copied = generator.copy_service_files([service], tmp_path) + + assert data["services"]["postgres"]["image"] == "postgres:18-alpine" + assert "build" not in data["services"]["postgres"] + assert copied == [] + assert not (tmp_path / "images" / "postgres" / "Dockerfile").exists() + + def test_conditional_environment_supports_list_environment_and_user_overrides(tmp_path) -> None: service = ServiceDefinition( name="conditional", diff --git a/tests/scripts/test_generated_image_matrix.py b/tests/scripts/test_generated_image_matrix.py new file mode 100644 index 00000000000..aa839dc4ff3 --- /dev/null +++ b/tests/scripts/test_generated_image_matrix.py @@ -0,0 +1,140 @@ +"""Tests for the generated GHCR publication matrix.""" + +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[2] +_spec = importlib.util.spec_from_file_location( + "generated_image_matrix", REPO_ROOT / "scripts" / "generated_image_matrix.py" +) +assert _spec and _spec.loader +generated_image_matrix = importlib.util.module_from_spec(_spec) +sys.modules["generated_image_matrix"] = generated_image_matrix +_spec.loader.exec_module(generated_image_matrix) + + +def test_publication_matrix_deduplicates_shared_images(tmp_path: Path) -> None: + context = tmp_path / ".phlo" + context.mkdir() + dockerfile = context / "openmetadata" / "Dockerfile" + dockerfile.parent.mkdir() + dockerfile.touch() + build = { + "context": str(context), + "dockerfile": "openmetadata/Dockerfile", + "args": {"VERSION": "1.2.3"}, + } + compose = { + "services": { + "openmetadata": { + "image": "ghcr.io/phlohouse/phlo-openmetadata:1.2.3", + "build": build, + }, + "openmetadata-setup": { + "image": "ghcr.io/phlohouse/phlo-openmetadata:1.2.3", + "build": build, + }, + } + } + + assert generated_image_matrix.publication_matrix(compose, tmp_path) == { + "include": [ + { + "service": "openmetadata", + "services": ["openmetadata", "openmetadata-setup"], + "image": "ghcr.io/phlohouse/phlo-openmetadata:1.2.3", + "root": "generated", + "context": ".phlo", + "dockerfile": ".phlo/openmetadata/Dockerfile", + "build_args": {"VERSION": "1.2.3"}, + } + ] + } + + +def test_publication_matrix_rejects_unpublished_build(tmp_path: Path) -> None: + compose = { + "services": { + "local": { + "image": "local/image:1", + "build": {"context": str(tmp_path)}, + } + } + } + + with pytest.raises(ValueError, match="has no Phlo GHCR image"): + generated_image_matrix.publication_matrix(compose, tmp_path) + + +def test_publication_matrix_accepts_checked_out_source_context(tmp_path: Path) -> None: + project = tmp_path / "project" + source = tmp_path / "source" + context = source / "packages" / "observatory" + context.mkdir(parents=True) + (context / "Dockerfile").touch() + compose = { + "services": { + "observatory": { + "image": "ghcr.io/phlohouse/phlo-observatory:0.7.0", + "build": {"context": str(context), "dockerfile": "Dockerfile"}, + } + } + } + + target = generated_image_matrix.publication_matrix(compose, project, source)["include"][0] + + assert target["root"] == "source" + assert target["context"] == "packages/observatory" + + +def test_publication_matrix_selects_only_requested_shared_image(tmp_path: Path) -> None: + build = {"context": str(tmp_path), "dockerfile": "Dockerfile"} + (tmp_path / "Dockerfile").touch() + compose = { + "services": { + "dagster": { + "image": "ghcr.io/phlohouse/phlo-dagster:1", + "build": build, + }, + "dagster-daemon": { + "image": "ghcr.io/phlohouse/phlo-dagster:1", + "build": build, + }, + "postgres": { + "image": "ghcr.io/phlohouse/phlo-postgres:1", + "build": build, + }, + } + } + + matrix = generated_image_matrix.publication_matrix( + compose, + tmp_path, + selected_services={"dagster-daemon"}, + ) + + assert [target["service"] for target in matrix["include"]] == ["dagster"] + + +def test_publication_matrix_rejects_unknown_selected_service(tmp_path: Path) -> None: + (tmp_path / "Dockerfile").touch() + compose = { + "services": { + "postgres": { + "image": "ghcr.io/phlohouse/phlo-postgres:1", + "build": {"context": str(tmp_path)}, + } + } + } + + with pytest.raises(ValueError, match="unknown"): + generated_image_matrix.publication_matrix( + compose, + tmp_path, + selected_services={"unknown"}, + ) diff --git a/tests/scripts/test_recovery_drill.py b/tests/scripts/test_recovery_drill.py index e573b0a4d82..72b0488d7f0 100644 --- a/tests/scripts/test_recovery_drill.py +++ b/tests/scripts/test_recovery_drill.py @@ -22,9 +22,10 @@ def test_compose_uses_isolated_ports_and_supported_stack_images(tmp_path): compose = recovery_drill.compose_yaml(stack) - assert "postgres:16-alpine" in compose + assert "postgres:18-alpine" in compose assert "minio/minio:RELEASE.2025-09-07T16-13-09Z" in compose - assert "ghcr.io/projectnessie/nessie:0.107.2" in compose + assert "ghcr.io/projectnessie/nessie:0.108.3" in compose + assert "postgres-data:/var/lib/postgresql" in compose assert "127.0.0.1::5432" in compose assert "condition: service_healthy" in compose assert "jdbc:postgresql://postgres:5432/phlo?currentSchema=public" in compose diff --git a/tests/scripts/test_release_golden_path.py b/tests/scripts/test_release_golden_path.py index 0bdc6d33a8d..70f73c08786 100644 --- a/tests/scripts/test_release_golden_path.py +++ b/tests/scripts/test_release_golden_path.py @@ -407,6 +407,37 @@ def urlopen(request, **_kwargs): assert requests[1].get_header("Authorization") == f"Bearer {config.report_token}" +def test_rejected_wap_report_waits_for_rejection_projection(tmp_path: Path, monkeypatch) -> None: + config = _config(tmp_path) + wap_run = release_golden_path.WapRun("rejected", "dagster-rejected") + reports = iter( + ( + { + "run_id": "rejected", + "quality": [{"blocking": True, "passed": False}], + "catalog_changes": [], + }, + { + "run_id": "rejected", + "quality": [{"blocking": True, "passed": False}], + "catalog_changes": [{"merge_outcome": "rejected_quality"}], + }, + ) + ) + monkeypatch.setattr(release_golden_path, "fetch_run_report", lambda *_: next(reports)) + monkeypatch.setattr(release_golden_path.time, "sleep", lambda _: None) + monkeypatch.setattr(release_golden_path, "service_url", lambda *_: "http://dagster/graphql") + monkeypatch.setattr(release_golden_path, "service_token", lambda *_: "service-token") + monkeypatch.setattr(release_golden_path, "wap_service_secret", lambda _: "secret") + monkeypatch.setattr( + release_golden_path, + "graphql", + lambda *_: {"data": {"pipelineRunOrError": {"tags": []}}}, + ) + + release_golden_path.verify_rejected_wap_report(config, wap_run) + + def test_rejected_wap_report_requires_failed_quality_and_rejection_evidence( tmp_path: Path, monkeypatch ) -> None: @@ -763,7 +794,7 @@ def test_dagster_build_receives_a_local_wheelhouse_arg() -> None: assert "PHLO_WHEELHOUSE: ${PHLO_WHEELHOUSE:-}" in service assert "PHLO_WHEELHOUSE: ${PHLO_WHEELHOUSE:-}" in daemon assert 'ARG PHLO_WHEELHOUSE=""' in dockerfile - assert "FROM python:3.12-slim AS phlo-build-context" in dockerfile + assert "FROM python:3.12-alpine AS phlo-build-context" in dockerfile assert "COPY . ." in dockerfile assert "RUN mkdir -p /opt/phlo-build-context/wheelhouse" in dockerfile assert ( @@ -792,6 +823,15 @@ def test_dagster_build_receives_a_local_wheelhouse_arg() -> None: assert "--no-index --no-deps --reinstall --find-links" in api_dockerfile +def test_dagster_stable_version_install_keeps_base_requirements_unconditional() -> None: + dockerfile = (REPO_ROOT / "packages/phlo-dagster/src/phlo_dagster/Dockerfile").read_text() + + assert 'base_requirements=("phlo[defaults]==$PHLO_VERSION"' in dockerfile + assert 'base_requirements+=("${prerelease_requirements[@]}")' in dockerfile + assert 'uv pip install --system --prerelease explicit "${base_requirements[@]}"' in dockerfile + assert 'if [ -n "$PHLO_PRERELEASE_REQUIREMENTS" ]; then' in dockerfile + + def test_configure_non_dev_compose_uses_docker_ephemeral_ports(tmp_path: Path, monkeypatch) -> None: config = _config(tmp_path) config.project_dir.mkdir() diff --git a/tests/tooling/test_generated_image_publication.py b/tests/tooling/test_generated_image_publication.py new file mode 100644 index 00000000000..0eab5873256 --- /dev/null +++ b/tests/tooling/test_generated_image_publication.py @@ -0,0 +1,78 @@ +"""Contracts for generated service image publication and remote CI scanning.""" + +from __future__ import annotations + +import re +from pathlib import Path + +import yaml + +REPO_ROOT = Path(__file__).resolve().parents[2] + + +def _published_image(raw_image: str) -> str: + match = re.fullmatch(r"\$\{[^:}]+:-(.+)}", raw_image) + return match.group(1) if match else raw_image + + +def test_every_generated_build_uses_a_versioned_ghcr_image() -> None: + build_definitions: list[tuple[Path, str]] = [] + for service_file in sorted((REPO_ROOT / "packages").glob("*/src/*/*.yaml")): + service = yaml.safe_load(service_file.read_text(encoding="utf-8")) + if not isinstance(service, dict) or not service.get("build"): + continue + image = service.get("image") + assert isinstance(image, str), f"{service_file} has a build but no image" + build_definitions.append((service_file, _published_image(image))) + + assert build_definitions + for service_file, image in build_definitions: + assert image.startswith("ghcr.io/phlohouse/phlo-"), service_file + assert ":" in image.rsplit("/", 1)[-1], service_file + assert not image.endswith(":latest"), service_file + + +def test_publication_workflow_publishes_only_on_release_or_manual_dispatch() -> None: + workflow = (REPO_ROOT / ".github/workflows/build-core-services.yml").read_text(encoding="utf-8") + + assert "pull_request:" not in workflow + assert "\n push:" not in workflow + assert "workflow_dispatch:" in workflow + assert "release:" in workflow + assert "PUBLISH_SERVICES" in workflow + assert "packages: write" in workflow + assert "docker/build-push-action@" in workflow + assert "ubuntu-24.04-arm" in workflow + assert "platform: linux/amd64" in workflow + assert "platform: linux/arm64" in workflow + assert "setup-qemu-action@" not in workflow + assert "push-by-digest=true" in workflow + assert "name=${{ steps.image.outputs.repository }},push-by-digest=true" in workflow + assert "name=${{ matrix.target.image }},push-by-digest=true" not in workflow + assert "if: always() && needs.prepare.result == 'success'" in workflow + assert "name: digest-${{ matrix.target.service }}-amd64" in workflow + assert "name: digest-${{ matrix.target.service }}-arm64" in workflow + assert "pattern: digest-${{ matrix.target.service }}-*" not in workflow + assert "docker buildx imagetools create" in workflow + assert "timeout-minutes: 45" in workflow + assert "max-parallel: 8" in workflow + assert "org.opencontainers.image.source=https://github.com/${{ github.repository }}" in workflow + + +def test_ci_scans_published_images_remotely() -> None: + workflow = (REPO_ROOT / ".github/workflows/ci.yml").read_text(encoding="utf-8") + checker = (REPO_ROOT / "src/phlo/cli/commands/plugin/check.py").read_text(encoding="utf-8") + + command = re.search(r"phlo --no-color plugin check --containers[^\n]*", workflow) + assert command + assert "--remote-images" in command.group(0) + trivy_image = re.search(r'TRIVY_IMAGE = \(\n\s+"([^"]+)"', checker) + assert trivy_image + trivy_pull = f"docker pull {trivy_image.group(1)}" + assert trivy_pull in workflow + assert workflow.index(trivy_pull) < workflow.index(command.group(0)) + clickstack_waiver = ( + "clickstack=ghcr.io/phlohouse/phlo-clickstack:2.31.0-security-patches=" + "adc46e9eb99e4f3c0ea11a6cab55ebf0d720c844128a33b5bb8c7c7efae79224" + ) + assert clickstack_waiver in workflow diff --git a/uv.lock b/uv.lock index a058c12ce7e..e75b98ee0fc 100644 --- a/uv.lock +++ b/uv.lock @@ -1626,14 +1626,14 @@ wheels = [ [[package]] name = "gitpython" -version = "3.1.53" +version = "3.1.55" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "gitdb" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/17/24/0e0c12cb6f7cb864779a9d2fefee9ca91838f6db402c8780c9d28a8d7ebe/gitpython-3.1.53.tar.gz", hash = "sha256:06ae8d9623b0ed0d67b8adeac5c7008d0a5a404b087a9e0d0c7163bdd3a6b497", size = 224597, upload-time = "2026-07-20T13:41:52.839Z" } +sdist = { url = "https://files.pythonhosted.org/packages/b2/ab/ba0d29f2fa2277ed6256b2ac09003494045355f3a10bf32f351761287870/gitpython-3.1.55.tar.gz", hash = "sha256:781e3b1624dad81b24e9524bf0297b69786a0706db2cbceec1e2b05c38e5152f", size = 225071, upload-time = "2026-07-23T02:52:43.246Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/cf/a6/bff12b3238885eeef7d28ef908b24e0cba91c476c31cb876a00a0986ce2c/gitpython-3.1.53-py3-none-any.whl", hash = "sha256:187885556b64ab357bd4ea84e2c4cce2861a613a7f4268b3f7f7ba05f2ce4ab0", size = 216237, upload-time = "2026-07-20T13:41:51.473Z" }, + { url = "https://files.pythonhosted.org/packages/20/6a/d3b8208d2f8aac66abe8ccc1c23fa2c89464ec42cc71a601e95d05902428/gitpython-3.1.55-py3-none-any.whl", hash = "sha256:7c9ec1e69c158c081632ab35c41471e302c96db2ae42165036a5d2403378812e", size = 216590, upload-time = "2026-07-23T02:52:41.932Z" }, ] [[package]] @@ -3664,7 +3664,8 @@ source = { editable = "packages/phlo-iceberg" } dependencies = [ { name = "pandera" }, { name = "phlo" }, - { name = "pyiceberg", extra = ["pyarrow", "s3fs"] }, + { name = "pyarrow" }, + { name = "pyiceberg", extra = ["s3fs"] }, ] [package.optional-dependencies] @@ -3685,7 +3686,8 @@ requires-dist = [ { name = "phlo", editable = "." }, { name = "phlo-minio", marker = "extra == 'minio'", editable = "packages/phlo-minio" }, { name = "phlo-nessie", marker = "extra == 'nessie'", editable = "packages/phlo-nessie" }, - { name = "pyiceberg", extras = ["s3fs", "pyarrow"], specifier = ">=0.11.0" }, + { name = "pyarrow", specifier = ">=21.0.0" }, + { name = "pyiceberg", extras = ["s3fs"], specifier = ">=0.11.0" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=7.0" }, { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.1.0" }, ] @@ -4773,27 +4775,10 @@ wheels = [ ] [package.optional-dependencies] -pyarrow = [ - { name = "pyarrow" }, - { name = "pyiceberg-core" }, -] s3fs = [ { name = "s3fs" }, ] -[[package]] -name = "pyiceberg-core" -version = "0.8.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/9c/a0/0bcedbbe901484aacb6c605505f8574fd65954826e592fdb163e1cfb09f2/pyiceberg_core-0.8.0.tar.gz", hash = "sha256:59021ca5bc7ca95f2b06fb0730280fb3f60ed898060bcd874c156d093853b5f3", size = 618882, upload-time = "2026-01-20T00:50:40.076Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/77/e0/9a8fa537d29d34e3265682056d6517b926975107b5b1af6057d1713557d6/pyiceberg_core-0.8.0-cp310-abi3-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:d60c75a741a1d9199277a9e50fc3adbc84ab286a881f9b1f721fa120e7197912", size = 24733948, upload-time = "2026-01-20T00:50:20.566Z" }, - { url = "https://files.pythonhosted.org/packages/6d/3e/f5522c1e9c20c3e89bfd76b2f54ba38e57389e5a2872233e49e60a131e04/pyiceberg_core-0.8.0-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1d71e566b2d56141760ff8734667eede5a5d60963dfbcdce80c2dd3cf2edb39d", size = 11682041, upload-time = "2026-01-20T00:50:22.843Z" }, - { url = "https://files.pythonhosted.org/packages/95/4b/f799e5c7a2b2ede75514e64901503358a7a134ca1ea217fd86535af533b6/pyiceberg_core-0.8.0-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:82782d1b974200c5526d069391ba2bc235a868b5d0d6ac17ca406df735ab89a3", size = 13835428, upload-time = "2026-01-20T00:50:25.021Z" }, - { url = "https://files.pythonhosted.org/packages/0b/ff/2dbd6f7c99a2f782f908be2cc997371de45cc1df61abeeff1fc0165c05b6/pyiceberg_core-0.8.0-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:0e14b2aea26293ba5878c398adc880fff0f1ce5d989e00d4b1a930c143541114", size = 14580807, upload-time = "2026-01-20T00:50:27.158Z" }, - { url = "https://files.pythonhosted.org/packages/bc/13/176c2b00a9b804af79d8b697ba1a2525f4390e959be777076972071ca069/pyiceberg_core-0.8.0-cp310-abi3-win_amd64.whl", hash = "sha256:a5726cc62f9ac2582a0d5dde92e4140b711b5e29ec0c6c636d6d2782d984031b", size = 13354110, upload-time = "2026-01-20T00:50:29.785Z" }, -] - [[package]] name = "pyjwt" version = "2.13.0"