diff --git a/.github/workflows/build-azure-capi-image.yml b/.github/workflows/build-azure-capi-image.yml new file mode 100644 index 0000000000..8181274c69 --- /dev/null +++ b/.github/workflows/build-azure-capi-image.yml @@ -0,0 +1,91 @@ +name: Build Azure CAPI VM image + +on: + workflow_dispatch: + inputs: + version: + description: Kuberentes version + required: true + type: string + tag: + description: ck8s-capi tag + required: true + type: string + builder_image: + description: image builder image + required: true + type: string + default: "ghcr.io/elastisys/image-builder-amd64:main" + + workflow_call: + inputs: + version: + description: Kubernetes version + required: true + type: string + tag: + description: ck8s-capi tag + required: true + type: string + builder_image: + description: image builder image + required: true + type: string + default: "ghcr.io/elastisys/image-builder-amd64:main" + +env: + version: ${{ inputs.version }} + tag: ${{ inputs.tag }} + docker_image: ${{ inputs.builder_image }} + +defaults: + run: + working-directory: ./images/capi + shell: bash + +jobs: + build-image: + runs-on: ubuntu-24.04 + steps: + - name: Checkout repo + uses: actions/checkout@v5 + + - name: replace variables + run: | + package="${version}-1.1" + series="${version%.*}" + + sed -r \ + -e "s/\\\$KUBERNETES_SERIES/${series}/" \ + -e "s/\\\$KUBERNETES_VERSION/${version}/" \ + -e "s/\\\$KUBERNETES_DEB_VERSION/${package}/" \ + -e "s/\\\$IMAGE_TAG/${tag}/" \ + <"template.json" >"kubernetes.json" + + - name: build azure image + run: | + image_name="ubuntu-2404-kube-${version%%-*}-ck8s-capi-${tag}" + + export SIG_IMAGE_DEFINITION="${image_name}" + export SIG_PUBLISHER="elastisys" + export SIG_OFFER="ck8s-capi" + export SIG_SKU="${image_name}" + + docker run -i --rm \ + -e PACKER_VAR_FILES -e PACKER_GITHUB_API_TOKEN=${{ secrets.GITHUB_TOKEN }} \ + -e SIG_IMAGE_DEFINITION -e SIG_PUBLISHER -e SIG_OFFER -e SIG_SKU \ + -e AZURE_SUBSCRIPTION_ID -e AZURE_CLIENT_ID -e AZURE_CLIENT_SECRET -e AZURE_TENANT_ID -e AZURE_LOCATION \ + -e RESOURCE_GROUP_NAME -e GALLERY_NAME -e BUILD_RESOURCE_GROUP_NAME \ + -v ${{ github.workspace }}/images/capi:/tmp/host \ + ${{ env.docker_image }} build-azure-sig-ubuntu-2404-gen2 + + env: + PACKER_VAR_FILES: /tmp/host/kubernetes.json + AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID}} + AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} + AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }} + AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} + AZURE_LOCATION: ${{ secrets.AZURE_LOCATION }} + RESOURCE_GROUP_NAME: ${{ secrets.RESOURCE_GROUP_NAME }} + GALLERY_NAME: ${{ secrets.GALLERY_NAME }} + BUILD_RESOURCE_GROUP_NAME: ${{ secrets.RESOURCE_GROUP_NAME }} diff --git a/.github/workflows/build-capi-vm-images.yml b/.github/workflows/build-capi-vm-images.yml new file mode 100644 index 0000000000..b1887d1d90 --- /dev/null +++ b/.github/workflows/build-capi-vm-images.yml @@ -0,0 +1,38 @@ +name: Build CAPI VM image with manual input + +on: + workflow_dispatch: + inputs: + version: + description: k8s version + required: true + type: string + default: "1.33.1" + tag: + description: ck8s capi version + required: true + type: string + default: "0.8" + builder_image: + description: image builder image + required: true + type: string + default: "ghcr.io/elastisys/image-builder-amd64:main" + +env: + PACKER_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }} + +jobs: + build-azure-image: + uses: ./.github/workflows/build-azure-capi-image.yml + with: + version: ${{ inputs.version }} + tag: ${{ inputs.tag }} + docker_image: ${{ inputs.docker_image }} + secrets: inherit + build-openstack-image: + uses: ./.github/workflows/build-openstack-capi-image.yml + with: + version: ${{ inputs.version }} + tag: ${{ inputs.tag }} + docker_image: ${{ inputs.docker_image }} diff --git a/.github/workflows/build-image-builder.yml b/.github/workflows/build-image-builder.yml new file mode 100644 index 0000000000..4c09bb6bfe --- /dev/null +++ b/.github/workflows/build-image-builder.yml @@ -0,0 +1,33 @@ +name: Build CAPI image builder + +on: + push: + branches: + - main + +env: + IMAGE_NAME: image-builder + REGISTRY: ghcr.io/elastisys + +jobs: + build-image-builder: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v5 + + - name: "Login to GitHub Container Registry" + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: run make docker-build + run: make docker-build + env: + TAG: main + + - name: run make docker-push + run: make docker-push + env: + TAG: main diff --git a/.github/workflows/build-openstack-capi-image.yml b/.github/workflows/build-openstack-capi-image.yml new file mode 100644 index 0000000000..c639a6dc3a --- /dev/null +++ b/.github/workflows/build-openstack-capi-image.yml @@ -0,0 +1,93 @@ +name: Build OpenStack VM CAPI image + +on: + workflow_dispatch: + inputs: + version: + description: Kubernetes version + required: true + type: string + tag: + description: ck8s-capi tag + required: true + type: string + builder_image: + description: image builder image + required: true + type: string + default: "ghcr.io/elastisys/image-builder-amd64:main" + + workflow_call: + inputs: + version: + description: Kubernetes version + required: true + type: string + tag: + description: ck8s-capi tag + required: true + type: string + builder_image: + description: image builder image + required: true + type: string + default: "ghcr.io/elastisys/image-builder-amd64:main" + + +env: + version: ${{ inputs.version }} + tag: ${{ inputs.tag }} + docker_image: ${{ inputs.builder_image }} + +defaults: + run: + working-directory: ./images/capi + shell: bash + +jobs: + build-image: + runs-on: ubuntu-24.04 + + steps: + - uses: actions/checkout@v5 + + - name: run patchs + run: | + git apply patches/dockerfile.patch + + - name: Enable KVM + run: | + echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules + sudo udevadm control --reload-rules + sudo udevadm trigger --name-match=kvm + + - name: replace variables + run: | + package="${version}-1.1" + series="${version%.*}" + + sed -r \ + -e "s/\\\$KUBERNETES_SERIES/${series}/" \ + -e "s/\\\$KUBERNETES_VERSION/${version}/" \ + -e "s/\\\$KUBERNETES_DEB_VERSION/${package}/" \ + -e "s/\\\$IMAGE_TAG/${tag}/" \ + <"template.json" >"kubernetes.json" + + - name: add user + run: | + mkdir -p ${{ github.workspace }}/output + sudo useradd -ms /bin/bash imagebuilder + sudo chmod -R 777 ${{ github.workspace }}/output + + - name: build openstack image + run: | + docker run --device=/dev/kvm -i --rm \ + -e PACKER_VAR_FILES=/tmp/host/kubernetes.json -e PACKER_LOG -e PACKER_GITHUB_API_TOKEN=${{ secrets.GITHUB_TOKEN }} \ + -v ${{ github.workspace }}/images/capi:/tmp/host -v ${{ github.workspace }}/output:/home/imagebuilder/output:rw \ + ${{ env.docker_image }} build-qemu-ubuntu-2404-efi + + - name: store openstack image + uses: actions/upload-artifact@v4 + with: + name: ubuntu-2404-efi-kube-${{ env.version }}-ck8s-capi-${{ env.tag }} + path: ${{ github.workspace }}/output/ubuntu-2404-efi-kube-${{ env.version }}-ck8s-capi-${{ env.tag }} diff --git a/images/capi/.dockerignore b/images/capi/.dockerignore index e474674ca1..565293fe80 100644 --- a/images/capi/.dockerignore +++ b/images/capi/.dockerignore @@ -9,3 +9,4 @@ !packer !Makefile !azure_targets.sh +!template.json diff --git a/images/capi/ansible/roles/kubernetes/files/etc/audit-policy/apiserver-audit-policy.yaml b/images/capi/ansible/roles/kubernetes/files/etc/audit-policy/apiserver-audit-policy.yaml new file mode 100644 index 0000000000..e6ff29ee89 --- /dev/null +++ b/images/capi/ansible/roles/kubernetes/files/etc/audit-policy/apiserver-audit-policy.yaml @@ -0,0 +1,641 @@ +apiVersion: audit.k8s.io/v1 +kind: Policy +rules: + # The following requests were manually identified as high-volume and low-risk, + # so drop them. + - level: None + users: ["system:kube-proxy"] + verbs: ["watch"] + resources: + - group: "" # core + resources: ["endpoints", "services", "services/status"] + - level: None + userGroups: ["system:nodes"] + verbs: ["get"] + resources: + - group: "" # core + resources: ["nodes", "nodes/status"] + - level: None + users: + - system:kube-controller-manager + - system:kube-scheduler + - system:serviceaccount:kube-system:endpoint-controller + verbs: ["get", "update"] + namespaces: ["kube-system"] + resources: + - group: "" # core + resources: ["endpoints"] + - level: None + users: ["system:apiserver"] + verbs: ["get"] + resources: + - group: "" # core + resources: ["namespaces", "namespaces/status", "namespaces/finalize"] + # Don't log HPA fetching metrics. + - level: None + users: + - system:kube-controller-manager + verbs: ["get", "list"] + resources: + - group: "metrics.k8s.io" + # Don't log these read-only URLs. + - level: None + nonResourceURLs: + - /healthz* + - /version + - /swagger* + - /readyz* + - /livez* + # Don't log events requests. + - level: None + resources: + - group: "" # core + resources: ["events"] + + ## Leases ## + + # Don't log lease requests + - level: None + resources: + - group: "coordination.k8s.io" + resources: ["leases"] + + ## Starboard ## + + # Don't log watch requests + - level: None + verbs: ["watch"] + users: + - system:serviceaccount:monitoring:starboard-operator + resources: + - group: "" # core + resources: ["configmaps", "secrets", "pods", "serviceaccounts", "replicationcontrollers", "nodes"] + - group: "aquasecurity.github.io" + resources: ["ciskubebenchreports", "vulnerabilityreports"] + - group: "apps" + - group: "batch" + + # Request responses are large, skip them + - level: Request + verbs: ["get", "list", "watch"] + resources: + - group: "aquasecurity.github.io" + omitStages: + - "RequestReceived" + + ## Gatekeeper ## + + - level: None + verbs: ["watch"] + users: + - system:serviceaccount:gatekeeper-system:gatekeeper-admin + resources: + - group: "" # core + resources: ["namespaces"] + - group: "status.gatekeeper.sh" + - group: "config.gatekeeper.sh" + - group: "externaldata.gatekeeper.sh" + - group: "constraints.gatekeeper.sh" + - group: "admissionregistration.k8s.io" + - group: "mutations.gatekeeper.sh" + - group: "templates.gatekeeper.sh" + - group: "apiextensions.k8s.io" + - group: "admissionregistration.k8s.io" + - group: "networking.k8s.io" + resources: ["networkpolicies"] + + - level: None + verbs: ["watch"] + namespaces: ["gatekeeper-system"] + users: + - system:serviceaccount:gatekeeper-system:gatekeeper-admin + resources: + - group: "" + resources: ["secrets"] + + - level: None + verbs: ["get"] + users: + - system:serviceaccount:gatekeeper-system:gatekeeper-admin + resources: + - group: "constraints.gatekeeper.sh" + - group: "" # core + resources: ["namespaces"] + - group: "apiextensions.k8s.io" + + - level: None + verbs: ["get"] + users: + - system:serviceaccount:gatekeeper-system:gatekeeper-admin + resources: + - group: "constraints.gatekeeper.sh" + - group: "" # core + resources: ["pods", "replicationcontrollers"] + - group: "batch" + - group: "apps" + + # Don't log update status requests + - level: None + verbs: ["update"] + users: + - system:serviceaccount:gatekeeper-system:gatekeeper-admin + resources: + - group: status.gatekeeper.sh + resources: ["constraintpodstatuses"] + - group: constraints.gatekeeper.sh + + ## Ingress-nginx ## + + # Don't log requests to the leader configmap + - level: None + users: + - system:serviceaccount:ingress-nginx:ingress-nginx + namespaces: ["ingress-nginx"] + resources: + - group: "" # core + resources: ["configmaps"] + + # Don't log get nodes requests + - level: None + verbs: ["get"] + users: + - system:serviceaccount:ingress-nginx:ingress-nginx + resources: + - group: "" # core + resources: ["nodes"] + + # Don't log list controller pods requests + - level: None + verbs: ["list"] + namespaces: ["ingress-nginx"] + users: + - system:serviceaccount:ingress-nginx:ingress-nginx + resources: + - group: "" # core + resources: ["pods"] + + - level: None + verbs: ["watch"] + users: + - system:serviceaccount:ingress-nginx:ingress-nginx + resources: + - group: "networking.k8s.io" + resources: ["ingresses", "ingressclasses"] + - group: "" # core + resources: ["endpoints", "secrets", "configmaps", "services"] + + ## Nodes ## + + # Don't log node status patch request + - level: None + userGroups: ["system:nodes"] + verbs: ["patch"] + resources: + - group: "" # core + resources: ["nodes", "nodes/status"] + + # Don't log watch requests by system:nodes + - level: None + userGroups: ["system:nodes"] + verbs: ["watch"] + + ## Calico ## + + # Don't log get pod requests from calico-node + - level: None + verbs: ["get"] + users: + - system:serviceaccount:kube-system:calico-node + resources: + - group: "" # core + resources: ["pods"] + + # Don't log watch requests + - level: None + verbs: ["watch"] + users: + - system:serviceaccount:kube-system:calico-node + resources: + - group: "" # core + resources: ["endpoints", "nodes", "services", "namespaces", "serviceaccounts"] + - group: "crd.projectcalico.org" + - group: "networking.k8s.io" + resources: ["networkpolicies"] + + # Ignore get node requests from controller + - level: None + verbs: ["get"] + users: + - system:serviceaccount:kube-system:calico-kube-controllers + resources: + - group: "" # core + resources: ["nodes"] + + ## Openstack cloud controller ## + + - level: None + verbs: ["watch"] + namespaces: ["kube-system"] + users: + - system:serviceaccount:kube-system:cloud-controller-manager + resources: + - group: "" # core + resources: ["configmaps"] + + - level: None + verbs: ["watch"] + users: + - system:serviceaccount:kube-system:cloud-controller-manager + resources: + - group: "" # core + resources: ["nodes"] + + ## Openstack cinder-csi ## + + - level: None + verbs: ["watch"] + users: + - system:serviceaccount:kube-system:csi-cinder-controller-sa + resources: + - group: "" # core + resources: ["nodes", "persistentvolumeclaims"] + - group: "storage.k8s.io" + + ## Jaeger-operator ## + + - level: None + verbs: ["get", "list"] + users: + - system:serviceaccount:jaeger-system:jaeger-operator-rbac-sa + + ## Generic-garbage-collector ## + + - level: None + verbs: ["get"] + users: + - system:serviceaccount:kube-system:generic-garbage-collector + + ## DNS ## + + - level: None + verbs: ["watch"] + users: + - system:serviceaccount:kube-system:coredns + resources: + - group: "" # core + resources: ["services", "namespaces"] + - group: "discovery.k8s.io" + resources: ["endpointslices"] + + # Don't log gets from dns-autoscaler + - level: None + verbs: ["get"] + namespaces: ["kube-system"] + users: + - system:serviceaccount:kube-system:dns-autoscaler + resources: + - group: "" # core + resources: ["configmaps"] + - group: "apps" + resources: ["deployments/scale"] + + ## Cert-manager ## + + # Cert-manager, kube-state-metrics + - level: None + verbs: ["watch"] + users: + - system:serviceaccount:cert-manager:cert-manager-cainjector + - system:serviceaccount:cert-manager:cert-manager + - system:serviceaccount:cert-manager:cert-manager-webhook + resources: + - group: "" # core + resources: ["secrets", "services", "pods"] + - group: "cert-manager.io" + - group: "apiextensions.k8s.io" + - group: "acme.cert-manager.io" + - group: "admissionregistration.k8s.io" + - group: "apiregistration.k8s.io" + + ## Kube-state-metrics ## + + - level: None + verbs: ["watch"] + users: + - system:serviceaccount:monitoring:kube-prometheus-stack-kube-state-metrics + resources: + - group: "certificates.k8s.io" + resources: ["certificatesigningrequests"] + - group: "" # core + - group: "networking.k8s.io" + resources: ["ingresses", "networkpolicies"] + - group: "batch" + - group: "storage.k8s.io" + resources: ["storageclasses", "volumeattachments"] + + ## Prometheus ## + + # Prometheus watches + - level: None + verbs: ["watch"] + resources: + - group: "" # core + resources: ["pods", "services", "endpoints"] + users: + - system:serviceaccount:monitoring:kube-prometheus-stack-prometheus + + # Don't log watch request from prometheus-operator + - level: None + verbs: ["watch"] + users: + - system:serviceaccount:monitoring:kube-prometheus-stack-operator + resources: + - group: "" # core + resources: ["secrets", "configmaps", "namespaces"] + - group: "apps" + resources: ["statefulsets"] + - group: "monitoring.coreos.com" + + # Don't log get and updates to kube-system/kube-prometheus-stack-kubelet + - level: None + verbs: ["get", "update"] + namespacees: ["kube-system"] + users: + - system:serviceaccount:monitoring:kube-prometheus-stack-operator + resources: + - group: "" # core + resources: ["services", "endpoints"] + + # Don't log list nodes + - level: None + verbs: ["list"] + users: + - system:serviceaccount:monitoring:kube-prometheus-stack-operator + resources: + - group: "" # core + resources: ["nodes"] + + ## Fluentd ## + + # Don't log read requests from fluentd + - level: None + verbs: ["get", "watch", "list"] + users: + - system:serviceaccount:fluentd:fluentd-fluentd-elasticsearch + - system:serviceaccount:fluentd-system:fluentd-forwarder + - system:serviceaccount:kube-system:fluentd-system-fluentd-elasticsearch + resources: + - group: "" # core + resources: ["pods", "namespaces"] + + ## API server ## + + - level: None + verbs: ["get", "list"] + namespaces: ["default"] + users: + - system:apiserver + resources: + - group: "" # core + resources: ["services", "endpoints", "resourcequotas"] + - group: "discovery.k8s.io" + resources: ["endpointslices"] + + ## HNC ## + + # Don't log watch requests from controller + - level: None + verbs: ["watch"] + users: + - system:serviceaccount:hnc-system:hnc-controller + resources: + - group: "" # core + resources: ["namespaces"] + - group: "rbac.authorization.k8s.io" + resources: ["rolebindings", "roles"] + - group: "apiextensions.k8s.io" + resources: ["customresourcedefinitions"] + - group: "hnc.x-k8s.io" + - group: "networking.k8s.io" + resources: ["networkpolicies"] + + ## Rook-ceph ## + + - level: None + verbs: ["watch", "list", "get"] + users: + - system:serviceaccount:rook-ceph:rook-ceph-system + + ## ArgoCD ## + + - level: None + verbs: ["watch", "get"] + namespaces: ["argocd-system"] + users: + - system:serviceaccount:argocd-system:argocd-server + - system:serviceaccount:argocd-system:argocd-application-controller + - system:serviceaccount:argocd-system:argocd-notifications-controller + - system:serviceaccount:argocd-system:argocd-applicationset-controller + resources: + - group: "" # core + resources: ["secrets", "configmaps"] + - group: "argoproj.io" + resources: ["appprojects", "applications"] + + ## Resourcequota-controller + + # Ignore get requests + - level: None + verbs: ["get"] + users: + - system:serviceaccount:kube-system:resourcequota-controller + + ## Kube-controller-manager ## + + # Only log metadata + - level: Metadata + verbs: ["get"] + users: + - system:kube-controller-manager + omitStages: + - "RequestReceived" + + # Drop watch + - level: None + verbs: ["watch"] + users: + - system:kube-controller-manager + + ## Postgres ## + + # Don't log endpointslice mirroring in postgres-system + - level: None + verbs: ["update"] + namespaces: ["postgres-system"] + users: + - system:serviceaccount:kube-system:endpointslicemirroring-controller + resources: + - group: "discovery.k8s.io" + resources: ["endpointslices"] + + # Don't log watch requests from postgres-operator + - level: None + verbs: ["watch"] + users: + - system:serviceaccount:postgres-system:postgres-operator + resources: + - group: "" + resources: ["pods", "nodes"] + - group: "postgresqls" + resources: ["acid.zalan.do"] + + # Get responses can be large; skip them. + - level: Request + verbs: ["list"] + resources: + - group: "postgresqls" + resources: ["acid.zalan.do"] + omitStages: + - "RequestReceived" + + # Don't log get requests to kubernetes endpoint from patroni + - level: None + verbs: ["get"] + namespaces: ["default"] + users: + - system:serviceaccount:postgres-system:postgres-pod + resources: + - group: "" # core + resources: ["endpoints"] + + # Don't log watch and patch requests for endpoints and pods from patroni + - level: None + verbs: ["watch", "patch"] + namespaces: ["postgres-system"] + users: + - system:serviceaccount:postgres-system:postgres-pod + resources: + - group: "" # core + resources: ["endpoints", "pods"] + + ## RabbitMQ ## + + - level: None + verbs: ["watch"] + users: + - system:serviceaccount:rabbitmq-system:rabbitmq-cluster-operator + resources: + - group: "rabbitmq.com" + resources: ["rabbitmqclusters"] + - group: "apps" + resources: ["statefulsets"] + - group: "" # core + resources: ["configmaps", "serviceaccounts", "services", "endpoints", "secrets"] + - group: "rbac.authorization.k8s.io" + resources: ["rolebindings", "roles"] + + ## Valkey ## + + - level: None + namespaces: ["valkey-system"] + verbs: ["get", "list"] + users: + - system:serviceaccount:valkey-system:valkey-operator-redis-operator + resources: + - group: "" # core + resources: ["pods", "configmaps", "services"] + - group: "apps" + resources: ["deployments", "statefulsets"] + - group: "policy" + resources: ["poddisruptionbudgets"] + + - level: None + namespaces: ["valkey-system"] + verbs: ["update"] + users: + - system:serviceaccount:valkey-system:valkey-operator-redis-operator + resources: + - group: "" # core + resources: ["configmaps"] + - group: "policy" + resources: ["poddisruptionbudgets"] + + # Ignore deployment status updates + - level: None + namespaces: ["valkey-system"] + verbs: ["update"] + users: + - system:serviceaccount:kube-system:deployment-controller + resources: + - group: "apps" + resources: ["deployments/status"] + + ## Velero ## + + - level: None + verbs: ["watch"] + users: + - system:serviceaccount:velero:velero-server + resources: + - group: "" # core + resources: ["pods", "secrets", "persistentvolumeclaims"] + - group: "velero.io" + resources: ["podvolumebackups", "podvolumerestores", "backupstoragelocations"] + + - level: None + verbs: ["watch"] + namespaces: ["velero"] + users: + - system:serviceaccount:velero:velero-server + resources: + - group: "velero.io" + resources: ["downloadrequests", "resticrepositories", "schedules", "backups", "serverstatusrequests", "deletebackuprequests"] + + # Don't log updates to the default backupstoragelocation + # Normally just patch for status.lastSyncTime + - level: None + verbs: ["patch"] + namespaces: ["velero"] + users: + - system:serviceaccount:velero:velero-server + resources: + - group: "velero.io" + resources: ["backupstoragelocations"] + + # Secrets, ConfigMaps, TokenRequest and TokenReviews can contain sensitive & binary data, + # so only log at the Metadata level. + - level: Metadata + resources: + - group: "" # core + resources: ["secrets", "configmaps", "serviceaccounts/token"] + - group: authentication.k8s.io + resources: ["tokenreviews"] + omitStages: + - "RequestReceived" + # Get responses can be large; skip them. + - level: Request + verbs: ["get", "list", "watch"] + resources: + - group: "" # core + - group: "admissionregistration.k8s.io" + - group: "apiextensions.k8s.io" + - group: "apiregistration.k8s.io" + - group: "apps" + - group: "authentication.k8s.io" + - group: "authorization.k8s.io" + - group: "autoscaling" + - group: "batch" + - group: "certificates.k8s.io" + - group: "extensions" + - group: "metrics.k8s.io" + - group: "networking.k8s.io" + - group: "policy" + - group: "rbac.authorization.k8s.io" + - group: "settings.k8s.io" + - group: "storage.k8s.io" + omitStages: + - "RequestReceived" + # Default level for all other requests. + - level: RequestResponse + omitStages: + - "RequestReceived" diff --git a/images/capi/ansible/roles/kubernetes/tasks/main.yml b/images/capi/ansible/roles/kubernetes/tasks/main.yml index 4bfe1e9111..cb2d246e20 100644 --- a/images/capi/ansible/roles/kubernetes/tasks/main.yml +++ b/images/capi/ansible/roles/kubernetes/tasks/main.yml @@ -117,6 +117,20 @@ group: root mode: "0644" +- name: Create Kubernetes audit policy directory + ansible.builtin.file: + path: /etc/kubernetes/audit-policy + state: directory + mode: '0755' + +- name: Drop Kubernetes audit policy file + ansible.builtin.copy: + src: files/etc/audit-policy/apiserver-audit-policy.yaml + dest: /etc/kubernetes/audit-policy/apiserver-audit-policy.yaml + owner: root + group: root + mode: "0644" + # TODO: This section will be deprecated once https://github.com/containerd/cri/issues/1131 is fixed. It is used to support ECR with containerd. - name: Check if Kubernetes container registry is using Amazon ECR ansible.builtin.set_fact: diff --git a/images/capi/ansible/roles/sysprep/tasks/debian.yml b/images/capi/ansible/roles/sysprep/tasks/debian.yml index 6f5b7e933e..8abe6d50b2 100644 --- a/images/capi/ansible/roles/sysprep/tasks/debian.yml +++ b/images/capi/ansible/roles/sysprep/tasks/debian.yml @@ -17,13 +17,18 @@ last_log_mode: "0664" machine_id_mode: "0644" -- name: Pin all installed packages with apt-mark - ansible.builtin.shell: | - set -o pipefail - dpkg-query -f '${binary:Package}\n' -W | xargs apt-mark hold - - args: - executable: /bin/bash +# This is required to ensure any apt upgrade will not break kubernetes +- name: Tell Debian hosts not to change the kuberetes versions with apt upgrade + dpkg_selections: + name: "{{ item }}" + selection: hold + when: ansible_pkg_mgr == 'apt' + changed_when: false + with_items: + - kubelet + - kubeadm + - kubectl + - kubernetes-cni - name: Unpin grub packages with apt-mark for MaaS ansible.builtin.shell: | @@ -90,25 +95,6 @@ - { path: /var/lib/apt/lists, state: absent, mode: "0755" } - { path: /var/lib/apt/lists, state: directory, mode: "0755" } -- name: Disable apt-daily services - ansible.builtin.systemd: - name: "{{ item }}" - state: stopped - enabled: false - loop: - - apt-daily.timer - - apt-daily-upgrade.timer - -- name: Get installed packages - ansible.builtin.package_facts: - -- name: Disable unattended upgrades if installed - ansible.builtin.systemd: - name: unattended-upgrades - state: stopped - enabled: false - when: "'unattended-upgrades' in ansible_facts.packages" - - name: Reset network interface IDs ansible.builtin.file: state: absent diff --git a/images/capi/packer/goss/goss-vars.yaml b/images/capi/packer/goss/goss-vars.yaml index 08c9bc48d3..4c86878022 100644 --- a/images/capi/packer/goss/goss-vars.yaml +++ b/images/capi/packer/goss/goss-vars.yaml @@ -417,11 +417,11 @@ ubuntu: <<: *common_debs common-service: apt-daily.timer: - enabled: false - running: false + enabled: true + running: true apt-daily-upgrade.timer: - enabled: false - running: false + enabled: true + running: true azure: command: iptables -C FORWARD -d 168.63.129.16/32 -p tcp -m tcp --dport 80 -m comment --comment "block traffic to 168.63.129.16 for cve-2021-27075" -j DROP: diff --git a/images/capi/patches/dockerfile.patch b/images/capi/patches/dockerfile.patch new file mode 100644 index 0000000000..31aacd099b --- /dev/null +++ b/images/capi/patches/dockerfile.patch @@ -0,0 +1,21 @@ +diff --git a/images/capi/.dockerignore b/images/capi/.dockerignore +index e474674ca..565293fe8 100644 +--- a/images/capi/.dockerignore ++++ b/images/capi/.dockerignore +@@ -9,3 +9,4 @@ + !packer + !Makefile + !azure_targets.sh ++!template.json +diff --git a/images/capi/Dockerfile b/images/capi/Dockerfile +index e9ace3ed6..c0bb40356 100644 +--- a/images/capi/Dockerfile ++++ b/images/capi/Dockerfile +@@ -55,6 +55,7 @@ COPY --chown=imagebuilder:imagebuilder hack hack/ + COPY --chown=imagebuilder:imagebuilder packer packer/ + COPY --chown=imagebuilder:imagebuilder Makefile Makefile + COPY --chown=imagebuilder:imagebuilder azure_targets.sh azure_targets.sh ++COPY --chown=imagebuilder:imagebuilder template.json template.json + + ENV PATH="/home/imagebuilder/.local/bin:${PATH}" + ENV PACKER_ARGS='' diff --git a/images/capi/template.json b/images/capi/template.json new file mode 100644 index 0000000000..da74772f2b --- /dev/null +++ b/images/capi/template.json @@ -0,0 +1,38 @@ +{ + "crictl_arch": "amd64", + "crictl_sha256": "https://github.com/kubernetes-sigs/cri-tools/releases/download/v{{user `crictl_version`}}/crictl-v{{user `crictl_version`}}-linux-{{user `crictl_arch`}}.tar.gz.sha256", + "crictl_source_type": "pkg", + "crictl_url": "https://github.com/kubernetes-sigs/cri-tools/releases/download/v{{user `crictl_version`}}/crictl-v{{user `crictl_version`}}-linux-{{user `crictl_arch`}}.tar.gz", + "crictl_version": "$KUBERNETES_SERIES.0", + "kubeadm_template": "etc/kubeadm.yml", + "kubernetes_apiserver_port": "6443", + "kubernetes_container_registry": "registry.k8s.io", + "kubernetes_deb_gpg_key": "https://pkgs.k8s.io/core:/stable:/{{ user `kubernetes_series` }}/deb/Release.key", + "kubernetes_deb_repo": "https://pkgs.k8s.io/core:/stable:/{{ user `kubernetes_series` }}/deb/", + "kubernetes_deb_version": "$KUBERNETES_DEB_VERSION", + "kubernetes_goarch": "amd64", + "kubernetes_http_source": "https://dl.k8s.io/release", + "kubernetes_load_additional_imgs": "false", + "kubernetes_rpm_gpg_check": "True", + "kubernetes_rpm_gpg_key": "https://pkgs.k8s.io/core:/stable:/{{ user `kubernetes_series` }}/rpm/repodata/repomd.xml.key", + "kubernetes_rpm_repo": "https://pkgs.k8s.io/core:/stable:/{{ user `kubernetes_series` }}/rpm/", + "kubernetes_rpm_repo_arch": "x86_64", + "kubernetes_rpm_version": "$KUBERNETES_VERSION", + "kubernetes_semver": "v$KUBERNETES_VERSION", + "kubernetes_series": "v$KUBERNETES_SERIES", + "kubernetes_source_type": "pkg", + "node_custom_roles_post": "sshca", + "systemd_prefix": "/usr/lib/systemd", + "sysusr_prefix": "/usr", + "sysusrlocal_prefix": "/usr/local", + "vm_name": "{{user `build_name`}}-kube-$KUBERNETES_VERSION-ck8s-capi-$IMAGE_TAG", + "artifact_name": "{{user `build_name`}}-kube-$KUBERNETES_VERSION-ck8s-capi-$IMAGE_TAG", + "output_directory": "./output/{{user `build_name`}}-kube-$KUBERNETES_VERSION-ck8s-capi-$IMAGE_TAG", + "image_name": "{{user `distribution`}}-{{user `distribution_version`}}-kube-$KUBERNETES_VERSION-ck8s-capi-$IMAGE_TAG", + "aws_region": "eu-north-1", + "ami_regions": "eu-north-1", + "ami_groups": "", + "snapshot_groups": "", + "containerd_version": "1.7.27", + "containerd_url": "https://github.com/containerd/containerd/releases/download/v1.7.27/containerd-1.7.27-linux-amd64.tar.gz" +} \ No newline at end of file