diff --git a/.github/actions/publish-compose/action.yml b/.github/actions/publish-compose/action.yml new file mode 100644 index 0000000000..89774c1715 --- /dev/null +++ b/.github/actions/publish-compose/action.yml @@ -0,0 +1,116 @@ +# Publishes a Compose model that uses local includes as one portable OCI +# application. OCI Compose artifacts cannot contain local includes, so the +# action flattens the model before applying the publication-only override while +# leaving interpolation and project resource naming deferred to the consumer. +name: "Publish Compose application" +description: "Flatten local Compose includes and publish the resulting OCI application" + +inputs: + compose-file: + description: "Path to the root Compose file" + required: true + mode: + description: "Publication mode: dry-run or publish" + required: true + registry-prefix: + description: "Registry hostname and namespace, for example ghcr.io/0xmiden" + required: true + package: + description: "OCI package name" + required: true + tag: + description: "OCI package tag" + required: true + publish-override-file: + description: "Path to the portability override applied before publishing" + required: false + default: "compose/publish.yml" + +runs: + using: "composite" + steps: + - name: Validate inputs + shell: bash + env: + COMPOSE_FILE: ${{ inputs.compose-file }} + MODE: ${{ inputs.mode }} + PUBLISH_OVERRIDE_FILE: ${{ inputs.publish-override-file }} + run: | + set -euo pipefail + + case "${MODE}" in + dry-run | publish) ;; + *) + echo "::error::Unsupported Compose publication mode: ${MODE}" + exit 1 + ;; + esac + + if [[ ! -f "${COMPOSE_FILE}" ]]; then + echo "::error::Compose file does not exist: ${COMPOSE_FILE}" + exit 1 + fi + + if [[ ! -f "${PUBLISH_OVERRIDE_FILE}" ]]; then + echo "::error::Compose publish override does not exist: ${PUBLISH_OVERRIDE_FILE}" + exit 1 + fi + + - name: Build publishable Compose model + shell: bash + env: + COMPOSE_FILE: ${{ inputs.compose-file }} + PUBLISH_OVERRIDE_FILE: ${{ inputs.publish-override-file }} + run: | + set -euo pipefail + + docker compose \ + -f "${COMPOSE_FILE}" \ + --profile "*" \ + config \ + --no-interpolate \ + --no-normalize \ + --output "${RUNNER_TEMP}/compose.flattened.yml" + docker compose \ + -f "${RUNNER_TEMP}/compose.flattened.yml" \ + -f "${PUBLISH_OVERRIDE_FILE}" \ + --profile "*" \ + config \ + --no-interpolate \ + --no-normalize \ + --output "${RUNNER_TEMP}/compose.publish.yml" + + - name: Publish Compose application + shell: bash + env: + MODE: ${{ inputs.mode }} + PACKAGE: ${{ inputs.package }} + REGISTRY_PREFIX: ${{ inputs.registry-prefix }} + TAG: ${{ inputs.tag }} + run: | + set -euo pipefail + + artifact_destination="${REGISTRY_PREFIX}/${PACKAGE}:${TAG}" + + case "${MODE}" in + dry-run) + docker compose \ + -f "${RUNNER_TEMP}/compose.publish.yml" \ + --profile "*" \ + publish \ + --with-env \ + --yes \ + --dry-run \ + "${artifact_destination}" + ;; + publish) + docker compose \ + -f "${RUNNER_TEMP}/compose.publish.yml" \ + --profile "*" \ + publish \ + --with-env \ + --yes \ + --resolve-image-digests \ + "${artifact_destination}" + ;; + esac diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index 15ab218e54..2bafc35fe5 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -97,19 +97,16 @@ jobs: persist-credentials: false - name: Dry-run Compose publishing + uses: ./.github/actions/publish-compose env: MIDEN_NODE_IMAGE: ghcr.io/0xmiden/miden-node:dry-run MIDEN_VALIDATOR_IMAGE: ghcr.io/0xmiden/miden-validator:dry-run MIDEN_NTX_BUILDER_IMAGE: ghcr.io/0xmiden/miden-ntx-builder:dry-run MIDEN_REMOTE_PROVER_IMAGE: ghcr.io/0xmiden/miden-remote-prover:dry-run MIDEN_NETWORK_MONITOR_IMAGE: ghcr.io/0xmiden/miden-network-monitor:dry-run - run: | - docker compose \ - -f docker-compose.yml \ - -f compose/publish.yml \ - --profile "*" \ - publish \ - --with-env \ - --yes \ - --dry-run \ - ghcr.io/0xmiden/miden-local-network:dry-run + with: + compose-file: docker-compose.yml + mode: dry-run + package: miden-local-network + registry-prefix: ghcr.io/0xmiden + tag: dry-run diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c6ebf432e3..e05f6bd757 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -314,24 +314,19 @@ jobs: persist-credentials: false - name: Dry-run Compose publishing + uses: ./.github/actions/publish-compose env: - COMPOSE_PACKAGE: ${{ env.REGISTRY }}/0xmiden/miden-local-network MIDEN_NODE_IMAGE: ${{ env.REGISTRY }}/0xmiden/miden-node:${{ needs.preflight.outputs.tag }} MIDEN_VALIDATOR_IMAGE: ${{ env.REGISTRY }}/0xmiden/miden-validator:${{ needs.preflight.outputs.tag }} MIDEN_NTX_BUILDER_IMAGE: ${{ env.REGISTRY }}/0xmiden/miden-ntx-builder:${{ needs.preflight.outputs.tag }} MIDEN_REMOTE_PROVER_IMAGE: ${{ env.REGISTRY }}/0xmiden/miden-remote-prover:${{ needs.preflight.outputs.tag }} MIDEN_NETWORK_MONITOR_IMAGE: ${{ env.REGISTRY }}/0xmiden/miden-network-monitor:${{ needs.preflight.outputs.tag }} - TAG: ${{ needs.preflight.outputs.tag }} - run: | - docker compose \ - -f docker-compose.yml \ - -f compose/publish.yml \ - --profile "*" \ - publish \ - --with-env \ - --yes \ - --dry-run \ - "${COMPOSE_PACKAGE}:${TAG}" + with: + compose-file: docker-compose.yml + mode: dry-run + package: miden-local-network + registry-prefix: ${{ env.REGISTRY }}/0xmiden + tag: ${{ needs.preflight.outputs.tag }} publish-docker: name: Publish Docker ${{ matrix.component }} @@ -433,24 +428,19 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Publish Compose application + uses: ./.github/actions/publish-compose env: - COMPOSE_PACKAGE: ${{ env.REGISTRY }}/0xmiden/miden-local-network MIDEN_NODE_IMAGE: ${{ env.REGISTRY }}/0xmiden/miden-node:${{ needs.preflight.outputs.tag }} MIDEN_VALIDATOR_IMAGE: ${{ env.REGISTRY }}/0xmiden/miden-validator:${{ needs.preflight.outputs.tag }} MIDEN_NTX_BUILDER_IMAGE: ${{ env.REGISTRY }}/0xmiden/miden-ntx-builder:${{ needs.preflight.outputs.tag }} MIDEN_REMOTE_PROVER_IMAGE: ${{ env.REGISTRY }}/0xmiden/miden-remote-prover:${{ needs.preflight.outputs.tag }} MIDEN_NETWORK_MONITOR_IMAGE: ${{ env.REGISTRY }}/0xmiden/miden-network-monitor:${{ needs.preflight.outputs.tag }} - TAG: ${{ needs.preflight.outputs.tag }} - run: | - docker compose \ - -f docker-compose.yml \ - -f compose/publish.yml \ - --profile "*" \ - publish \ - --with-env \ - --yes \ - --resolve-image-digests \ - "${COMPOSE_PACKAGE}:${TAG}" + with: + compose-file: docker-compose.yml + mode: publish + package: miden-local-network + registry-prefix: ${{ env.REGISTRY }}/0xmiden + tag: ${{ needs.preflight.outputs.tag }} publish-github-release: name: Publish GitHub release diff --git a/compose/bootstrap.yml b/compose/bootstrap.yml new file mode 100644 index 0000000000..c1b543335d --- /dev/null +++ b/compose/bootstrap.yml @@ -0,0 +1,103 @@ +services: + bootstrap-validator: + image: ${MIDEN_VALIDATOR_IMAGE:-miden-validator} + pull_policy: missing + volumes: + - node-data:/data + - type: bind + source: ${MIDEN_GENESIS_CONFIG_FILE:-./genesis/dummy.toml} + target: /genesis.toml + read_only: true + environment: + MIDEN_NODE_VALIDATOR_USE_GENESIS_CONFIG: ${MIDEN_GENESIS_CONFIG_FILE:-} + entrypoint: ["/bin/sh", "-c"] + command: + - | + set -e + if [ -f /data/validator/.bootstrapped ]; then + echo "Validator already bootstrapped." + exit 0 + fi + + echo "Cleaning incomplete validator bootstrap state..." + rm -rf /data/genesis /data/validator /data/accounts + mkdir -p /data/genesis /data/validator /data/accounts + + GENESIS_CONFIG_ARGS="" + if [ -n "$${MIDEN_NODE_VALIDATOR_USE_GENESIS_CONFIG:-}" ]; then + if [ ! -f /genesis.toml ]; then + echo "Genesis config requested but /genesis.toml is not mounted." + exit 1 + fi + + GENESIS_CONFIG_ARGS="--config /genesis.toml" + fi + + echo "Building genesis block..." + miden-validator genesis \ + --genesis-block-directory /data/genesis \ + --accounts-directory /data/accounts \ + $$GENESIS_CONFIG_ARGS + + echo "Bootstrapping validator..." + miden-validator bootstrap \ + --data-directory /data/validator \ + --genesis /data/genesis/genesis.dat + + touch /data/validator/.bootstrapped + + bootstrap-node: + image: ${MIDEN_NODE_IMAGE:-miden-node} + pull_policy: missing + volumes: + - node-data:/data + entrypoint: ["/bin/sh", "-c"] + depends_on: + bootstrap-validator: + condition: service_completed_successfully + command: + - | + set -e + if [ -f /data/node/.bootstrapped ]; then + echo "Node already bootstrapped." + exit 0 + fi + + echo "Cleaning incomplete node bootstrap state..." + rm -rf /data/node + mkdir -p /data/node + + echo "Bootstrapping node..." + miden-node bootstrap \ + --data-directory /data/node \ + --genesis /data/genesis/genesis.dat + + touch /data/node/.bootstrapped + + bootstrap-ntx-builder: + image: ${MIDEN_NTX_BUILDER_IMAGE:-miden-ntx-builder} + pull_policy: missing + volumes: + - node-data:/data + entrypoint: ["/bin/sh", "-c"] + depends_on: + bootstrap-node: + condition: service_completed_successfully + command: + - | + set -e + if [ -f /data/ntx-builder/.bootstrapped ]; then + echo "Network transaction builder already bootstrapped." + exit 0 + fi + + echo "Cleaning incomplete network transaction builder bootstrap state..." + rm -rf /data/ntx-builder + mkdir -p /data/ntx-builder + + echo "Bootstrapping network transaction builder..." + miden-ntx-builder bootstrap \ + --data-directory /data/ntx-builder \ + --genesis /data/genesis/genesis.dat + + touch /data/ntx-builder/.bootstrapped diff --git a/compose/monitor.yml b/compose/monitor.yml new file mode 100644 index 0000000000..23465b564d --- /dev/null +++ b/compose/monitor.yml @@ -0,0 +1,20 @@ +services: + monitor: + profiles: [monitor] + image: ${MIDEN_NETWORK_MONITOR_IMAGE:-miden-network-monitor} + pull_policy: missing + depends_on: + otel-collector: + condition: service_started + sequencer: + condition: service_started + command: + - miden-network-monitor + - start + environment: + MIDEN_MONITOR_RPC_URL: http://sequencer:57291 + MIDEN_MONITOR_PORT: "3001" + MIDEN_MONITOR_NETWORK_NAME: Localhost + OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4317 + ports: + - "127.0.0.1:3001:3001" diff --git a/compose/node.yml b/compose/node.yml new file mode 100644 index 0000000000..866b92cab2 --- /dev/null +++ b/compose/node.yml @@ -0,0 +1,26 @@ +services: + sequencer: + image: ${MIDEN_NODE_IMAGE:-miden-node} + pull_policy: missing + volumes: + - node-data:/data + depends_on: + bootstrap-node: + condition: service_completed_successfully + otel-collector: + condition: service_started + validator: + condition: service_started + command: + - miden-node + - sequencer + - --rpc.listen=0.0.0.0:57291 + - --data-directory=/data/node + - --validator.url=http://validator:50101 + - --ntx-builder.url=http://ntx-builder:50301 + - --rpc.network-tx-auth-header-value=secret_value + environment: + OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4317 + OTEL_RESOURCE_ATTRIBUTES: service.instance.id=sequencer + ports: + - "127.0.0.1:57291:57291" diff --git a/compose/ntx-builder.yml b/compose/ntx-builder.yml new file mode 100644 index 0000000000..a1ebc2290b --- /dev/null +++ b/compose/ntx-builder.yml @@ -0,0 +1,25 @@ +services: + ntx-builder: + image: ${MIDEN_NTX_BUILDER_IMAGE:-miden-ntx-builder} + pull_policy: missing + volumes: + - node-data:/data + depends_on: + bootstrap-ntx-builder: + condition: service_completed_successfully + otel-collector: + condition: service_started + sequencer: + condition: service_started + tx-prover: + condition: service_started + command: + - miden-ntx-builder + - start + - --listen=0.0.0.0:50301 + - --rpc.url=http://sequencer:57291 + - --rpc.auth-header-value=secret_value + - --tx-prover.url=${MIDEN_REMOTE_PROVER_URL:-http://tx-prover:50051} + - --data-directory=/data/ntx-builder + environment: + OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4317 diff --git a/compose/otel-collector.yml b/compose/otel-collector.yml new file mode 100644 index 0000000000..af1d9f905c --- /dev/null +++ b/compose/otel-collector.yml @@ -0,0 +1,85 @@ +services: + otel-collector: + image: otel/opentelemetry-collector-contrib:0.157.0 + configs: + - source: otel-collector + target: /etc/otelcol-contrib/config.yaml + command: ["--config=/etc/otelcol-contrib/config.yaml"] + +configs: + otel-collector: + content: | + receivers: + # Miden services always send traces to this stable in-stack endpoint. + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + # An unset caller endpoint loops back through this sink receiver. This + # gives the upstream exporter a valid, quiet default destination. + otlp/null: + protocols: + grpc: + endpoint: 127.0.0.1:14317 + + processors: + batch: + + exporters: + otlp_grpc/tempo: + endpoint: tempo:4317 + tls: + insecure: true + retry_on_failure: + enabled: false + sending_queue: + enabled: false + otlp_grpc/upstream: + endpoint: ${OTEL_EXPORTER_OTLP_ENDPOINT:-127.0.0.1:14317} + tls: + insecure: true + retry_on_failure: + enabled: false + sending_queue: + enabled: false + file/null: + path: /dev/null + + connectors: + # Each destination fails over independently. Unavailable destinations + # drop traces through file/null and are retried every five seconds. + failover/tempo: + priority_levels: + - [traces/tempo] + - [traces/null-tempo] + retry_interval: 5s + failover/upstream: + priority_levels: + - [traces/upstream] + - [traces/null-upstream] + retry_interval: 5s + + service: + pipelines: + traces: + receivers: [otlp] + processors: [batch] + exporters: [failover/tempo, failover/upstream] + traces/tempo: + receivers: [failover/tempo] + exporters: [otlp_grpc/tempo] + traces/null-tempo: + receivers: [failover/tempo] + exporters: [file/null] + traces/upstream: + receivers: [failover/upstream] + exporters: [otlp_grpc/upstream] + traces/null-upstream: + receivers: [failover/upstream] + exporters: [file/null] + traces/local-null: + receivers: [otlp/null] + exporters: [file/null] + telemetry: + logs: + level: error diff --git a/compose/telemetry.yml b/compose/telemetry.yml new file mode 100644 index 0000000000..d633f9c5ef --- /dev/null +++ b/compose/telemetry.yml @@ -0,0 +1,220 @@ +services: + tempo: + profiles: [telemetry] + image: grafana/tempo:2.10.4 + configs: + - source: tempo + target: /etc/tempo.yaml + command: ["-config.file=/etc/tempo.yaml"] + ports: + - "127.0.0.1:3200:3200" + - "127.0.0.1:4317:4317" + + grafana: + profiles: [telemetry] + image: grafana/grafana:12.4 + configs: + - source: grafana-datasources + target: /etc/grafana/provisioning/datasources/datasources.yaml + - source: grafana-dashboards + target: /etc/grafana/provisioning/dashboards/dashboards.yaml + - source: grafana-miden-node-dashboard + target: /var/lib/grafana/dashboards/miden-node.json + environment: + GF_AUTH_ANONYMOUS_ENABLED: "true" + GF_AUTH_ANONYMOUS_ORG_ROLE: Admin + GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH: /var/lib/grafana/dashboards/miden-node.json + ports: + - "127.0.0.1:3000:3000" + depends_on: + - tempo + +configs: + tempo: + content: | + stream_over_http_enabled: true + + server: + http_listen_port: 3200 + + distributor: + receivers: + otlp: + protocols: + grpc: + endpoint: "0.0.0.0:4317" + + storage: + trace: + backend: local + local: + path: /var/tempo/traces + wal: + path: /var/tempo/wal + + metrics_generator: + registry: + external_labels: + source: tempo + storage: + path: /var/tempo/metrics/wal + traces_storage: + path: /var/tempo/metrics/traces + + overrides: + metrics_generator_processors: [service-graphs, span-metrics, local-blocks] # enables metrics generator + + grafana-datasources: + content: | + apiVersion: 1 + + datasources: + - name: Tempo + type: tempo + access: proxy + url: http://tempo:3200 + isDefault: true + version: 1 + editable: false + uid: tempo + + grafana-dashboards: + content: | + apiVersion: 1 + + providers: + - name: default + orgId: 1 + folder: '' + type: file + disableDeletion: false + editable: true + options: + path: /var/lib/grafana/dashboards + foldersFromFilesStructure: false + + grafana-miden-node-dashboard: + content: | + { + "annotations": { + "list": [] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "panels": [ + { + "datasource": { + "type": "tempo", + "uid": "tempo" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "targets": [ + { + "datasource": { + "type": "tempo", + "uid": "tempo" + }, + "queryType": "traceqlSearch", + "limit": 20, + "tableType": "traces", + "filters": [ + { + "id": "service-name", + "tag": "service.name", + "operator": "=", + "scope": "resource" + } + ] + } + ], + "title": "Recent Traces", + "type": "table" + }, + { + "datasource": { + "type": "tempo", + "uid": "tempo" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 2, + "targets": [ + { + "datasource": { + "type": "tempo", + "uid": "tempo" + }, + "queryType": "traceql", + "query": "{name =~ \"block_builder.*\"}", + "limit": 20, + "tableType": "traces" + } + ], + "title": "Block Building Traces", + "type": "table" + }, + { + "datasource": { + "type": "tempo", + "uid": "tempo" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 20 + }, + "id": 3, + "targets": [ + { + "datasource": { + "type": "tempo", + "uid": "tempo" + }, + "queryType": "traceql", + "query": "{name =~ \"batch_builder.*\"}", + "limit": 20, + "tableType": "traces" + } + ], + "title": "Batch Building Traces", + "type": "table" + } + ], + "schemaVersion": 39, + "tags": ["miden"], + "templating": { + "list": [] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "title": "Miden Node", + "uid": "miden-node", + "version": 2 + } diff --git a/compose/tx-prover.yml b/compose/tx-prover.yml new file mode 100644 index 0000000000..ee1dc8fc26 --- /dev/null +++ b/compose/tx-prover.yml @@ -0,0 +1,13 @@ +services: + tx-prover: + image: ${MIDEN_REMOTE_PROVER_IMAGE:-miden-remote-prover} + pull_policy: missing + depends_on: + otel-collector: + condition: service_started + command: + - miden-remote-prover + - --kind=transaction + - --port=50051 + environment: + OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4317 diff --git a/compose/validator.yml b/compose/validator.yml new file mode 100644 index 0000000000..68d2716985 --- /dev/null +++ b/compose/validator.yml @@ -0,0 +1,18 @@ +services: + validator: + image: ${MIDEN_VALIDATOR_IMAGE:-miden-validator} + pull_policy: missing + volumes: + - node-data:/data + depends_on: + bootstrap-validator: + condition: service_completed_successfully + otel-collector: + condition: service_started + command: + - miden-validator + - start + - --listen=0.0.0.0:50101 + - --data-directory=/data/validator + environment: + OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4317 diff --git a/docker-compose.yml b/docker-compose.yml index b1b5860ec1..0ae61c374f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,515 +13,17 @@ # are always sent to the bundled collector, which forwards them to this # endpoint and to Tempo when the telemetry profile is enabled. -services: - bootstrap-validator: - image: ${MIDEN_VALIDATOR_IMAGE:-miden-validator} - pull_policy: if_not_present - volumes: - - node-data:/data - - type: bind - source: ${MIDEN_GENESIS_CONFIG_FILE:-./compose/genesis/dummy.toml} - target: /genesis.toml - read_only: true - environment: - MIDEN_NODE_VALIDATOR_USE_GENESIS_CONFIG: ${MIDEN_GENESIS_CONFIG_FILE:-} - entrypoint: ["/bin/sh", "-c"] - command: - - | - set -e - if [ -f /data/validator/.bootstrapped ]; then - echo "Validator already bootstrapped." - exit 0 - fi - - echo "Cleaning incomplete validator bootstrap state..." - rm -rf /data/genesis /data/validator /data/accounts - mkdir -p /data/genesis /data/validator /data/accounts - - GENESIS_CONFIG_ARGS="" - if [ -n "$${MIDEN_NODE_VALIDATOR_USE_GENESIS_CONFIG:-}" ]; then - if [ ! -f /genesis.toml ]; then - echo "Genesis config requested but /genesis.toml is not mounted." - exit 1 - fi - - GENESIS_CONFIG_ARGS="--config /genesis.toml" - fi - - echo "Building genesis block..." - miden-validator genesis \ - --genesis-block-directory /data/genesis \ - --accounts-directory /data/accounts \ - $$GENESIS_CONFIG_ARGS - - echo "Bootstrapping validator..." - miden-validator bootstrap \ - --data-directory /data/validator \ - --genesis /data/genesis/genesis.dat - - touch /data/validator/.bootstrapped - - bootstrap-node: - image: ${MIDEN_NODE_IMAGE:-miden-node} - pull_policy: if_not_present - volumes: - - node-data:/data - entrypoint: ["/bin/sh", "-c"] - depends_on: - bootstrap-validator: - condition: service_completed_successfully - command: - - | - set -e - if [ -f /data/node/.bootstrapped ]; then - echo "Node already bootstrapped." - exit 0 - fi - - echo "Cleaning incomplete node bootstrap state..." - rm -rf /data/node - mkdir -p /data/node - - echo "Bootstrapping node..." - miden-node bootstrap \ - --data-directory /data/node \ - --genesis /data/genesis/genesis.dat - - touch /data/node/.bootstrapped - - bootstrap-ntx-builder: - image: ${MIDEN_NTX_BUILDER_IMAGE:-miden-ntx-builder} - pull_policy: if_not_present - volumes: - - node-data:/data - entrypoint: ["/bin/sh", "-c"] - depends_on: - bootstrap-node: - condition: service_completed_successfully - command: - - | - set -e - if [ -f /data/ntx-builder/.bootstrapped ]; then - echo "Network transaction builder already bootstrapped." - exit 0 - fi - - echo "Cleaning incomplete network transaction builder bootstrap state..." - rm -rf /data/ntx-builder - mkdir -p /data/ntx-builder - - echo "Bootstrapping network transaction builder..." - miden-ntx-builder bootstrap \ - --data-directory /data/ntx-builder \ - --genesis /data/genesis/genesis.dat - - touch /data/ntx-builder/.bootstrapped - - sequencer: - image: ${MIDEN_NODE_IMAGE:-miden-node} - pull_policy: if_not_present - volumes: - - node-data:/data - depends_on: - bootstrap-node: - condition: service_completed_successfully - otel-collector: - condition: service_started - validator: - condition: service_started - command: - - miden-node - - sequencer - - --rpc.listen=0.0.0.0:57291 - - --data-directory=/data/node - - --validator.url=http://validator:50101 - - --ntx-builder.url=http://ntx-builder:50301 - - --rpc.network-tx-auth-header-value=secret_value - environment: - OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4317 - OTEL_RESOURCE_ATTRIBUTES: service.instance.id=sequencer - ports: - - "127.0.0.1:57291:57291" - - validator: - image: ${MIDEN_VALIDATOR_IMAGE:-miden-validator} - pull_policy: if_not_present - volumes: - - node-data:/data - depends_on: - bootstrap-validator: - condition: service_completed_successfully - otel-collector: - condition: service_started - command: - - miden-validator - - start - - --listen=0.0.0.0:50101 - - --data-directory=/data/validator - environment: - OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4317 - - tx-prover: - image: ${MIDEN_REMOTE_PROVER_IMAGE:-miden-remote-prover} - pull_policy: if_not_present - depends_on: - otel-collector: - condition: service_started - command: - - miden-remote-prover - - --kind=transaction - - --port=50051 - environment: - OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4317 - - ntx-builder: - image: ${MIDEN_NTX_BUILDER_IMAGE:-miden-ntx-builder} - pull_policy: if_not_present - volumes: - - node-data:/data - depends_on: - bootstrap-ntx-builder: - condition: service_completed_successfully - otel-collector: - condition: service_started - sequencer: - condition: service_started - tx-prover: - condition: service_started - command: - - miden-ntx-builder - - start - - --listen=0.0.0.0:50301 - - --rpc.url=http://sequencer:57291 - - --rpc.auth-header-value=secret_value - - --tx-prover.url=${MIDEN_REMOTE_PROVER_URL:-http://tx-prover:50051} - - --data-directory=/data/ntx-builder - environment: - OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4317 - - otel-collector: - image: otel/opentelemetry-collector-contrib:0.157.0 - configs: - - source: otel-collector - target: /etc/otelcol-contrib/config.yaml - command: ["--config=/etc/otelcol-contrib/config.yaml"] - - tempo: - profiles: [telemetry] - image: grafana/tempo:2.10.4 - configs: - - source: tempo - target: /etc/tempo.yaml - command: ["-config.file=/etc/tempo.yaml"] - ports: - - "127.0.0.1:3200:3200" - - "127.0.0.1:4317:4317" - - grafana: - profiles: [telemetry] - image: grafana/grafana:12.4 - configs: - - source: grafana-datasources - target: /etc/grafana/provisioning/datasources/datasources.yaml - - source: grafana-dashboards - target: /etc/grafana/provisioning/dashboards/dashboards.yaml - - source: grafana-miden-node-dashboard - target: /var/lib/grafana/dashboards/miden-node.json - environment: - GF_AUTH_ANONYMOUS_ENABLED: "true" - GF_AUTH_ANONYMOUS_ORG_ROLE: Admin - GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH: /var/lib/grafana/dashboards/miden-node.json - ports: - - "127.0.0.1:3000:3000" - depends_on: - - tempo - - monitor: - profiles: [monitor] - image: ${MIDEN_NETWORK_MONITOR_IMAGE:-miden-network-monitor} - pull_policy: if_not_present - depends_on: - otel-collector: - condition: service_started - sequencer: - condition: service_started - command: - - miden-network-monitor - - start - environment: - MIDEN_MONITOR_RPC_URL: http://sequencer:57291 - MIDEN_MONITOR_PORT: "3001" - MIDEN_MONITOR_NETWORK_NAME: Localhost - OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4317 - ports: - - "127.0.0.1:3001:3001" +# Keep the complete local application discoverable from the root so direct +# `docker compose` commands and profiles do not require additional `-f` flags. +include: + - compose/bootstrap.yml + - compose/validator.yml + - compose/node.yml + - compose/ntx-builder.yml + - compose/tx-prover.yml + - compose/otel-collector.yml + - compose/telemetry.yml + - compose/monitor.yml volumes: node-data: - -configs: - otel-collector: - content: | - receivers: - # Miden services always send traces to this stable in-stack endpoint. - otlp: - protocols: - grpc: - endpoint: 0.0.0.0:4317 - # An unset caller endpoint loops back through this sink receiver. This - # gives the upstream exporter a valid, quiet default destination. - otlp/null: - protocols: - grpc: - endpoint: 127.0.0.1:14317 - - processors: - batch: - - exporters: - otlp_grpc/tempo: - endpoint: tempo:4317 - tls: - insecure: true - retry_on_failure: - enabled: false - sending_queue: - enabled: false - otlp_grpc/upstream: - endpoint: ${OTEL_EXPORTER_OTLP_ENDPOINT:-127.0.0.1:14317} - tls: - insecure: true - retry_on_failure: - enabled: false - sending_queue: - enabled: false - file/null: - path: /dev/null - - connectors: - # Each destination fails over independently. Unavailable destinations - # drop traces through file/null and are retried every five seconds. - failover/tempo: - priority_levels: - - [traces/tempo] - - [traces/null-tempo] - retry_interval: 5s - failover/upstream: - priority_levels: - - [traces/upstream] - - [traces/null-upstream] - retry_interval: 5s - - service: - pipelines: - traces: - receivers: [otlp] - processors: [batch] - exporters: [failover/tempo, failover/upstream] - traces/tempo: - receivers: [failover/tempo] - exporters: [otlp_grpc/tempo] - traces/null-tempo: - receivers: [failover/tempo] - exporters: [file/null] - traces/upstream: - receivers: [failover/upstream] - exporters: [otlp_grpc/upstream] - traces/null-upstream: - receivers: [failover/upstream] - exporters: [file/null] - traces/local-null: - receivers: [otlp/null] - exporters: [file/null] - telemetry: - logs: - level: error - - tempo: - content: | - stream_over_http_enabled: true - - server: - http_listen_port: 3200 - - distributor: - receivers: - otlp: - protocols: - grpc: - endpoint: "0.0.0.0:4317" - - storage: - trace: - backend: local - local: - path: /var/tempo/traces - wal: - path: /var/tempo/wal - - metrics_generator: - registry: - external_labels: - source: tempo - storage: - path: /var/tempo/metrics/wal - traces_storage: - path: /var/tempo/metrics/traces - - overrides: - metrics_generator_processors: [service-graphs, span-metrics, local-blocks] # enables metrics generator - - grafana-datasources: - content: | - apiVersion: 1 - - datasources: - - name: Tempo - type: tempo - access: proxy - url: http://tempo:3200 - isDefault: true - version: 1 - editable: false - uid: tempo - - grafana-dashboards: - content: | - apiVersion: 1 - - providers: - - name: default - orgId: 1 - folder: '' - type: file - disableDeletion: false - editable: true - options: - path: /var/lib/grafana/dashboards - foldersFromFilesStructure: false - - grafana-miden-node-dashboard: - content: | - { - "annotations": { - "list": [] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "links": [], - "panels": [ - { - "datasource": { - "type": "tempo", - "uid": "tempo" - }, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 1, - "targets": [ - { - "datasource": { - "type": "tempo", - "uid": "tempo" - }, - "queryType": "traceqlSearch", - "limit": 20, - "tableType": "traces", - "filters": [ - { - "id": "service-name", - "tag": "service.name", - "operator": "=", - "scope": "resource" - } - ] - } - ], - "title": "Recent Traces", - "type": "table" - }, - { - "datasource": { - "type": "tempo", - "uid": "tempo" - }, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 10 - }, - "id": 2, - "targets": [ - { - "datasource": { - "type": "tempo", - "uid": "tempo" - }, - "queryType": "traceql", - "query": "{name =~ \"block_builder.*\"}", - "limit": 20, - "tableType": "traces" - } - ], - "title": "Block Building Traces", - "type": "table" - }, - { - "datasource": { - "type": "tempo", - "uid": "tempo" - }, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 20 - }, - "id": 3, - "targets": [ - { - "datasource": { - "type": "tempo", - "uid": "tempo" - }, - "queryType": "traceql", - "query": "{name =~ \"batch_builder.*\"}", - "limit": 20, - "tableType": "traces" - } - ], - "title": "Batch Building Traces", - "type": "table" - } - ], - "schemaVersion": 39, - "tags": ["miden"], - "templating": { - "list": [] - }, - "time": { - "from": "now-30m", - "to": "now" - }, - "title": "Miden Node", - "uid": "miden-node", - "version": 2 - } diff --git a/docs/internal/src/releases.md b/docs/internal/src/releases.md index f398133c29..ee83c463fe 100644 --- a/docs/internal/src/releases.md +++ b/docs/internal/src/releases.md @@ -28,9 +28,14 @@ delete release tags. 4. After those checks pass, the workflow creates the GitHub release and release notes, then starts the crates.io and Debian publishing workflows. -The Compose publication jobs merge `docker-compose.yml` with `compose/publish.yml`. The main model supports custom -genesis configuration through a local bind mount, which cannot be included in a portable OCI artifact. -`compose/publish.yml` uses Compose's `!override` tag to replace that volume list and configures the published +The root `docker-compose.yml` includes the component models under `compose/`, keeping direct local Compose commands and +profiles independent of additional `-f` arguments. Local includes cannot be published directly, so the +`.github/actions/publish-compose` action first renders the complete, all-profile model without interpolating its +variables or normalizing project resource names. It then merges `compose/publish.yml` into that flattened model using +the same options and either dry-runs or performs the publication. + +The local model supports custom genesis configuration through a bind mount, which cannot be included in a portable OCI +artifact. `compose/publish.yml` uses Compose's `!override` tag to replace that volume list and configures the published application to use the validator's built-in genesis configuration. The workflow uses a broad `v*` trigger because GitHub Actions does not use the same pattern language as repository