diff --git a/.github/workflows/proto-ci.yaml b/.github/workflows/proto-ci.yaml new file mode 100644 index 000000000..3ebb72534 --- /dev/null +++ b/.github/workflows/proto-ci.yaml @@ -0,0 +1,68 @@ +name: Proto CI +on: + pull_request: + paths: + - 'pcp-defs/**' + - '.github/workflows/proto-ci.yaml' + workflow_dispatch: + workflow_call: + secrets: + ORB_GIT_HUB_TOKEN: + required: true + CACHIX_AUTH_TOKEN: + required: false + push: + branches: + - main + - prod + paths: + - 'pcp-defs/**' + - '.github/workflows/proto-ci.yaml' + +permissions: + contents: read + +jobs: + lint: + name: Lint + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # pin@v3 + with: + token: ${{ secrets.ORB_GIT_HUB_TOKEN }} + - uses: cachix/install-nix-action@0b0e072294b088b73964f1d72dfdac0951439dbd # pin@v31.8.4 + with: + github_access_token: ${{ secrets.ORB_GIT_HUB_TOKEN }} + - uses: cachix/cachix-action@ad2ddac53f961de1989924296a1f236fcfbaa4fc # pin@v15 + continue-on-error: true + with: + name: worldcoin + authToken: ${{ secrets.CACHIX_AUTH_TOKEN }} + - name: Print environment + run: | + uname -a + nix develop -c env + + - name: buf lint + working-directory: pcp-defs + run: nix develop -c buf lint + + format: + name: Format + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # pin@v3 + with: + token: ${{ secrets.ORB_GIT_HUB_TOKEN }} + - uses: cachix/install-nix-action@0b0e072294b088b73964f1d72dfdac0951439dbd # pin@v31.8.4 + with: + github_access_token: ${{ secrets.ORB_GIT_HUB_TOKEN }} + - uses: cachix/cachix-action@ad2ddac53f961de1989924296a1f236fcfbaa4fc # pin@v15 + continue-on-error: true + with: + name: worldcoin + authToken: ${{ secrets.CACHIX_AUTH_TOKEN }} + + - name: buf format + working-directory: pcp-defs + run: nix develop -c buf format --diff --exit-code diff --git a/Cargo.lock b/Cargo.lock index db8d25551..ebdae6289 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8620,6 +8620,14 @@ dependencies = [ "tracing", ] +[[package]] +name = "orb-pcp-defs" +version = "0.0.0" +dependencies = [ + "prost 0.14.1", + "prost-build 0.14.1", +] + [[package]] name = "orb-qr-link" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 5c9f94aaa..1ab4bbc5c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -41,6 +41,7 @@ members = [ "orb-info", "orb-jobs-agent", "ota-backend", + "pcp-defs/rust", "prelude", "qr-link", "s3-helpers", diff --git a/nix/shells/development.nix b/nix/shells/development.nix index 5e286e809..aaf603001 100644 --- a/nix/shells/development.nix +++ b/nix/shells/development.nix @@ -128,6 +128,7 @@ in awscli2 bacon # better cargo-watch black # Python autoformatter + buf # Protobuf workflow cargo-binutils # Contains common native development utilities cargo-deb # Generates .deb packages for orb-os cargo-expand # Useful for inspecting macros diff --git a/pcp-defs/README.md b/pcp-defs/README.md new file mode 100644 index 000000000..53aa8ee6f --- /dev/null +++ b/pcp-defs/README.md @@ -0,0 +1,3 @@ +# pcp-defs + +Protobuf definitions for PCP (Personal Custody Package) messages. diff --git a/pcp-defs/buf.yaml b/pcp-defs/buf.yaml new file mode 100644 index 000000000..c7e30e381 --- /dev/null +++ b/pcp-defs/buf.yaml @@ -0,0 +1,9 @@ +version: v2 +modules: + - path: proto +lint: + use: + - STANDARD +breaking: + use: + - FILE diff --git a/pcp-defs/go/README.md b/pcp-defs/go/README.md new file mode 100644 index 000000000..4d3aa1e56 --- /dev/null +++ b/pcp-defs/go/README.md @@ -0,0 +1,3 @@ +# go + +Placeholder for the Go bindings of `pcp-defs`. diff --git a/pcp-defs/proto/pcp/v1/di_iris_embedding_shares.proto b/pcp-defs/proto/pcp/v1/di_iris_embedding_shares.proto new file mode 100644 index 000000000..acb6f0f23 --- /dev/null +++ b/pcp-defs/proto/pcp/v1/di_iris_embedding_shares.proto @@ -0,0 +1,17 @@ +syntax = "proto3"; + +package pcp.v1; + +message DiIrisEmbeddingShares { + DiIrisEmbeddingShareV1 share_v1 = 1; +} + +message DiIrisEmbeddingShareV1 { + string model_version = 1; + string shares_version = 2; + + repeated uint32 left_share = 3; + repeated uint32 left_mirror_share = 4; + repeated uint32 right_share = 5; + repeated uint32 right_mirror_share = 6; +} diff --git a/pcp-defs/proto/pcp/v1/di_iris_embeddings.proto b/pcp-defs/proto/pcp/v1/di_iris_embeddings.proto new file mode 100644 index 000000000..1579ccaf1 --- /dev/null +++ b/pcp-defs/proto/pcp/v1/di_iris_embeddings.proto @@ -0,0 +1,23 @@ +syntax = "proto3"; + +package pcp.v1; + +message DiIrisEmbeddings { + DiIrisEmbeddingV1 embedding_v1 = 1; +} + +message DiIrisEmbeddingV1 { + string model_version = 1; + string embedding_inference_backend = 2; + string embedding_version = 3; + + repeated sint32 left_embedding = 4; + repeated sint32 left_mirror_embedding = 5; + repeated sint32 right_embedding = 6; + repeated sint32 right_mirror_embedding = 7; + + repeated float left_embedding_f32 = 8; + repeated float left_mirror_embedding_f32 = 9; + repeated float right_embedding_f32 = 10; + repeated float right_mirror_embedding_f32 = 11; +} diff --git a/pcp-defs/rust/Cargo.toml b/pcp-defs/rust/Cargo.toml new file mode 100644 index 000000000..2534aba7b --- /dev/null +++ b/pcp-defs/rust/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "orb-pcp-defs" +version = "0.0.0" +description = "Protobuf definitions for Personal Custody Package messages." + +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +publish = false + +[dependencies] +prost = { version = "0.14.1", default-features = false, features = ["std", "derive"] } + +[build-dependencies] +prost-build = "0.14.1" diff --git a/pcp-defs/rust/build.rs b/pcp-defs/rust/build.rs new file mode 100644 index 000000000..100a89dcb --- /dev/null +++ b/pcp-defs/rust/build.rs @@ -0,0 +1,19 @@ +use std::io::Result; + +fn main() -> Result<()> { + let proto_root = "./proto"; + let proto_files = [ + "./proto/pcp/v1/di_iris_embeddings.proto", + "./proto/pcp/v1/di_iris_embedding_shares.proto", + ]; + + for f in &proto_files { + println!("cargo:rerun-if-changed={f}"); + } + println!("cargo:rerun-if-changed={proto_root}"); + println!("cargo:rerun-if-changed=build.rs"); + + prost_build::Config::new().compile_protos(&proto_files, &[proto_root])?; + + Ok(()) +} diff --git a/pcp-defs/rust/proto b/pcp-defs/rust/proto new file mode 120000 index 000000000..5c8d35253 --- /dev/null +++ b/pcp-defs/rust/proto @@ -0,0 +1 @@ +../proto \ No newline at end of file diff --git a/pcp-defs/rust/src/lib.rs b/pcp-defs/rust/src/lib.rs new file mode 100644 index 000000000..58031f742 --- /dev/null +++ b/pcp-defs/rust/src/lib.rs @@ -0,0 +1,7 @@ +#![forbid(unsafe_code)] + +pub use prost; + +pub mod v1 { + include!(concat!(env!("OUT_DIR"), "/pcp.v1.rs")); +} diff --git a/pcp-defs/rust/tests/round_trip.rs b/pcp-defs/rust/tests/round_trip.rs new file mode 100644 index 000000000..45b9acc3f --- /dev/null +++ b/pcp-defs/rust/tests/round_trip.rs @@ -0,0 +1,69 @@ +use orb_pcp_defs::prost::Message; +use orb_pcp_defs::v1::{ + DiIrisEmbeddingShareV1, DiIrisEmbeddingShares, DiIrisEmbeddingV1, DiIrisEmbeddings, +}; + +const VECTOR_LEN: usize = 512; + +#[test] +fn embeddings_round_trip() { + let i32_payload: Vec = + (0..VECTOR_LEN).map(|i| ((i as i32) % 256) - 128).collect(); + let f32_payload: Vec = (0..VECTOR_LEN).map(|i| (i as f32) * 0.001).collect(); + + let original = DiIrisEmbeddings { + embedding_v1: Some(DiIrisEmbeddingV1 { + model_version: "deep-identifier-1.0.0".into(), + embedding_inference_backend: "deep-identifier".into(), + embedding_version: "1.0.0".into(), + left_embedding: i32_payload.clone(), + left_mirror_embedding: i32_payload.clone(), + right_embedding: i32_payload.clone(), + right_mirror_embedding: i32_payload.clone(), + left_embedding_f32: f32_payload.clone(), + left_mirror_embedding_f32: f32_payload.clone(), + right_embedding_f32: f32_payload.clone(), + right_mirror_embedding_f32: f32_payload.clone(), + }), + }; + + let wire = original.encode_to_vec(); + let decoded = DiIrisEmbeddings::decode(wire.as_slice()) + .expect("encode + decode should round-trip"); + + assert_eq!(original, decoded); +} + +#[test] +fn shares_round_trip() { + let u32_payload: Vec = (0..VECTOR_LEN) + .map(|i| ((i * 127) % 65536) as u32) + .collect(); + + let original = DiIrisEmbeddingShares { + share_v1: Some(DiIrisEmbeddingShareV1 { + model_version: "deep-identifier-1.0.0".into(), + shares_version: "c2d631d821fe96827e8a92fd3bfd457afdd02b9e".into(), + left_share: u32_payload.clone(), + left_mirror_share: u32_payload.clone(), + right_share: u32_payload.clone(), + right_mirror_share: u32_payload.clone(), + }), + }; + + let wire = original.encode_to_vec(); + let decoded = DiIrisEmbeddingShares::decode(wire.as_slice()) + .expect("encode + decode should round-trip"); + + assert_eq!(original, decoded); +} + +#[test] +fn embeddings_unset_round_trips_as_none() { + let original = DiIrisEmbeddings { embedding_v1: None }; + + let wire = original.encode_to_vec(); + let decoded = DiIrisEmbeddings::decode(wire.as_slice()).expect("decode"); + + assert!(decoded.embedding_v1.is_none()); +}