-
-
Notifications
You must be signed in to change notification settings - Fork 718
refactor(zipapp): implement rust exe_zip_maker program #4151
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,10 @@ | ||
| package(default_visibility = ["//:__subpackages__"]) | ||
|
|
||
| licenses(["notice"]) | ||
|
|
||
| filegroup( | ||
| name = "distribution", | ||
| srcs = glob(["**"]) + [ | ||
| "//crates/exe_zip_maker:distribution", | ||
| ], | ||
| ) | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,25 @@ | ||
| load("@rules_rust//rust:defs.bzl", "rust_binary", "rust_library") | ||
|
|
||
| package(default_visibility = ["//:__subpackages__"]) | ||
|
|
||
| licenses(["notice"]) | ||
|
|
||
| rust_library( | ||
| name = "exe_zip_maker_lib", | ||
| srcs = ["src/lib.rs"], | ||
| edition = "2021", | ||
| deps = ["@crates//:sha2"], | ||
| ) | ||
|
|
||
| rust_binary( | ||
| name = "exe_zip_maker", | ||
| srcs = ["src/main.rs"], | ||
| edition = "2021", | ||
| visibility = ["//visibility:public"], | ||
| deps = [":exe_zip_maker_lib"], | ||
| ) | ||
|
|
||
| filegroup( | ||
| name = "distribution", | ||
| srcs = glob(["**"]), | ||
| ) |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,79 @@ | ||
| //! Library supporting creating self-executable zip files. | ||
|
|
||
| use std::fs::{self, File}; | ||
| use std::io::{self, BufReader, BufWriter, Read, Write}; | ||
| use std::path::Path; | ||
|
|
||
| use sha2::{Digest, Sha256}; | ||
|
|
||
| pub const BLOCK_SIZE: usize = 256 * 1024; | ||
| pub const PLACEHOLDER: &[u8] = b"%ZIP_HASH%"; | ||
|
|
||
| /// Replaces all occurrences of `from` with `to` in `src`. | ||
| pub fn replace_bytes(src: &[u8], from: &[u8], to: &[u8]) -> Vec<u8> { | ||
| if from.is_empty() { | ||
| return src.to_vec(); | ||
| } | ||
| let mut result = Vec::new(); | ||
| let mut i = 0; | ||
| while i < src.len() { | ||
| if src[i..].starts_with(from) { | ||
| result.extend_from_slice(to); | ||
| i += from.len(); | ||
| } else { | ||
| result.push(src[i]); | ||
| i += 1; | ||
| } | ||
| } | ||
| result | ||
| } | ||
|
|
||
| /// Computes the SHA256 hex digest of the file at `path`. | ||
| pub fn compute_file_sha256_hex(path: &Path) -> io::Result<String> { | ||
| let mut file = File::open(path)?; | ||
| let mut hasher = Sha256::new(); | ||
| let mut buffer = [0u8; BLOCK_SIZE]; | ||
| loop { | ||
| let n = file.read(&mut buffer)?; | ||
| if n == 0 { | ||
| break; | ||
| } | ||
| hasher.update(&buffer[..n]); | ||
| } | ||
| let digest = hasher.finalize(); | ||
| Ok(format!("{:x}", digest)) | ||
| } | ||
|
|
||
| /// Creates a self-executable zip archive by prepending a preamble to a zip archive | ||
| /// and substituting `%ZIP_HASH%` with the SHA-256 hash of the zip archive. | ||
| pub fn create_exe_zip(preamble_path: &Path, zip_path: &Path, output_path: &Path) -> io::Result<()> { | ||
| if let Some(parent) = output_path.parent() { | ||
| if !parent.as_os_str().is_empty() { | ||
| fs::create_dir_all(parent)?; | ||
| } | ||
| } | ||
|
|
||
| let zip_hash = compute_file_sha256_hex(zip_path)?; | ||
|
|
||
| let preamble_content = fs::read(preamble_path)?; | ||
| let modified_preamble = replace_bytes(&preamble_content, PLACEHOLDER, zip_hash.as_bytes()); | ||
|
|
||
| let mut out_file = BufWriter::with_capacity(BLOCK_SIZE, File::create(output_path)?); | ||
| out_file.write_all(&modified_preamble)?; | ||
|
|
||
| let zip_file = File::open(zip_path)?; | ||
| let mut zip_reader = BufReader::with_capacity(BLOCK_SIZE, zip_file); | ||
| io::copy(&mut zip_reader, &mut out_file)?; | ||
| out_file.flush()?; | ||
|
|
||
| #[cfg(unix)] | ||
| { | ||
| use std::os::unix::fs::PermissionsExt; | ||
| let metadata = fs::metadata(output_path)?; | ||
| let mut perms = metadata.permissions(); | ||
| perms.set_mode(perms.mode() | 0o111); | ||
| fs::set_permissions(output_path, perms)?; | ||
| } | ||
|
|
||
| Ok(()) | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,24 @@ | ||
| use std::env; | ||
| use std::path::Path; | ||
| use std::process; | ||
|
|
||
| fn main() { | ||
| let args: Vec<_> = env::args_os().collect(); | ||
| if args.len() != 4 { | ||
| let prog_name = args | ||
| .first() | ||
| .map(|s| s.to_string_lossy().into_owned()) | ||
| .unwrap_or_else(|| "exe_zip_maker".to_string()); | ||
| eprintln!("Usage: {} <preamble> <zip> <output>", prog_name); | ||
| process::exit(1); | ||
| } | ||
|
|
||
| let preamble_path = Path::new(&args[1]); | ||
| let zip_path = Path::new(&args[2]); | ||
| let output_path = Path::new(&args[3]); | ||
|
|
||
| if let Err(e) = exe_zip_maker_lib::create_exe_zip(preamble_path, zip_path, output_path) { | ||
| eprintln!("exe_zip_maker: error: {}", e); | ||
| process::exit(1); | ||
| } | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,14 @@ | ||
| load("@rules_rust//rust:defs.bzl", "rust_test") | ||
|
|
||
| package(default_visibility = ["//:__subpackages__"]) | ||
|
|
||
| licenses(["notice"]) | ||
|
|
||
| rust_test( | ||
| name = "exe_zip_maker_test", | ||
| size = "small", | ||
| srcs = ["exe_zip_maker_test.rs"], | ||
| deps = [ | ||
| "//crates/exe_zip_maker:exe_zip_maker_lib", | ||
| ], | ||
| ) |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,155 @@ | ||
| use std::env; | ||
| use std::fs; | ||
|
|
||
| use exe_zip_maker_lib::{ | ||
| compute_file_sha256_hex, create_exe_zip, replace_bytes, PLACEHOLDER, | ||
| }; | ||
|
|
||
| #[test] | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. My thinking is that we should write idiomatic Rust code and what I've seen in the past is:
What do you think about this convention?
Collaborator
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. It originally generated code like that and I thought it was just being lazy. Another person confirmed it, too, though. I think that's a wacky convention. But, if that'd idiomatic, then, well, when in Rust-ome
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I personally like that the tests become sort of executable docs. But I agree, it's a little whacky :D Though it allows to keep the exposed symbols to minimum and keep the unit tests exercising the implementation details in optimum way. Kind of no longer need to do "buildifier ignore private" if you follow this convention.
Collaborator
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Hm, I also saw the rust_test target was outside of //tests. I'm more resistant to that -- I really like being able to run |
||
| fn test_replace_bytes_none() { | ||
| let src = b"hello world"; | ||
| assert_eq!(replace_bytes(src, b"foo", b"bar"), b"hello world"); | ||
| } | ||
|
|
||
| #[test] | ||
| fn test_replace_bytes_single() { | ||
| let src = b"EXPECTED_HASH='%ZIP_HASH%'"; | ||
| let replaced = replace_bytes(src, PLACEHOLDER, b"12345678"); | ||
| assert_eq!(replaced, b"EXPECTED_HASH='12345678'"); | ||
| } | ||
|
|
||
| #[test] | ||
| fn test_replace_bytes_multiple() { | ||
| let src = b"%ZIP_HASH% and %ZIP_HASH%"; | ||
| let replaced = replace_bytes(src, PLACEHOLDER, b"abc"); | ||
| assert_eq!(replaced, b"abc and abc"); | ||
| } | ||
|
|
||
| #[test] | ||
| fn test_replace_bytes_empty_from() { | ||
| let src = b"unchanged"; | ||
| assert_eq!(replace_bytes(src, b"", b"abc"), b"unchanged"); | ||
| } | ||
|
|
||
| #[test] | ||
| fn test_compute_file_sha256_hex() { | ||
| let temp_dir = env::temp_dir().join(format!("sha256_test_{}", std::process::id())); | ||
| fs::create_dir_all(&temp_dir).unwrap(); | ||
| let file_path = temp_dir.join("sample.txt"); | ||
|
|
||
| fs::write(&file_path, b"hello world\n").unwrap(); | ||
| let hash = compute_file_sha256_hex(&file_path).unwrap(); | ||
| assert_eq!( | ||
| hash, | ||
| "a948904f2f0f479b8f8197694b30184b0d2ed1c1cd2a1ec0fb85d299a192a447" | ||
| ); | ||
|
|
||
| let _ = fs::remove_dir_all(&temp_dir); | ||
| } | ||
|
|
||
| #[test] | ||
| fn test_create_exe_zip_successful() { | ||
| let temp_dir = env::temp_dir().join(format!("create_exe_zip_test_{}", std::process::id())); | ||
| fs::create_dir_all(&temp_dir).unwrap(); | ||
|
|
||
| let preamble_path = temp_dir.join("preamble.sh"); | ||
| let zip_path = temp_dir.join("data.zip"); | ||
| let output_path = temp_dir.join("output.exe"); | ||
|
|
||
| let zip_content = b"PK\x03\x04dummyzipcontent"; | ||
| fs::write(&zip_path, zip_content).unwrap(); | ||
|
|
||
| let preamble_text = b"#!/bin/bash\nEXPECTED_HASH='%ZIP_HASH%'\n# ... logic ...\n"; | ||
| fs::write(&preamble_path, preamble_text).unwrap(); | ||
|
|
||
| create_exe_zip(&preamble_path, &zip_path, &output_path).unwrap(); | ||
|
|
||
| assert!(output_path.exists()); | ||
|
|
||
| #[cfg(unix)] | ||
| { | ||
| use std::os::unix::fs::PermissionsExt; | ||
| let st = fs::metadata(&output_path).unwrap(); | ||
| assert_ne!( | ||
| st.permissions().mode() & 0o100, | ||
| 0, | ||
| "Expected executable permission on output file" | ||
| ); | ||
| } | ||
|
|
||
| let content = fs::read(&output_path).unwrap(); | ||
| let expected_hash = "65e39989ca91c49484998aa3f0429f6943c029609bfd2f3c18c77bf9ded72c59"; | ||
| let expected_preamble = replace_bytes(preamble_text, PLACEHOLDER, expected_hash.as_bytes()); | ||
|
|
||
| assert!(content.starts_with(&expected_preamble)); | ||
| assert!(content.ends_with(zip_content)); | ||
| assert_eq!(content.len(), expected_preamble.len() + zip_content.len()); | ||
|
|
||
| let _ = fs::remove_dir_all(&temp_dir); | ||
| } | ||
|
|
||
| #[test] | ||
| fn test_create_exe_zip_multiple_placeholders() { | ||
| let temp_dir = env::temp_dir().join(format!("create_exe_zip_multi_{}", std::process::id())); | ||
| fs::create_dir_all(&temp_dir).unwrap(); | ||
|
|
||
| let preamble_path = temp_dir.join("preamble.sh"); | ||
| let zip_path = temp_dir.join("data.zip"); | ||
| let output_path = temp_dir.join("output.exe"); | ||
|
|
||
| let zip_content = b"PK\x03\x04dummyzipcontent"; | ||
| fs::write(&zip_path, zip_content).unwrap(); | ||
|
|
||
| let preamble_text = b"# First: %ZIP_HASH%\n# Second: %ZIP_HASH%\n"; | ||
| fs::write(&preamble_path, preamble_text).unwrap(); | ||
|
|
||
| create_exe_zip(&preamble_path, &zip_path, &output_path).unwrap(); | ||
|
|
||
| let content = fs::read(&output_path).unwrap(); | ||
| let expected_hash = "65e39989ca91c49484998aa3f0429f6943c029609bfd2f3c18c77bf9ded72c59"; | ||
| let expected_preamble = replace_bytes(preamble_text, PLACEHOLDER, expected_hash.as_bytes()); | ||
|
|
||
| assert!(content.starts_with(&expected_preamble)); | ||
| assert!(content.ends_with(zip_content)); | ||
|
|
||
| let _ = fs::remove_dir_all(&temp_dir); | ||
| } | ||
|
|
||
| #[test] | ||
| fn test_create_exe_zip_creates_parent_dir() { | ||
| let temp_dir = env::temp_dir().join(format!("create_exe_zip_parent_{}", std::process::id())); | ||
| fs::create_dir_all(&temp_dir).unwrap(); | ||
|
|
||
| let preamble_path = temp_dir.join("preamble.sh"); | ||
| let zip_path = temp_dir.join("data.zip"); | ||
| let output_path = temp_dir.join("nested").join("sub").join("output.exe"); | ||
|
|
||
| fs::write(&zip_path, b"content").unwrap(); | ||
| fs::write(&preamble_path, b"preamble").unwrap(); | ||
|
|
||
| create_exe_zip(&preamble_path, &zip_path, &output_path).unwrap(); | ||
| assert!(output_path.exists()); | ||
|
|
||
| let _ = fs::remove_dir_all(&temp_dir); | ||
| } | ||
|
|
||
| #[test] | ||
| fn test_create_exe_zip_missing_files() { | ||
| let temp_dir = env::temp_dir().join(format!("create_exe_zip_err_{}", std::process::id())); | ||
| fs::create_dir_all(&temp_dir).unwrap(); | ||
|
|
||
| let missing_preamble = temp_dir.join("nonexistent_preamble.sh"); | ||
| let zip_path = temp_dir.join("data.zip"); | ||
| let output_path = temp_dir.join("output.exe"); | ||
| fs::write(&zip_path, b"dummy").unwrap(); | ||
|
|
||
| assert!(create_exe_zip(&missing_preamble, &zip_path, &output_path).is_err()); | ||
|
|
||
| let preamble_path = temp_dir.join("preamble.sh"); | ||
| fs::write(&preamble_path, b"preamble").unwrap(); | ||
| let missing_zip = temp_dir.join("nonexistent_data.zip"); | ||
|
|
||
| assert!(create_exe_zip(&preamble_path, &missing_zip, &output_path).is_err()); | ||
|
|
||
| let _ = fs::remove_dir_all(&temp_dir); | ||
| } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| workspace(name = "rules_rust") |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,3 @@ | ||
| package(default_visibility = ["//visibility:public"]) | ||
|
|
||
| exports_files(["defs.bzl"]) |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Should we have a convention to keep the rust tools part of
//tools?There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
maybe? idk.
IMHO, tools/ is for things that are run as build tools. That would be most of the rust stuff, but there's a couple cases that wouldn't be:
There were two other reasons I ended up going with the
crates/<project>layoutcrates/<project>seems the common idiom? This was just after some light searching, didn't look universalI don't have a strong opinion, though I haven't thought about this much, and my brain is mostly mush right now.
The most important thing on my mind is preventing unnecessary dependencies from sneaking into prod code (e.g. a dev/test-only load occurs and then a BULID file can't load)