From 32ffc3ef61db74d8c7528cf824d410df9a6968f7 Mon Sep 17 00:00:00 2001 From: Willem Date: Wed, 3 Sep 2025 16:13:51 -0400 Subject: [PATCH] fix: cache works, modified tar building function to ignore adding autogenerated files from harmony, .git, .github, target, and node_modules file paths\ > tar building function orders files and sets mtime to zero so it is consistent across docker image builds --- Cargo.lock | 48 +++++++++++++++ harmony/Cargo.toml | 1 + harmony/src/modules/application/rust.rs | 81 +++++++++++++++++++------ 3 files changed, 112 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b1295be..ad3f964 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1685,6 +1685,16 @@ dependencies = [ "url", ] +[[package]] +name = "example-try-rust-webapp" +version = "0.1.0" +dependencies = [ + "harmony", + "harmony_cli", + "tokio", + "url", +] + [[package]] name = "example-tui" version = "0.1.0" @@ -1700,6 +1710,15 @@ dependencies = [ "url", ] +[[package]] +name = "example_validate_ceph_cluster_health" +version = "0.1.0" +dependencies = [ + "harmony", + "harmony_cli", + "tokio", +] + [[package]] name = "eyre" version = "0.6.12" @@ -2097,6 +2116,7 @@ dependencies = [ "tokio-util", "url", "uuid", + "walkdir", ] [[package]] @@ -4671,6 +4691,15 @@ dependencies = [ "cipher", ] +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "schannel" version = "0.1.27" @@ -5961,6 +5990,16 @@ dependencies = [ "libc", ] +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "want" version = "0.3.1" @@ -6115,6 +6154,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +[[package]] +name = "winapi-util" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0978bf7171b3d90bac376700cb56d606feb40f251a475a5d6634613564460b22" +dependencies = [ + "windows-sys 0.60.2", +] + [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" diff --git a/harmony/Cargo.toml b/harmony/Cargo.toml index 1ba4c94..47e6c5c 100644 --- a/harmony/Cargo.toml +++ b/harmony/Cargo.toml @@ -68,6 +68,7 @@ tar.workspace = true base64.workspace = true once_cell = "1.21.3" harmony-secret-derive = { version = "0.1.0", path = "../harmony_secret_derive" } +walkdir = "2.5.0" [dev-dependencies] pretty_assertions.workspace = true diff --git a/harmony/src/modules/application/rust.rs b/harmony/src/modules/application/rust.rs index 9a4eddf..84605f8 100644 --- a/harmony/src/modules/application/rust.rs +++ b/harmony/src/modules/application/rust.rs @@ -1,4 +1,5 @@ -use std::fs; +use std::fs::{self, File}; +use std::io::Read; use std::path::{Path, PathBuf}; use std::process; use std::sync::Arc; @@ -12,7 +13,8 @@ use dockerfile_builder::instruction_builder::CopyBuilder; use futures_util::StreamExt; use log::{debug, info, log_enabled}; use serde::Serialize; -use tar::Archive; +use tar::{Archive, Builder, Header}; +use walkdir::WalkDir; use crate::config::{REGISTRY_PROJECT, REGISTRY_URL}; use crate::{ @@ -162,7 +164,6 @@ impl RustWebapp { ) -> Result> { debug!("Generating Dockerfile for '{}'", self.name); let dockerfile = self.get_or_build_dockerfile(); - let docker = Docker::connect_with_socket_defaults().unwrap(); let quiet = !log_enabled!(log::Level::Debug); match dockerfile .unwrap() @@ -171,6 +172,14 @@ impl RustWebapp { { Some(path_str) => { debug!("Building from dockerfile {}", path_str); + + let tar_data = self + .create_deterministic_tar(&self.project_root.clone()) + .await + .unwrap(); + + let docker = Docker::connect_with_socket_defaults().unwrap(); + let build_image_options = bollard::query_parameters::BuildImageOptionsBuilder::default() .dockerfile(path_str) @@ -178,25 +187,11 @@ impl RustWebapp { .q(quiet) .version(bollard::query_parameters::BuilderVersion::BuilderV1) .platform("linux/x86_64"); - let mut temp_tar_builder = tar::Builder::new(Vec::new()); - temp_tar_builder - .append_dir_all("", self.project_root.clone()) - .unwrap(); - let archive = temp_tar_builder - .into_inner() - .expect("couldn't finish creating tar"); - let archived_files = Archive::new(archive.as_slice()) - .entries() - .unwrap() - .map(|entry| entry.unwrap().path().unwrap().into_owned()) - .collect::>(); - - debug!("files in docker tar: {:#?}", archived_files); let mut image_build_stream = docker.build_image( build_image_options.build(), None, - Some(body_full(archive.into())), + Some(body_full(tar_data.into())), ); while let Some(msg) = image_build_stream.next().await { @@ -213,6 +208,56 @@ impl RustWebapp { } } + ///normalizes timestamp and ignores files that will bust the docker cach + async fn create_deterministic_tar( + &self, + project_root: &std::path::Path, + ) -> Result, Box> { + debug!("building tar file from project root {:#?}", project_root); + let mut tar_data = Vec::new(); + { + let mut builder = Builder::new(&mut tar_data); + let ignore_prefixes = [ + "target", + ".git", + ".github", + ".harmony_generated", + "node_modules", + ]; + let mut entries: Vec<_> = WalkDir::new(project_root) + .into_iter() + .filter_map(Result::ok) + .filter(|e| e.file_type().is_file()) + .filter(|e| { + let rel_path = e.path().strip_prefix(project_root).unwrap(); + !ignore_prefixes + .iter() + .any(|prefix| rel_path.starts_with(prefix)) + }) + .collect(); + entries.sort_by_key(|e| e.path().to_owned()); + + for entry in entries { + let path = entry.path(); + let rel_path = path.strip_prefix(project_root).unwrap(); + + let mut file = fs::File::open(path)?; + let mut header = Header::new_gnu(); + + header.set_size(entry.metadata()?.len()); + header.set_mode(0o644); + header.set_mtime(0); + header.set_uid(0); + header.set_gid(0); + + builder.append_data(&mut header, rel_path, &mut file)?; + } + + builder.finish()?; + } + Ok(tar_data) + } + /// Tags and pushes a Docker image to the configured remote registry. async fn push_docker_image( &self,