From ccc26e07eb94d95ed43f7f9094177eb9417a3164 Mon Sep 17 00:00:00 2001 From: Jean-Gabriel Gill-Couture Date: Sat, 21 Mar 2026 11:10:51 -0400 Subject: [PATCH] feat: harmony_asset crate to manage assets, local, s3, http urls, etc --- Cargo.lock | 868 +++++++++++++++++++++++++++-- Cargo.toml | 4 + harmony_assets/Cargo.toml | 56 ++ harmony_assets/src/asset.rs | 80 +++ harmony_assets/src/cli/checksum.rs | 25 + harmony_assets/src/cli/download.rs | 82 +++ harmony_assets/src/cli/mod.rs | 49 ++ harmony_assets/src/cli/upload.rs | 166 ++++++ harmony_assets/src/cli/verify.rs | 32 ++ harmony_assets/src/errors.rs | 37 ++ harmony_assets/src/hash.rs | 233 ++++++++ harmony_assets/src/lib.rs | 14 + harmony_assets/src/store/local.rs | 137 +++++ harmony_assets/src/store/mod.rs | 27 + harmony_assets/src/store/s3.rs | 235 ++++++++ 15 files changed, 1994 insertions(+), 51 deletions(-) create mode 100644 harmony_assets/Cargo.toml create mode 100644 harmony_assets/src/asset.rs create mode 100644 harmony_assets/src/cli/checksum.rs create mode 100644 harmony_assets/src/cli/download.rs create mode 100644 harmony_assets/src/cli/mod.rs create mode 100644 harmony_assets/src/cli/upload.rs create mode 100644 harmony_assets/src/cli/verify.rs create mode 100644 harmony_assets/src/errors.rs create mode 100644 harmony_assets/src/hash.rs create mode 100644 harmony_assets/src/lib.rs create mode 100644 harmony_assets/src/store/local.rs create mode 100644 harmony_assets/src/store/mod.rs create mode 100644 harmony_assets/src/store/s3.rs diff --git a/Cargo.lock b/Cargo.lock index 8293f03..238e89e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -359,6 +359,18 @@ dependencies = [ "rustversion", ] +[[package]] +name = "arrayref" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" + +[[package]] +name = "arrayvec" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" + [[package]] name = "askama" version = "0.14.0" @@ -528,6 +540,476 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +[[package]] +name = "aws-config" +version = "1.8.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11493b0bad143270fb8ad284a096dd529ba91924c5409adeac856cc1bf047dbc" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-sdk-sso", + "aws-sdk-ssooidc", + "aws-sdk-sts", + "aws-smithy-async", + "aws-smithy-http 0.63.6", + "aws-smithy-json 0.62.5", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "hex", + "http 1.4.0", + "sha1", + "time", + "tokio", + "tracing", + "url", + "zeroize", +] + +[[package]] +name = "aws-credential-types" +version = "1.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f20799b373a1be121fe3005fba0c2090af9411573878f224df44b42727fcaf7" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "zeroize", +] + +[[package]] +name = "aws-lc-rs" +version = "1.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a054912289d18629dc78375ba2c3726a3afe3ff71b4edba9dedfca0e3446d1fc" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.39.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa7e52a4c5c547c741610a2c6f123f3881e409b714cd27e6798ef020c514f0a" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", +] + +[[package]] +name = "aws-runtime" +version = "1.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fc0651c57e384202e47153c1260b84a9936e19803d747615edf199dc3b98d17" +dependencies = [ + "aws-credential-types", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-eventstream", + "aws-smithy-http 0.63.6", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "bytes-utils", + "fastrand", + "http 0.2.12", + "http 1.4.0", + "http-body 0.4.6", + "http-body 1.0.1", + "percent-encoding", + "pin-project-lite", + "tracing", + "uuid", +] + +[[package]] +name = "aws-sdk-s3" +version = "1.119.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d65fddc3844f902dfe1864acb8494db5f9342015ee3ab7890270d36fbd2e01c" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-checksums", + "aws-smithy-eventstream", + "aws-smithy-http 0.62.6", + "aws-smithy-json 0.61.9", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "bytes", + "fastrand", + "hex", + "hmac", + "http 0.2.12", + "http 1.4.0", + "http-body 0.4.6", + "lru", + "percent-encoding", + "regex-lite", + "sha2", + "tracing", + "url", +] + +[[package]] +name = "aws-sdk-sso" +version = "1.97.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aadc669e184501caaa6beafb28c6267fc1baef0810fb58f9b205485ca3f2567" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http 0.63.6", + "aws-smithy-json 0.62.5", + "aws-smithy-observability", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 0.2.12", + "http 1.4.0", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-ssooidc" +version = "1.99.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1342a7db8f358d3de0aed2007a0b54e875458e39848d54cc1d46700b2bfcb0a8" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http 0.63.6", + "aws-smithy-json 0.62.5", + "aws-smithy-observability", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 0.2.12", + "http 1.4.0", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-sts" +version = "1.101.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab41ad64e4051ecabeea802d6a17845a91e83287e1dd249e6963ea1ba78c428a" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http 0.63.6", + "aws-smithy-json 0.62.5", + "aws-smithy-observability", + "aws-smithy-query", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "fastrand", + "http 0.2.12", + "http 1.4.0", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sigv4" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0b660013a6683ab23797778e21f1f854744fdf05f68204b4cca4c8c04b5d1f4" +dependencies = [ + "aws-credential-types", + "aws-smithy-eventstream", + "aws-smithy-http 0.63.6", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "crypto-bigint 0.5.5", + "form_urlencoded", + "hex", + "hmac", + "http 0.2.12", + "http 1.4.0", + "p256 0.11.1", + "percent-encoding", + "ring", + "sha2", + "subtle", + "time", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-smithy-async" +version = "1.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ffcaf626bdda484571968400c326a244598634dc75fd451325a54ad1a59acfc" +dependencies = [ + "futures-util", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "aws-smithy-checksums" +version = "0.63.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87294a084b43d649d967efe58aa1f9e0adc260e13a6938eb904c0ae9b45824ae" +dependencies = [ + "aws-smithy-http 0.62.6", + "aws-smithy-types", + "bytes", + "crc-fast", + "hex", + "http 0.2.12", + "http-body 0.4.6", + "md-5", + "pin-project-lite", + "sha1", + "sha2", + "tracing", +] + +[[package]] +name = "aws-smithy-eventstream" +version = "0.60.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faf09d74e5e32f76b8762da505a3cd59303e367a664ca67295387baa8c1d7548" +dependencies = [ + "aws-smithy-types", + "bytes", + "crc32fast", +] + +[[package]] +name = "aws-smithy-http" +version = "0.62.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "826141069295752372f8203c17f28e30c464d22899a43a0c9fd9c458d469c88b" +dependencies = [ + "aws-smithy-eventstream", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "bytes-utils", + "futures-core", + "futures-util", + "http 0.2.12", + "http 1.4.0", + "http-body 0.4.6", + "percent-encoding", + "pin-project-lite", + "pin-utils", + "tracing", +] + +[[package]] +name = "aws-smithy-http" +version = "0.63.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba1ab2dc1c2c3749ead27180d333c42f11be8b0e934058fb4b2258ee8dbe5231" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "bytes-utils", + "futures-core", + "futures-util", + "http 1.4.0", + "http-body 1.0.1", + "http-body-util", + "percent-encoding", + "pin-project-lite", + "pin-utils", + "tracing", +] + +[[package]] +name = "aws-smithy-http-client" +version = "1.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a2f165a7feee6f263028b899d0a181987f4fa7179a6411a32a439fba7c5f769" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "h2 0.3.27", + "h2 0.4.13", + "http 0.2.12", + "http 1.4.0", + "http-body 0.4.6", + "hyper 0.14.32", + "hyper 1.8.1", + "hyper-rustls 0.24.2", + "hyper-rustls 0.27.7", + "hyper-util", + "pin-project-lite", + "rustls 0.21.12", + "rustls 0.23.37", + "rustls-native-certs 0.8.3", + "rustls-pki-types", + "tokio", + "tokio-rustls 0.26.4", + "tower", + "tracing", +] + +[[package]] +name = "aws-smithy-json" +version = "0.61.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49fa1213db31ac95288d981476f78d05d9cbb0353d22cdf3472cc05bb02f6551" +dependencies = [ + "aws-smithy-types", +] + +[[package]] +name = "aws-smithy-json" +version = "0.62.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9648b0bb82a2eedd844052c6ad2a1a822d1f8e3adee5fbf668366717e428856a" +dependencies = [ + "aws-smithy-types", +] + +[[package]] +name = "aws-smithy-observability" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06c2315d173edbf1920da8ba3a7189695827002e4c0fc961973ab1c54abca9c" +dependencies = [ + "aws-smithy-runtime-api", +] + +[[package]] +name = "aws-smithy-query" +version = "0.60.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a56d79744fb3edb5d722ef79d86081e121d3b9422cb209eb03aea6aa4f21ebd" +dependencies = [ + "aws-smithy-types", + "urlencoding", +] + +[[package]] +name = "aws-smithy-runtime" +version = "1.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "028999056d2d2fd58a697232f9eec4a643cf73a71cf327690a7edad1d2af2110" +dependencies = [ + "aws-smithy-async", + "aws-smithy-http 0.63.6", + "aws-smithy-http-client", + "aws-smithy-observability", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "fastrand", + "http 0.2.12", + "http 1.4.0", + "http-body 0.4.6", + "http-body 1.0.1", + "http-body-util", + "pin-project-lite", + "pin-utils", + "tokio", + "tracing", +] + +[[package]] +name = "aws-smithy-runtime-api" +version = "1.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "876ab3c9c29791ba4ba02b780a3049e21ec63dabda09268b175272c3733a79e6" +dependencies = [ + "aws-smithy-async", + "aws-smithy-types", + "bytes", + "http 0.2.12", + "http 1.4.0", + "pin-project-lite", + "tokio", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-smithy-types" +version = "1.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d73dbfbaa8e4bc57b9045137680b958d274823509a360abfd8e1d514d40c95c" +dependencies = [ + "base64-simd", + "bytes", + "bytes-utils", + "futures-core", + "http 0.2.12", + "http 1.4.0", + "http-body 0.4.6", + "http-body 1.0.1", + "http-body-util", + "itoa", + "num-integer", + "pin-project-lite", + "pin-utils", + "ryu", + "serde", + "time", + "tokio", + "tokio-util", +] + +[[package]] +name = "aws-smithy-xml" +version = "0.60.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce02add1aa3677d022f8adf81dcbe3046a95f17a1b1e8979c145cd21d3d22b3" +dependencies = [ + "xmlparser", +] + +[[package]] +name = "aws-types" +version = "1.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47c8323699dd9b3c8d5b3c13051ae9cdef58fd179957c882f8374dd8725962d9" +dependencies = [ + "aws-credential-types", + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "rustc_version", + "tracing", +] + [[package]] name = "backon" version = "1.6.0" @@ -554,6 +1036,12 @@ dependencies = [ "windows-link", ] +[[package]] +name = "base16ct" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349a06037c7bf932dd7e7d1f653678b2038b9ad46a74102f1fc7bd7872678cce" + [[package]] name = "base16ct" version = "0.2.0" @@ -572,6 +1060,16 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64-simd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "339abbe78e73178762e23bea9dfd08e697eb3f3301cd4be981c0f78ba5859195" +dependencies = [ + "outref", + "vsimd", +] + [[package]] name = "base64ct" version = "1.8.3" @@ -625,6 +1123,20 @@ dependencies = [ "wyz", ] +[[package]] +name = "blake3" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2468ef7d57b3fb7e16b576e8377cdbde2320c60e1491e961d11da40fc4f02a2d" +dependencies = [ + "arrayref", + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", + "cpufeatures 0.2.17", +] + [[package]] name = "block-buffer" version = "0.10.4" @@ -803,6 +1315,16 @@ dependencies = [ "serde", ] +[[package]] +name = "bytes-utils" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dafe3a8757b027e2be6e4e5601ed563c55989fcf1546e933c66c8eb3a058d35" +dependencies = [ + "bytes", + "either", +] + [[package]] name = "bytestring" version = "1.5.0" @@ -1020,6 +1542,15 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a822ea5bc7590f9d40f1ba12c0dc3c2760f3482c6984db1573ad11031420831" +[[package]] +name = "cmake" +version = "0.1.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75443c44cd6b379beb8c5b45d85d0773baf31cce901fe7bb252f4eff3008ef7d" +dependencies = [ + "cc", +] + [[package]] name = "color-eyre" version = "0.6.5" @@ -1115,6 +1646,12 @@ dependencies = [ "tiny-keccak", ] +[[package]] +name = "constant_time_eq" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" + [[package]] name = "convert_case" version = "0.8.0" @@ -1203,6 +1740,19 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" +[[package]] +name = "crc-fast" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ddc2d09feefeee8bd78101665bd8645637828fa9317f9f292496dbbd8c65ff3" +dependencies = [ + "crc", + "digest", + "rand 0.9.2", + "regex", + "rustversion", +] + [[package]] name = "crc32fast" version = "1.5.0" @@ -1319,6 +1869,18 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" +[[package]] +name = "crypto-bigint" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef2b4b23cddf68b89b8f8069890e8c270d54e2d5fe1b143820234805e4cb17ef" +dependencies = [ + "generic-array", + "rand_core 0.6.4", + "subtle", + "zeroize", +] + [[package]] name = "crypto-bigint" version = "0.5.5" @@ -1529,6 +2091,16 @@ version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d7a1e2f27636f116493b8b860f5546edb47c8d8f8ea73e1d2a20be88e28d1fea" +[[package]] +name = "der" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1a467a65c5e759bce6e65eaf91cc29f466cdc57cb65777bd646872a8a1fd4de" +dependencies = [ + "const-oid", + "zeroize", +] + [[package]] name = "der" version = "0.7.10" @@ -1740,24 +2312,42 @@ version = "0.15.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + [[package]] name = "dyn-clone" version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" +[[package]] +name = "ecdsa" +version = "0.14.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413301934810f597c1d19ca71c8710e99a3f1ba28a0d2ebc01551a2daeea3c5c" +dependencies = [ + "der 0.6.1", + "elliptic-curve 0.12.3", + "rfc6979 0.3.1", + "signature 1.6.4", +] + [[package]] name = "ecdsa" version = "0.16.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" dependencies = [ - "der", + "der 0.7.10", "digest", - "elliptic-curve", - "rfc6979", - "signature", - "spki", + "elliptic-curve 0.13.8", + "rfc6979 0.4.0", + "signature 2.2.0", + "spki 0.7.3", ] [[package]] @@ -1766,8 +2356,8 @@ version = "2.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" dependencies = [ - "pkcs8", - "signature", + "pkcs8 0.10.2", + "signature 2.2.0", ] [[package]] @@ -1781,7 +2371,7 @@ dependencies = [ "rand_core 0.6.4", "serde", "sha2", - "signature", + "signature 2.2.0", "subtle", "zeroize", ] @@ -1807,23 +2397,43 @@ dependencies = [ "serde", ] +[[package]] +name = "elliptic-curve" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7bb888ab5300a19b8e5bceef25ac745ad065f3c9f7efc6de1b91958110891d3" +dependencies = [ + "base16ct 0.1.1", + "crypto-bigint 0.4.9", + "der 0.6.1", + "digest", + "ff 0.12.1", + "generic-array", + "group 0.12.1", + "pkcs8 0.9.0", + "rand_core 0.6.4", + "sec1 0.3.0", + "subtle", + "zeroize", +] + [[package]] name = "elliptic-curve" version = "0.13.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" dependencies = [ - "base16ct", - "crypto-bigint", + "base16ct 0.2.0", + "crypto-bigint 0.5.5", "digest", - "ff", + "ff 0.13.1", "generic-array", - "group", + "group 0.13.0", "hkdf", "pem-rfc7468", - "pkcs8", + "pkcs8 0.10.2", "rand_core 0.6.4", - "sec1", + "sec1 0.7.3", "subtle", "zeroize", ] @@ -2425,6 +3035,16 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +[[package]] +name = "ff" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d013fc25338cc558c5c2cfbad646908fb23591e2404481826742b651c9af7160" +dependencies = [ + "rand_core 0.6.4", + "subtle", +] + [[package]] name = "ff" version = "0.13.1" @@ -2522,6 +3142,12 @@ dependencies = [ "serde", ] +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + [[package]] name = "funty" version = "2.0.0" @@ -2737,13 +3363,24 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "group" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dfbfb3a6cfbd390d5c9564ab283a0349b9b9fcd46a706c1eb10e0db70bfbac7" +dependencies = [ + "ff 0.12.1", + "rand_core 0.6.4", + "subtle", +] + [[package]] name = "group" version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" dependencies = [ - "ff", + "ff 0.13.1", "rand_core 0.6.4", "subtle", ] @@ -2930,6 +3567,31 @@ dependencies = [ "url", ] +[[package]] +name = "harmony_assets" +version = "0.1.0" +dependencies = [ + "async-trait", + "aws-config", + "aws-sdk-s3", + "blake3", + "clap", + "directories", + "futures-util", + "httptest", + "indicatif", + "inquire 0.7.5", + "log", + "pretty_assertions", + "reqwest 0.12.28", + "sha2", + "tempfile", + "thiserror 2.0.18", + "tokio", + "tokio-test", + "url", +] + [[package]] name = "harmony_cli" version = "0.1.0" @@ -3430,6 +4092,7 @@ dependencies = [ "futures-util", "http 0.2.12", "hyper 0.14.32", + "log", "rustls 0.21.12", "tokio", "tokio-rustls 0.24.1", @@ -4603,20 +5266,37 @@ dependencies = [ "num-traits", ] +[[package]] +name = "outref" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" + [[package]] name = "owo-colors" version = "4.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d211803b9b6b570f68772237e415a029d5a50c65d382910b879fb19d3271f94d" +[[package]] +name = "p256" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51f44edd08f51e2ade572f141051021c5af22677e42b7dd28a88155151c33594" +dependencies = [ + "ecdsa 0.14.8", + "elliptic-curve 0.12.3", + "sha2", +] + [[package]] name = "p256" version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c9863ad85fa8f4460f9c48cb909d38a0d689dba1f6f6988a5e3e0d31071bcd4b" dependencies = [ - "ecdsa", - "elliptic-curve", + "ecdsa 0.16.9", + "elliptic-curve 0.13.8", "primeorder", "sha2", ] @@ -4627,8 +5307,8 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fe42f1670a52a47d448f14b6a5c61dd78fce51856e68edaa38f7ae3a46b8d6b6" dependencies = [ - "ecdsa", - "elliptic-curve", + "ecdsa 0.16.9", + "elliptic-curve 0.13.8", "primeorder", "sha2", ] @@ -4639,9 +5319,9 @@ version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fc9e2161f1f215afdfce23677034ae137bbd45016a880c2eb3ba8eb95f085b2" dependencies = [ - "base16ct", - "ecdsa", - "elliptic-curve", + "base16ct 0.2.0", + "ecdsa 0.16.9", + "elliptic-curve 0.13.8", "primeorder", "rand_core 0.6.4", "sha2", @@ -4821,9 +5501,9 @@ version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" dependencies = [ - "der", - "pkcs8", - "spki", + "der 0.7.10", + "pkcs8 0.10.2", + "spki 0.7.3", ] [[package]] @@ -4834,11 +5514,21 @@ checksum = "e847e2c91a18bfa887dd028ec33f2fe6f25db77db3619024764914affe8b69a6" dependencies = [ "aes", "cbc", - "der", + "der 0.7.10", "pbkdf2 0.12.2", "scrypt", "sha2", - "spki", + "spki 0.7.3", +] + +[[package]] +name = "pkcs8" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9eca2c590a5f85da82668fa685c09ce2888b9430e83299debf1f34b65fd4a4ba" +dependencies = [ + "der 0.6.1", + "spki 0.6.0", ] [[package]] @@ -4847,10 +5537,10 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" dependencies = [ - "der", + "der 0.7.10", "pkcs5", "rand_core 0.6.4", - "spki", + "spki 0.7.3", ] [[package]] @@ -4980,7 +5670,7 @@ version = "0.13.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "353e1ca18966c16d9deb1c69278edbc5f194139612772bd9537af60ac231e1e6" dependencies = [ - "elliptic-curve", + "elliptic-curve 0.13.8", ] [[package]] @@ -5396,6 +6086,17 @@ dependencies = [ "webpki-roots 1.0.6", ] +[[package]] +name = "rfc6979" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7743f17af12fa0b03b803ba12cd6a8d9483a587e89c69445e3909655c0b9fabb" +dependencies = [ + "crypto-bigint 0.4.9", + "hmac", + "zeroize", +] + [[package]] name = "rfc6979" version = "0.4.0" @@ -5447,11 +6148,11 @@ dependencies = [ "num-integer", "num-traits", "pkcs1", - "pkcs8", + "pkcs8 0.10.2", "rand_core 0.6.4", "sha2", - "signature", - "spki", + "signature 2.2.0", + "spki 0.7.3", "subtle", "zeroize", ] @@ -5473,7 +6174,7 @@ dependencies = [ "curve25519-dalek", "des", "digest", - "elliptic-curve", + "elliptic-curve 0.13.8", "flate2", "futures", "generic-array", @@ -5482,7 +6183,7 @@ dependencies = [ "log", "num-bigint", "once_cell", - "p256", + "p256 0.13.2", "p384", "p521", "poly1305", @@ -5523,11 +6224,11 @@ dependencies = [ "cbc", "ctr", "data-encoding", - "der", + "der 0.7.10", "digest", - "ecdsa", + "ecdsa 0.16.9", "ed25519-dalek", - "elliptic-curve", + "elliptic-curve 0.13.8", "futures", "hmac", "home", @@ -5535,22 +6236,22 @@ dependencies = [ "log", "md5", "num-integer", - "p256", + "p256 0.13.2", "p384", "p521", "pbkdf2 0.11.0", "pkcs1", "pkcs5", - "pkcs8", + "pkcs8 0.10.2", "rand 0.8.5", "rand_core 0.6.4", "rsa", "russh-cryptovec", - "sec1", + "sec1 0.7.3", "serde", "sha1", "sha2", - "spki", + "spki 0.7.3", "ssh-encoding", "ssh-key", "thiserror 1.0.69", @@ -5691,6 +6392,7 @@ version = "0.23.37" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "758025cb5fccfd3bc2fd74708fd4682be41d99e5dff73c377c0646c6012c73a4" dependencies = [ + "aws-lc-rs", "log", "once_cell", "ring", @@ -5779,6 +6481,7 @@ version = "0.103.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d7df23109aa6c1567d1c575b9952556388da57401e4ace1d15f79eedad0d8f53" dependencies = [ + "aws-lc-rs", "ring", "rustls-pki-types", "untrusted", @@ -5898,16 +6601,30 @@ dependencies = [ "untrusted", ] +[[package]] +name = "sec1" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3be24c1842290c45df0a7bf069e0c268a747ad05a192f2fd7dcfdbc1cba40928" +dependencies = [ + "base16ct 0.1.1", + "der 0.6.1", + "generic-array", + "pkcs8 0.9.0", + "subtle", + "zeroize", +] + [[package]] name = "sec1" version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" dependencies = [ - "base16ct", - "der", + "base16ct 0.2.0", + "der 0.7.10", "generic-array", - "pkcs8", + "pkcs8 0.10.2", "subtle", "zeroize", ] @@ -6232,12 +6949,22 @@ version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1e303f8205714074f6068773f0e29527e0453937fe837c9717d066635b65f31" dependencies = [ - "pkcs8", + "pkcs8 0.10.2", "rand_core 0.6.4", - "signature", + "signature 2.2.0", "zeroize", ] +[[package]] +name = "signature" +version = "1.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74233d3b3b2f6d4b006dc19dee745e73e2a6bfb6f93607cd3b02bd5b00797d7c" +dependencies = [ + "digest", + "rand_core 0.6.4", +] + [[package]] name = "signature" version = "2.2.0" @@ -6348,6 +7075,16 @@ dependencies = [ "lock_api", ] +[[package]] +name = "spki" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67cf02bbac7a337dc36e4f5a693db6c21e7863f45070f7064577eb4367a3212b" +dependencies = [ + "base64ct", + "der 0.6.1", +] + [[package]] name = "spki" version = "0.7.3" @@ -6355,7 +7092,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" dependencies = [ "base64ct", - "der", + "der 0.7.10", ] [[package]] @@ -6583,14 +7320,14 @@ dependencies = [ "bcrypt-pbkdf", "ed25519-dalek", "num-bigint-dig", - "p256", + "p256 0.13.2", "p384", "p521", "rand_core 0.6.4", "rsa", - "sec1", + "sec1 0.7.3", "sha2", - "signature", + "signature 2.2.0", "ssh-cipher", "ssh-encoding", "subtle", @@ -7029,6 +7766,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-test" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f6d24790a10a7af737693a3e8f1d03faef7e6ca0cc99aae5066f533766de545" +dependencies = [ + "futures-core", + "tokio", + "tokio-stream", +] + [[package]] name = "tokio-tungstenite" version = "0.26.2" @@ -7420,6 +8168,12 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + [[package]] name = "utf-8" version = "0.7.6" @@ -7488,6 +8242,12 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "vsimd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" + [[package]] name = "wait-timeout" version = "0.2.1" @@ -8185,6 +8945,12 @@ version = "0.8.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ae8337f8a065cfc972643663ea4279e04e7256de865aa66fe25cec5fb912d3f" +[[package]] +name = "xmlparser" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" + [[package]] name = "yansi" version = "1.0.1" diff --git a/Cargo.toml b/Cargo.toml index 20ea944..7e959b9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,6 +23,7 @@ members = [ "harmony_agent/deploy", "harmony_node_readiness", "harmony-k8s", + "harmony_assets", ] [workspace.package] @@ -37,6 +38,7 @@ derive-new = "0.7" async-trait = "0.1" tokio = { version = "1.40", features = [ "io-std", + "io-util", "fs", "macros", "rt-multi-thread", @@ -73,6 +75,7 @@ base64 = "0.22.1" tar = "0.4.44" lazy_static = "1.5.0" directories = "6.0.0" +futures-util = "0.3" thiserror = "2.0.14" serde = { version = "1.0.209", features = ["derive", "rc"] } serde_json = "1.0.127" @@ -86,3 +89,4 @@ reqwest = { version = "0.12", features = [ "json", ], default-features = false } assertor = "0.0.4" +tokio-test = "0.4" diff --git a/harmony_assets/Cargo.toml b/harmony_assets/Cargo.toml new file mode 100644 index 0000000..be6d9f4 --- /dev/null +++ b/harmony_assets/Cargo.toml @@ -0,0 +1,56 @@ +[package] +name = "harmony_assets" +edition = "2024" +version.workspace = true +readme.workspace = true +license.workspace = true + +[lib] +name = "harmony_assets" + +[[bin]] +name = "harmony_assets" +path = "src/cli/mod.rs" +required-features = ["cli"] + +[features] +default = ["blake3"] +sha256 = ["dep:sha2"] +blake3 = ["dep:blake3"] +s3 = [ + "dep:aws-sdk-s3", + "dep:aws-config", +] +cli = [ + "dep:clap", + "dep:indicatif", + "dep:inquire", +] +reqwest = ["dep:reqwest"] + +[dependencies] +log.workspace = true +tokio.workspace = true +thiserror.workspace = true +directories.workspace = true +sha2 = { version = "0.10", optional = true } +blake3 = { version = "1.5", optional = true } +reqwest = { version = "0.12", optional = true, default-features = false, features = ["stream", "rustls-tls"] } +futures-util.workspace = true +async-trait.workspace = true +url.workspace = true + +# CLI only +clap = { version = "4.5", features = ["derive"], optional = true } +indicatif = { version = "0.18", optional = true } +inquire = { version = "0.7", optional = true } + +# S3 only +aws-sdk-s3 = { version = "1", optional = true } +aws-config = { version = "1", optional = true } + +[dev-dependencies] +tempfile.workspace = true +httptest = "0.16" +pretty_assertions.workspace = true +tokio-test.workspace = true diff --git a/harmony_assets/src/asset.rs b/harmony_assets/src/asset.rs new file mode 100644 index 0000000..209f159 --- /dev/null +++ b/harmony_assets/src/asset.rs @@ -0,0 +1,80 @@ +use crate::hash::ChecksumAlgo; +use std::path::PathBuf; +use url::Url; + +#[derive(Debug, Clone)] +pub struct Asset { + pub url: Url, + pub checksum: String, + pub checksum_algo: ChecksumAlgo, + pub file_name: String, + pub size: Option, +} + +impl Asset { + pub fn new(url: Url, checksum: String, checksum_algo: ChecksumAlgo, file_name: String) -> Self { + Self { + url, + checksum, + checksum_algo, + file_name, + size: None, + } + } + + pub fn with_size(mut self, size: u64) -> Self { + self.size = Some(size); + self + } + + pub fn formatted_checksum(&self) -> String { + crate::hash::format_checksum(&self.checksum, self.checksum_algo.clone()) + } +} + +#[derive(Debug, Clone)] +pub struct LocalCache { + pub base_dir: PathBuf, +} + +impl LocalCache { + pub fn new(base_dir: PathBuf) -> Self { + Self { base_dir } + } + + pub fn path_for(&self, asset: &Asset) -> PathBuf { + let prefix = &asset.checksum[..16.min(asset.checksum.len())]; + self.base_dir.join(prefix).join(&asset.file_name) + } + + pub fn cache_key_dir(&self, asset: &Asset) -> PathBuf { + let prefix = &asset.checksum[..16.min(asset.checksum.len())]; + self.base_dir.join(prefix) + } + + pub async fn ensure_dir(&self, asset: &Asset) -> Result<(), crate::errors::AssetError> { + let dir = self.cache_key_dir(asset); + tokio::fs::create_dir_all(&dir) + .await + .map_err(|e| crate::errors::AssetError::IoError(e))?; + Ok(()) + } +} + +impl Default for LocalCache { + fn default() -> Self { + let base_dir = directories::ProjectDirs::from("io", "NationTech", "Harmony") + .map(|dirs| dirs.cache_dir().join("assets")) + .unwrap_or_else(|| PathBuf::from("/tmp/harmony_assets")); + Self::new(base_dir) + } +} + +#[derive(Debug, Clone)] +pub struct StoredAsset { + pub url: Url, + pub checksum: String, + pub checksum_algo: ChecksumAlgo, + pub size: u64, + pub key: String, +} diff --git a/harmony_assets/src/cli/checksum.rs b/harmony_assets/src/cli/checksum.rs new file mode 100644 index 0000000..5e4d780 --- /dev/null +++ b/harmony_assets/src/cli/checksum.rs @@ -0,0 +1,25 @@ +use clap::Parser; + +#[derive(Parser, Debug)] +pub struct ChecksumArgs { + pub path: String, + #[arg(short, long, default_value = "blake3")] + pub algo: String, +} + +pub async fn execute(args: ChecksumArgs) -> Result<(), Box> { + use harmony_assets::{ChecksumAlgo, checksum_for_path}; + + let path = std::path::Path::new(&args.path); + if !path.exists() { + eprintln!("Error: File not found: {}", args.path); + std::process::exit(1); + } + + let algo = ChecksumAlgo::from_str(&args.algo)?; + let checksum = checksum_for_path(path, algo.clone()).await?; + + println!("{}:{} {}", algo.name(), checksum, args.path); + + Ok(()) +} diff --git a/harmony_assets/src/cli/download.rs b/harmony_assets/src/cli/download.rs new file mode 100644 index 0000000..95693a5 --- /dev/null +++ b/harmony_assets/src/cli/download.rs @@ -0,0 +1,82 @@ +use clap::Parser; + +#[derive(Parser, Debug)] +pub struct DownloadArgs { + pub url: String, + pub checksum: String, + #[arg(short, long)] + pub output: Option, + #[arg(short, long, default_value = "blake3")] + pub algo: String, +} + +pub async fn execute(args: DownloadArgs) -> Result<(), Box> { + use harmony_assets::{ + Asset, AssetStore, ChecksumAlgo, LocalCache, LocalStore, verify_checksum, + }; + use indicatif::{ProgressBar, ProgressStyle}; + use url::Url; + + let url = Url::parse(&args.url).map_err(|e| format!("Invalid URL: {}", e))?; + + let file_name = args + .output + .or_else(|| { + std::path::Path::new(&args.url) + .file_name() + .and_then(|n| n.to_str()) + .map(|s| s.to_string()) + }) + .unwrap_or_else(|| "download".to_string()); + + let algo = ChecksumAlgo::from_str(&args.algo)?; + let asset = Asset::new(url, args.checksum.clone(), algo.clone(), file_name); + + let cache = LocalCache::default(); + + println!("Downloading: {}", asset.url); + println!("Checksum: {}:{}", algo.name(), args.checksum); + println!("Cache dir: {:?}", cache.base_dir); + + let total_size = asset.size.unwrap_or(0); + let pb = if total_size > 0 { + let pb = ProgressBar::new(total_size); + pb.set_style( + ProgressStyle::default_bar() + .template("{spinner:.green} [{elapsed_precise}] [{bar:40}] {bytes}/{total_bytes} ({bytes_per_sec})")? + .progress_chars("=>-"), + ); + Some(pb) + } else { + None + }; + + let progress_fn: Box) + Send> = Box::new({ + let pb = pb.clone(); + move |bytes, _total| { + if let Some(ref pb) = pb { + pb.set_position(bytes); + } + } + }); + + let store = LocalStore::default(); + let result = store.fetch(&asset, &cache, Some(progress_fn)).await; + + if let Some(pb) = pb { + pb.finish(); + } + + match result { + Ok(path) => { + verify_checksum(&path, &args.checksum, algo).await?; + println!("\nDownloaded to: {:?}", path); + println!("Checksum verified OK"); + Ok(()) + } + Err(e) => { + eprintln!("Download failed: {}", e); + std::process::exit(1); + } + } +} diff --git a/harmony_assets/src/cli/mod.rs b/harmony_assets/src/cli/mod.rs new file mode 100644 index 0000000..503955c --- /dev/null +++ b/harmony_assets/src/cli/mod.rs @@ -0,0 +1,49 @@ +pub mod checksum; +pub mod download; +pub mod upload; +pub mod verify; + +use clap::{Parser, Subcommand}; + +#[derive(Parser, Debug)] +#[command( + name = "harmony_assets", + version, + about = "Asset management CLI for downloading, uploading, and verifying large binary assets" +)] +pub struct Cli { + #[command(subcommand)] + pub command: Commands, +} + +#[derive(Subcommand, Debug)] +pub enum Commands { + Upload(upload::UploadArgs), + Download(download::DownloadArgs), + Checksum(checksum::ChecksumArgs), + Verify(verify::VerifyArgs), +} + +#[tokio::main] +async fn main() -> Result<(), Box> { + log::info!("Starting harmony_assets CLI"); + + let cli = Cli::parse(); + + match cli.command { + Commands::Upload(args) => { + upload::execute(args).await?; + } + Commands::Download(args) => { + download::execute(args).await?; + } + Commands::Checksum(args) => { + checksum::execute(args).await?; + } + Commands::Verify(args) => { + verify::execute(args).await?; + } + } + + Ok(()) +} diff --git a/harmony_assets/src/cli/upload.rs b/harmony_assets/src/cli/upload.rs new file mode 100644 index 0000000..f138a5d --- /dev/null +++ b/harmony_assets/src/cli/upload.rs @@ -0,0 +1,166 @@ +use clap::Parser; +use harmony_assets::{S3Config, S3Store, checksum_for_path_with_progress}; +use indicatif::{ProgressBar, ProgressStyle}; +use std::path::Path; + +#[derive(Parser, Debug)] +pub struct UploadArgs { + pub source: String, + pub key: Option, + #[arg(short, long)] + pub content_type: Option, + #[arg(short, long, default_value_t = true)] + pub public_read: bool, + #[arg(short, long)] + pub endpoint: Option, + #[arg(short, long)] + pub bucket: Option, + #[arg(short, long)] + pub region: Option, + #[arg(short, long)] + pub access_key_id: Option, + #[arg(short, long)] + pub secret_access_key: Option, + #[arg(short, long, default_value = "blake3")] + pub algo: String, + #[arg(short, long, default_value_t = false)] + pub yes: bool, +} + +pub async fn execute(args: UploadArgs) -> Result<(), Box> { + let source_path = Path::new(&args.source); + if !source_path.exists() { + eprintln!("Error: File not found: {}", args.source); + std::process::exit(1); + } + + let key = args.key.unwrap_or_else(|| { + source_path + .file_name() + .and_then(|n| n.to_str()) + .unwrap_or("upload") + .to_string() + }); + + let metadata = tokio::fs::metadata(source_path) + .await + .map_err(|e| format!("Failed to read file metadata: {}", e))?; + let total_size = metadata.len(); + + let endpoint = args + .endpoint + .or_else(|| std::env::var("S3_ENDPOINT").ok()) + .unwrap_or_default(); + let bucket = args + .bucket + .or_else(|| std::env::var("S3_BUCKET").ok()) + .unwrap_or_else(|| { + inquire::Text::new("S3 Bucket name:") + .with_default("harmony-assets") + .prompt() + .unwrap() + }); + let region = args + .region + .or_else(|| std::env::var("S3_REGION").ok()) + .unwrap_or_else(|| { + inquire::Text::new("S3 Region:") + .with_default("us-east-1") + .prompt() + .unwrap() + }); + let access_key_id = args + .access_key_id + .or_else(|| std::env::var("AWS_ACCESS_KEY_ID").ok()); + let secret_access_key = args + .secret_access_key + .or_else(|| std::env::var("AWS_SECRET_ACCESS_KEY").ok()); + + let config = S3Config { + endpoint: if endpoint.is_empty() { + None + } else { + Some(endpoint) + }, + bucket: bucket.clone(), + region: region.clone(), + access_key_id, + secret_access_key, + public_read: args.public_read, + }; + + println!("Upload Configuration:"); + println!(" Source: {}", args.source); + println!(" S3 Key: {}", key); + println!(" Bucket: {}", bucket); + println!(" Region: {}", region); + println!( + " Size: {} bytes ({} MB)", + total_size, + total_size as f64 / 1024.0 / 1024.0 + ); + println!(); + + if !args.yes { + let confirm = inquire::Confirm::new("Proceed with upload?") + .with_default(true) + .prompt()?; + if !confirm { + println!("Upload cancelled."); + return Ok(()); + } + } + + let store = S3Store::new(config) + .await + .map_err(|e| format!("Failed to initialize S3 client: {}", e))?; + + println!("Computing checksum while uploading...\n"); + + let pb = ProgressBar::new(total_size); + pb.set_style( + ProgressStyle::default_bar() + .template("{spinner:.green} [{elapsed_precise}] [{bar:40}] {bytes}/{total_bytes} ({bytes_per_sec})")? + .progress_chars("=>-"), + ); + + { + let algo = harmony_assets::ChecksumAlgo::from_str(&args.algo)?; + let rt = tokio::runtime::Handle::current(); + let pb_clone = pb.clone(); + let _checksum = rt.block_on(checksum_for_path_with_progress( + source_path, + algo, + |read, _total| { + pb_clone.set_position(read); + }, + ))?; + } + + pb.set_position(total_size); + + let result = store + .store(source_path, &key, args.content_type.as_deref()) + .await; + + pb.finish(); + + match result { + Ok(asset) => { + println!("\nUpload complete!"); + println!(" URL: {}", asset.url); + println!( + " Checksum: {}:{}", + asset.checksum_algo.name(), + asset.checksum + ); + println!(" Size: {} bytes", asset.size); + println!(" Key: {}", asset.key); + Ok(()) + } + Err(e) => { + eprintln!("Upload failed: {}", e); + std::process::exit(1); + } + } +} diff --git a/harmony_assets/src/cli/verify.rs b/harmony_assets/src/cli/verify.rs new file mode 100644 index 0000000..52ba450 --- /dev/null +++ b/harmony_assets/src/cli/verify.rs @@ -0,0 +1,32 @@ +use clap::Parser; + +#[derive(Parser, Debug)] +pub struct VerifyArgs { + pub path: String, + pub expected: String, + #[arg(short, long, default_value = "blake3")] + pub algo: String, +} + +pub async fn execute(args: VerifyArgs) -> Result<(), Box> { + use harmony_assets::{ChecksumAlgo, verify_checksum}; + + let path = std::path::Path::new(&args.path); + if !path.exists() { + eprintln!("Error: File not found: {}", args.path); + std::process::exit(1); + } + + let algo = ChecksumAlgo::from_str(&args.algo)?; + + match verify_checksum(path, &args.expected, algo).await { + Ok(()) => { + println!("Checksum verified OK"); + Ok(()) + } + Err(e) => { + eprintln!("Verification FAILED: {}", e); + std::process::exit(1); + } + } +} diff --git a/harmony_assets/src/errors.rs b/harmony_assets/src/errors.rs new file mode 100644 index 0000000..300b306 --- /dev/null +++ b/harmony_assets/src/errors.rs @@ -0,0 +1,37 @@ +use std::path::PathBuf; +use thiserror::Error; + +#[derive(Debug, Error)] +pub enum AssetError { + #[error("File not found: {0}")] + FileNotFound(PathBuf), + + #[error("Checksum mismatch for '{path}': expected {expected}, got {actual}")] + ChecksumMismatch { + path: PathBuf, + expected: String, + actual: String, + }, + + #[error("Checksum algorithm not available: {0}. Enable the corresponding feature flag.")] + ChecksumAlgoNotAvailable(String), + + #[error("Download failed: {0}")] + DownloadFailed(String), + + #[error("S3 error: {0}")] + S3Error(String), + + #[error("IO error: {0}")] + IoError(#[from] std::io::Error), + + #[cfg(feature = "reqwest")] + #[error("HTTP error: {0}")] + HttpError(#[from] reqwest::Error), + + #[error("Store error: {0}")] + StoreError(String), + + #[error("Configuration error: {0}")] + ConfigError(String), +} diff --git a/harmony_assets/src/hash.rs b/harmony_assets/src/hash.rs new file mode 100644 index 0000000..43cd0b4 --- /dev/null +++ b/harmony_assets/src/hash.rs @@ -0,0 +1,233 @@ +use crate::errors::AssetError; +use std::path::Path; + +#[cfg(feature = "blake3")] +use blake3::Hasher as B3Hasher; +#[cfg(feature = "sha256")] +use sha2::{Digest, Sha256}; + +#[derive(Debug, Clone)] +pub enum ChecksumAlgo { + BLAKE3, + SHA256, +} + +impl Default for ChecksumAlgo { + fn default() -> Self { + #[cfg(feature = "blake3")] + return ChecksumAlgo::BLAKE3; + #[cfg(not(feature = "blake3"))] + return ChecksumAlgo::SHA256; + } +} + +impl ChecksumAlgo { + pub fn name(&self) -> &'static str { + match self { + ChecksumAlgo::BLAKE3 => "blake3", + ChecksumAlgo::SHA256 => "sha256", + } + } + + pub fn from_str(s: &str) -> Result { + match s.to_lowercase().as_str() { + "blake3" | "b3" => Ok(ChecksumAlgo::BLAKE3), + "sha256" | "sha-256" => Ok(ChecksumAlgo::SHA256), + _ => Err(AssetError::ChecksumAlgoNotAvailable(s.to_string())), + } + } +} + +impl std::fmt::Display for ChecksumAlgo { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.name()) + } +} + +pub async fn checksum_for_file(reader: R, algo: ChecksumAlgo) -> Result +where + R: tokio::io::AsyncRead + Unpin, +{ + match algo { + #[cfg(feature = "blake3")] + ChecksumAlgo::BLAKE3 => { + let mut hasher = B3Hasher::new(); + let mut reader = reader; + let mut buf = vec![0u8; 65536]; + loop { + let n = tokio::io::AsyncReadExt::read(&mut reader, &mut buf).await?; + if n == 0 { + break; + } + hasher.update(&buf[..n]); + } + Ok(hasher.finalize().to_hex().to_string()) + } + #[cfg(not(feature = "blake3"))] + ChecksumAlgo::BLAKE3 => Err(AssetError::ChecksumAlgoNotAvailable("blake3".to_string())), + #[cfg(feature = "sha256")] + ChecksumAlgo::SHA256 => { + let mut hasher = Sha256::new(); + let mut reader = reader; + let mut buf = vec![0u8; 65536]; + loop { + let n = tokio::io::AsyncReadExt::read(&mut reader, &mut buf).await?; + if n == 0 { + break; + } + hasher.update(&buf[..n]); + } + Ok(format!("{:x}", hasher.finalize())) + } + #[cfg(not(feature = "sha256"))] + ChecksumAlgo::SHA256 => Err(AssetError::ChecksumAlgoNotAvailable("sha256".to_string())), + } +} + +pub async fn checksum_for_path(path: &Path, algo: ChecksumAlgo) -> Result { + let file = tokio::fs::File::open(path) + .await + .map_err(|e| AssetError::IoError(e))?; + let reader = tokio::io::BufReader::with_capacity(65536, file); + checksum_for_file(reader, algo).await +} + +pub async fn checksum_for_path_with_progress( + path: &Path, + algo: ChecksumAlgo, + mut progress: F, +) -> Result +where + F: FnMut(u64, Option) + Send, +{ + let file = tokio::fs::File::open(path) + .await + .map_err(|e| AssetError::IoError(e))?; + let metadata = file.metadata().await.map_err(|e| AssetError::IoError(e))?; + let total = Some(metadata.len()); + let reader = tokio::io::BufReader::with_capacity(65536, file); + + match algo { + #[cfg(feature = "blake3")] + ChecksumAlgo::BLAKE3 => { + let mut hasher = B3Hasher::new(); + let mut reader = reader; + let mut buf = vec![0u8; 65536]; + let mut read: u64 = 0; + loop { + let n = tokio::io::AsyncReadExt::read(&mut reader, &mut buf).await?; + if n == 0 { + break; + } + hasher.update(&buf[..n]); + read += n as u64; + progress(read, total); + } + Ok(hasher.finalize().to_hex().to_string()) + } + #[cfg(not(feature = "blake3"))] + ChecksumAlgo::BLAKE3 => Err(AssetError::ChecksumAlgoNotAvailable("blake3".to_string())), + #[cfg(feature = "sha256")] + ChecksumAlgo::SHA256 => { + let mut hasher = Sha256::new(); + let mut reader = reader; + let mut buf = vec![0u8; 65536]; + let mut read: u64 = 0; + loop { + let n = tokio::io::AsyncReadExt::read(&mut reader, &mut buf).await?; + if n == 0 { + break; + } + hasher.update(&buf[..n]); + read += n as u64; + progress(read, total); + } + Ok(format!("{:x}", hasher.finalize())) + } + #[cfg(not(feature = "sha256"))] + ChecksumAlgo::SHA256 => Err(AssetError::ChecksumAlgoNotAvailable("sha256".to_string())), + } +} + +pub async fn verify_checksum( + path: &Path, + expected: &str, + algo: ChecksumAlgo, +) -> Result<(), AssetError> { + let actual = checksum_for_path(path, algo).await?; + let expected_clean = expected + .trim_start_matches("blake3:") + .trim_start_matches("sha256:") + .trim_start_matches("b3:") + .trim_start_matches("sha-256:"); + if actual != expected_clean { + return Err(AssetError::ChecksumMismatch { + path: path.to_path_buf(), + expected: expected_clean.to_string(), + actual, + }); + } + Ok(()) +} + +pub fn format_checksum(checksum: &str, algo: ChecksumAlgo) -> String { + format!("{}:{}", algo.name(), checksum) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Write; + use tempfile::NamedTempFile; + + async fn create_temp_file(content: &[u8]) -> NamedTempFile { + let mut file = NamedTempFile::new().unwrap(); + file.write_all(content).unwrap(); + file.flush().unwrap(); + file + } + + #[tokio::test] + async fn test_checksum_blake3() { + let file = create_temp_file(b"hello world").await; + let checksum = checksum_for_path(file.path(), ChecksumAlgo::BLAKE3) + .await + .unwrap(); + assert_eq!( + checksum, + "d74981efa70a0c880b8d8c1985d075dbcbf679b99a5f9914e5aaf96b831a9e24" + ); + } + + #[tokio::test] + async fn test_verify_checksum_success() { + let file = create_temp_file(b"hello world").await; + let checksum = checksum_for_path(file.path(), ChecksumAlgo::BLAKE3) + .await + .unwrap(); + let result = verify_checksum(file.path(), &checksum, ChecksumAlgo::BLAKE3).await; + assert!(result.is_ok()); + } + + #[tokio::test] + async fn test_verify_checksum_failure() { + let file = create_temp_file(b"hello world").await; + let result = verify_checksum( + file.path(), + "blake3:0000000000000000000000000000000000000000000000000000000000000000", + ChecksumAlgo::BLAKE3, + ) + .await; + assert!(matches!(result, Err(AssetError::ChecksumMismatch { .. }))); + } + + #[tokio::test] + async fn test_checksum_with_prefix() { + let file = create_temp_file(b"hello world").await; + let checksum = checksum_for_path(file.path(), ChecksumAlgo::BLAKE3) + .await + .unwrap(); + let formatted = format_checksum(&checksum, ChecksumAlgo::BLAKE3); + assert!(formatted.starts_with("blake3:")); + } +} diff --git a/harmony_assets/src/lib.rs b/harmony_assets/src/lib.rs new file mode 100644 index 0000000..6e3a6df --- /dev/null +++ b/harmony_assets/src/lib.rs @@ -0,0 +1,14 @@ +pub mod asset; +pub mod errors; +pub mod hash; +pub mod store; + +pub use asset::{Asset, LocalCache, StoredAsset}; +pub use errors::AssetError; +pub use hash::{ChecksumAlgo, checksum_for_path, checksum_for_path_with_progress, verify_checksum}; +pub use store::AssetStore; + +#[cfg(feature = "s3")] +pub use store::{S3Config, S3Store}; + +pub use store::local::LocalStore; diff --git a/harmony_assets/src/store/local.rs b/harmony_assets/src/store/local.rs new file mode 100644 index 0000000..afbf94d --- /dev/null +++ b/harmony_assets/src/store/local.rs @@ -0,0 +1,137 @@ +use crate::asset::{Asset, LocalCache}; +use crate::errors::AssetError; +use crate::store::AssetStore; +use async_trait::async_trait; +use std::path::PathBuf; +use url::Url; + +#[cfg(feature = "reqwest")] +use crate::hash::verify_checksum; + +#[derive(Debug, Clone)] +pub struct LocalStore { + base_dir: PathBuf, +} + +impl LocalStore { + pub fn new(base_dir: PathBuf) -> Self { + Self { base_dir } + } + + pub fn with_cache(cache: LocalCache) -> Self { + Self { + base_dir: cache.base_dir.clone(), + } + } + + pub fn base_dir(&self) -> &PathBuf { + &self.base_dir + } +} + +impl Default for LocalStore { + fn default() -> Self { + Self::new(LocalCache::default().base_dir) + } +} + +#[async_trait] +impl AssetStore for LocalStore { + #[cfg(feature = "reqwest")] + async fn fetch( + &self, + asset: &Asset, + cache: &LocalCache, + progress: Option) + Send>>, + ) -> Result { + use futures_util::StreamExt; + + let dest_path = cache.path_for(asset); + + if dest_path.exists() { + let verification = + verify_checksum(&dest_path, &asset.checksum, asset.checksum_algo.clone()).await; + if verification.is_ok() { + log::debug!("Asset already cached at {:?}", dest_path); + return Ok(dest_path); + } else { + log::warn!("Cached file failed checksum verification, re-downloading"); + tokio::fs::remove_file(&dest_path) + .await + .map_err(|e| AssetError::IoError(e))?; + } + } + + cache.ensure_dir(asset).await?; + + log::info!("Downloading asset from {}", asset.url); + let client = reqwest::Client::new(); + let response = client + .get(asset.url.as_str()) + .send() + .await + .map_err(|e| AssetError::DownloadFailed(e.to_string()))?; + + if !response.status().is_success() { + return Err(AssetError::DownloadFailed(format!( + "HTTP {}: {}", + response.status(), + asset.url + ))); + } + + let total_size = response.content_length(); + + let mut file = tokio::fs::File::create(&dest_path) + .await + .map_err(|e| AssetError::IoError(e))?; + + let mut stream = response.bytes_stream(); + let mut downloaded: u64 = 0; + + while let Some(chunk_result) = stream.next().await { + let chunk = chunk_result.map_err(|e| AssetError::DownloadFailed(e.to_string()))?; + tokio::io::AsyncWriteExt::write_all(&mut file, &chunk) + .await + .map_err(|e| AssetError::IoError(e))?; + downloaded += chunk.len() as u64; + if let Some(ref p) = progress { + p(downloaded, total_size); + } + } + + tokio::io::AsyncWriteExt::flush(&mut file) + .await + .map_err(|e| AssetError::IoError(e))?; + + drop(file); + + verify_checksum(&dest_path, &asset.checksum, asset.checksum_algo.clone()).await?; + + log::info!("Asset downloaded and verified: {:?}", dest_path); + Ok(dest_path) + } + + #[cfg(not(feature = "reqwest"))] + async fn fetch( + &self, + _asset: &Asset, + _cache: &LocalCache, + _progress: Option) + Send>>, + ) -> Result { + Err(AssetError::DownloadFailed( + "HTTP downloads not available. Enable the 'reqwest' feature.".to_string(), + )) + } + + async fn exists(&self, key: &str) -> Result { + let path = self.base_dir.join(key); + Ok(path.exists()) + } + + fn url_for(&self, key: &str) -> Result { + let path = self.base_dir.join(key); + Url::from_file_path(&path) + .map_err(|_| AssetError::StoreError("Could not convert path to file URL".to_string())) + } +} diff --git a/harmony_assets/src/store/mod.rs b/harmony_assets/src/store/mod.rs new file mode 100644 index 0000000..c386c67 --- /dev/null +++ b/harmony_assets/src/store/mod.rs @@ -0,0 +1,27 @@ +use crate::asset::{Asset, LocalCache}; +use crate::errors::AssetError; +use async_trait::async_trait; +use std::path::PathBuf; +use url::Url; + +pub mod local; + +#[cfg(feature = "s3")] +pub mod s3; + +#[async_trait] +pub trait AssetStore: Send + Sync { + async fn fetch( + &self, + asset: &Asset, + cache: &LocalCache, + progress: Option) + Send>>, + ) -> Result; + + async fn exists(&self, key: &str) -> Result; + + fn url_for(&self, key: &str) -> Result; +} + +#[cfg(feature = "s3")] +pub use s3::{S3Config, S3Store}; diff --git a/harmony_assets/src/store/s3.rs b/harmony_assets/src/store/s3.rs new file mode 100644 index 0000000..97f0cd0 --- /dev/null +++ b/harmony_assets/src/store/s3.rs @@ -0,0 +1,235 @@ +use crate::asset::StoredAsset; +use crate::errors::AssetError; +use crate::hash::ChecksumAlgo; +use async_trait::async_trait; +use aws_sdk_s3::Client as S3Client; +use aws_sdk_s3::primitives::ByteStream; +use aws_sdk_s3::types::ObjectCannedAcl; +use std::path::Path; +use url::Url; + +#[derive(Debug, Clone)] +pub struct S3Config { + pub endpoint: Option, + pub bucket: String, + pub region: String, + pub access_key_id: Option, + pub secret_access_key: Option, + pub public_read: bool, +} + +impl Default for S3Config { + fn default() -> Self { + Self { + endpoint: None, + bucket: String::new(), + region: String::from("us-east-1"), + access_key_id: None, + secret_access_key: None, + public_read: true, + } + } +} + +#[derive(Debug, Clone)] +pub struct S3Store { + client: S3Client, + config: S3Config, +} + +impl S3Store { + pub async fn new(config: S3Config) -> Result { + let mut cfg_builder = aws_config::defaults(aws_config::BehaviorVersion::latest()); + + if let Some(ref endpoint) = config.endpoint { + cfg_builder = cfg_builder.endpoint_url(endpoint); + } + + let cfg = cfg_builder.load().await; + let client = S3Client::new(&cfg); + + Ok(Self { client, config }) + } + + pub fn config(&self) -> &S3Config { + &self.config + } + + fn public_url(&self, key: &str) -> Result { + let url_str = if let Some(ref endpoint) = self.config.endpoint { + format!( + "{}/{}/{}", + endpoint.trim_end_matches('/'), + self.config.bucket, + key + ) + } else { + format!( + "https://{}.s3.{}.amazonaws.com/{}", + self.config.bucket, self.config.region, key + ) + }; + Url::parse(&url_str).map_err(|e| AssetError::S3Error(e.to_string())) + } + + pub async fn store( + &self, + source: &Path, + key: &str, + content_type: Option<&str>, + ) -> Result { + let metadata = tokio::fs::metadata(source) + .await + .map_err(|e| AssetError::IoError(e))?; + let size = metadata.len(); + + let checksum = crate::checksum_for_path(source, ChecksumAlgo::default()) + .await + .map_err(|e| AssetError::StoreError(e.to_string()))?; + + let body = ByteStream::from_path(source).await.map_err(|e| { + AssetError::IoError(std::io::Error::new( + std::io::ErrorKind::Other, + e.to_string(), + )) + })?; + + let mut put_builder = self + .client + .put_object() + .bucket(&self.config.bucket) + .key(key) + .body(body) + .content_length(size as i64) + .metadata("checksum", &checksum); + + if self.config.public_read { + put_builder = put_builder.acl(ObjectCannedAcl::PublicRead); + } + + if let Some(ct) = content_type { + put_builder = put_builder.content_type(ct); + } + + put_builder + .send() + .await + .map_err(|e| AssetError::S3Error(e.to_string()))?; + + Ok(StoredAsset { + url: self.public_url(key)?, + checksum, + checksum_algo: ChecksumAlgo::default(), + size, + key: key.to_string(), + }) + } +} + +use crate::store::AssetStore; +use crate::{Asset, LocalCache}; + +#[async_trait] +impl AssetStore for S3Store { + async fn fetch( + &self, + asset: &Asset, + cache: &LocalCache, + progress: Option) + Send>>, + ) -> Result { + let dest_path = cache.path_for(asset); + + if dest_path.exists() { + let verification = + crate::verify_checksum(&dest_path, &asset.checksum, asset.checksum_algo.clone()) + .await; + if verification.is_ok() { + log::debug!("Asset already cached at {:?}", dest_path); + return Ok(dest_path); + } + } + + cache.ensure_dir(asset).await?; + + log::info!( + "Downloading asset from s3://{}/{}", + self.config.bucket, + asset.url + ); + + let key = extract_s3_key(&asset.url, &self.config.bucket)?; + let obj = self + .client + .get_object() + .bucket(&self.config.bucket) + .key(&key) + .send() + .await + .map_err(|e| AssetError::S3Error(e.to_string()))?; + + let total_size = obj.content_length.unwrap_or(0) as u64; + let mut file = tokio::fs::File::create(&dest_path) + .await + .map_err(|e| AssetError::IoError(e))?; + + let mut stream = obj.body; + let mut downloaded: u64 = 0; + + while let Some(chunk_result) = stream.next().await { + let chunk = chunk_result.map_err(|e| AssetError::S3Error(e.to_string()))?; + tokio::io::AsyncWriteExt::write_all(&mut file, &chunk) + .await + .map_err(|e| AssetError::IoError(e))?; + downloaded += chunk.len() as u64; + if let Some(ref p) = progress { + p(downloaded, Some(total_size)); + } + } + + tokio::io::AsyncWriteExt::flush(&mut file) + .await + .map_err(|e| AssetError::IoError(e))?; + + drop(file); + + crate::verify_checksum(&dest_path, &asset.checksum, asset.checksum_algo.clone()).await?; + + Ok(dest_path) + } + + async fn exists(&self, key: &str) -> Result { + match self + .client + .head_object() + .bucket(&self.config.bucket) + .key(key) + .send() + .await + { + Ok(_) => Ok(true), + Err(e) => { + let err_str = e.to_string(); + if err_str.contains("NoSuchKey") || err_str.contains("NotFound") { + Ok(false) + } else { + Err(AssetError::S3Error(err_str)) + } + } + } + } + + fn url_for(&self, key: &str) -> Result { + self.public_url(key) + } +} + +fn extract_s3_key(url: &Url, bucket: &str) -> Result { + let path = url.path().trim_start_matches('/'); + if let Some(stripped) = path.strip_prefix(&format!("{}/", bucket)) { + Ok(stripped.to_string()) + } else if path == bucket { + Ok(String::new()) + } else { + Ok(path.to_string()) + } +} -- 2.39.5