From 5eb7480565adfaf79bfdf90de6a7ff5ea5dcd0eb Mon Sep 17 00:00:00 2001 From: Weihang Lo Date: Mon, 18 Nov 2024 23:29:01 -0500 Subject: [PATCH] feat(SourceId): use stable hash from rustc-stable-hash This helps `-Ztrim-paths` build a stable cross-platform path for the registry and git sources. Sources files then can be found from the same path when debugging. It also helps cache registry index all at once for all platforms, for example the use case in rust-lang/cargo#14795 (despite they should use `cargo vendor` instead IMO). Some caveats: * Newer cargo will need to re-download files for global caches (index files, git/registry sources). The old cache is still kept and used when running with older cargoes. * Windows is not really covered by the "cross-platform" hash, because path prefix components like `C:` are always there. That means hashes of some sources kind, like local registry and local path, are not going to be real cross-platform stable. There might be hash collisions if you have two registries under the same domain. This won't happen to crates.io, as the infra would have to intentionally put another registry on index.crates.io to collide. We don't consider this is an actual threat model, so we are not going to use any cryptographically secure hash algorithm like BLAKE3. See also --- Cargo.lock | 7 ++ Cargo.toml | 2 + src/cargo/core/source_id.rs | 91 +++++++++++++++---------- src/cargo/util/hasher.rs | 23 +------ tests/testsuite/global_cache_tracker.rs | 16 ++++- 5 files changed, 80 insertions(+), 59 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bdba69d5f51..5518d5fcb96 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -357,6 +357,7 @@ dependencies = [ "regex", "rusqlite", "rustc-hash 2.0.0", + "rustc-stable-hash", "rustfix", "same-file", "semver", @@ -3069,6 +3070,12 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "583034fd73374156e66797ed8e5b0d5690409c9226b22d87cb7f19821c05d152" +[[package]] +name = "rustc-stable-hash" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2febf9acc5ee5e99d1ad0afcdbccc02d87aa3f857a1f01f825b80eacf8edfcd1" + [[package]] name = "rustfix" version = "0.9.0" diff --git a/Cargo.toml b/Cargo.toml index 8e676d2984d..bafde065c09 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -82,6 +82,7 @@ rand = "0.8.5" regex = "1.10.5" rusqlite = { version = "0.32.0", features = ["bundled"] } rustc-hash = "2.0.0" +rustc-stable-hash = "0.1.1" rustfix = { version = "0.9.0", path = "crates/rustfix" } same-file = "1.0.6" schemars = "0.8.21" @@ -194,6 +195,7 @@ rand.workspace = true regex.workspace = true rusqlite.workspace = true rustc-hash.workspace = true +rustc-stable-hash.workspace = true rustfix.workspace = true same-file.workspace = true semver.workspace = true diff --git a/src/cargo/core/source_id.rs b/src/cargo/core/source_id.rs index 663f69b5751..501c88cba72 100644 --- a/src/cargo/core/source_id.rs +++ b/src/cargo/core/source_id.rs @@ -782,70 +782,89 @@ mod tests { // Otherwise please just leave a comment in your PR as to why the hash value is // changing and why the old value can't be easily preserved. // - // The hash value depends on endianness and bit-width, so we only run this test on - // little-endian 64-bit CPUs (such as x86-64 and ARM64) where it matches the - // well-known value. + // The hash value should be stable across platforms, and doesn't depend on + // endianness and bit-width. One caveat is that absolute paths on Windows + // are inherently different than on Unix-like platforms. Unless we omit or + // strip the prefix components (e.g. `C:`), there is not way to have a true + // cross-platform stable hash for absolute paths. #[test] - #[cfg(all(target_endian = "little", target_pointer_width = "64"))] - fn test_cratesio_hash() { - let gctx = GlobalContext::default().unwrap(); - let crates_io = SourceId::crates_io(&gctx).unwrap(); - assert_eq!(crate::util::hex::short_hash(&crates_io), "1ecc6299db9ec823"); - } - - // See the comment in `test_cratesio_hash`. - // - // Only test on non-Windows as paths on Windows will get different hashes. - #[test] - #[cfg(all(target_endian = "little", target_pointer_width = "64", not(windows)))] fn test_stable_hash() { use std::hash::Hasher; use std::path::Path; + use crate::util::StableHasher; + + #[cfg(not(windows))] + let ws_root = Path::new("/tmp/ws"); + #[cfg(windows)] + let ws_root = Path::new(r"C:\\tmp\ws"); + let gen_hash = |source_id: SourceId| { - let mut hasher = std::collections::hash_map::DefaultHasher::new(); - source_id.stable_hash(Path::new("/tmp/ws"), &mut hasher); - hasher.finish() + let mut hasher = StableHasher::new(); + source_id.stable_hash(ws_root, &mut hasher); + Hasher::finish(&hasher) }; + let source_id = SourceId::crates_io(&GlobalContext::default().unwrap()).unwrap(); + assert_eq!(gen_hash(source_id), 7062945687441624357); + assert_eq!(crate::util::hex::short_hash(&source_id), "25cdd57fae9f0462"); + let url = "https://my-crates.io".into_url().unwrap(); let source_id = SourceId::for_registry(&url).unwrap(); - assert_eq!(gen_hash(source_id), 18108075011063494626); - assert_eq!(crate::util::hex::short_hash(&source_id), "fb60813d6cb8df79"); + assert_eq!(gen_hash(source_id), 8310250053664888498); + assert_eq!(crate::util::hex::short_hash(&source_id), "b2d65deb64f05373"); let url = "https://your-crates.io".into_url().unwrap(); let source_id = SourceId::for_alt_registry(&url, "alt").unwrap(); - assert_eq!(gen_hash(source_id), 12862859764592646184); - assert_eq!(crate::util::hex::short_hash(&source_id), "09c10fd0cbd74bce"); + assert_eq!(gen_hash(source_id), 14149534903000258933); + assert_eq!(crate::util::hex::short_hash(&source_id), "755952de063f5dc4"); let url = "sparse+https://my-crates.io".into_url().unwrap(); let source_id = SourceId::for_registry(&url).unwrap(); - assert_eq!(gen_hash(source_id), 8763561830438022424); - assert_eq!(crate::util::hex::short_hash(&source_id), "d1ea0d96f6f759b5"); + assert_eq!(gen_hash(source_id), 16249512552851930162); + assert_eq!(crate::util::hex::short_hash(&source_id), "327cfdbd92dd81e1"); let url = "sparse+https://your-crates.io".into_url().unwrap(); let source_id = SourceId::for_alt_registry(&url, "alt").unwrap(); - assert_eq!(gen_hash(source_id), 5159702466575482972); - assert_eq!(crate::util::hex::short_hash(&source_id), "135d23074253cb78"); + assert_eq!(gen_hash(source_id), 6156697384053352292); + assert_eq!(crate::util::hex::short_hash(&source_id), "64a713b6a6fb7055"); let url = "file:///tmp/ws/crate".into_url().unwrap(); let source_id = SourceId::for_git(&url, GitReference::DefaultBranch).unwrap(); - assert_eq!(gen_hash(source_id), 15332537265078583985); - assert_eq!(crate::util::hex::short_hash(&source_id), "73a808694abda756"); - - let path = Path::new("/tmp/ws/crate"); + assert_eq!(gen_hash(source_id), 473480029881867801); + assert_eq!(crate::util::hex::short_hash(&source_id), "199e591d94239206"); + let path = &ws_root.join("crate"); let source_id = SourceId::for_local_registry(path).unwrap(); - assert_eq!(gen_hash(source_id), 18446533307730842837); - assert_eq!(crate::util::hex::short_hash(&source_id), "52a84cc73f6fd48b"); + #[cfg(not(windows))] + { + assert_eq!(gen_hash(source_id), 11515846423845066584); + assert_eq!(crate::util::hex::short_hash(&source_id), "58d73c154f81d09f"); + } + #[cfg(windows)] + { + assert_eq!(gen_hash(source_id), 6146331155906064276); + assert_eq!(crate::util::hex::short_hash(&source_id), "946fb2239f274c55"); + } let source_id = SourceId::for_path(path).unwrap(); - assert_eq!(gen_hash(source_id), 8764714075439899829); - assert_eq!(crate::util::hex::short_hash(&source_id), "e1ddd48578620fc1"); + assert_eq!(gen_hash(source_id), 215644081443634269); + #[cfg(not(windows))] + assert_eq!(crate::util::hex::short_hash(&source_id), "64bace89c92b101f"); + #[cfg(windows)] + assert_eq!(crate::util::hex::short_hash(&source_id), "01e1e6c391813fb6"); let source_id = SourceId::for_directory(path).unwrap(); - assert_eq!(gen_hash(source_id), 17459999773908528552); - assert_eq!(crate::util::hex::short_hash(&source_id), "6568fe2c2fab5bfe"); + #[cfg(not(windows))] + { + assert_eq!(gen_hash(source_id), 6127590343904940368); + assert_eq!(crate::util::hex::short_hash(&source_id), "505191d1f3920955"); + } + #[cfg(windows)] + { + assert_eq!(gen_hash(source_id), 10423446877655960172); + assert_eq!(crate::util::hex::short_hash(&source_id), "6c8ad69db585a790"); + } } #[test] diff --git a/src/cargo/util/hasher.rs b/src/cargo/util/hasher.rs index 87586c0900f..93c88bb4bf9 100644 --- a/src/cargo/util/hasher.rs +++ b/src/cargo/util/hasher.rs @@ -1,25 +1,6 @@ -//! Implementation of a hasher that produces the same values across releases. +//! A hasher that produces the same values across releases and platforms. //! //! The hasher should be fast and have a low chance of collisions (but is not //! sufficient for cryptographic purposes). -#![allow(deprecated)] -use std::hash::{Hasher, SipHasher}; - -#[derive(Clone)] -pub struct StableHasher(SipHasher); - -impl StableHasher { - pub fn new() -> StableHasher { - StableHasher(SipHasher::new()) - } -} - -impl Hasher for StableHasher { - fn finish(&self) -> u64 { - self.0.finish() - } - fn write(&mut self, bytes: &[u8]) { - self.0.write(bytes) - } -} +pub use rustc_stable_hash::StableSipHasher128 as StableHasher; diff --git a/tests/testsuite/global_cache_tracker.rs b/tests/testsuite/global_cache_tracker.rs index dfd698f5f8b..9a20ef50c18 100644 --- a/tests/testsuite/global_cache_tracker.rs +++ b/tests/testsuite/global_cache_tracker.rs @@ -2004,7 +2004,16 @@ fn compatible_with_older_cargo() { assert_eq!(get_registry_names("src"), ["middle-1.0.0", "new-1.0.0"]); assert_eq!( get_registry_names("cache"), - ["middle-1.0.0.crate", "new-1.0.0.crate", "old-1.0.0.crate"] + // Duplicate crates from two different cache location + // because we're changing how SourceId is hashed. + // This change should be reverted once rust-lang/cargo#14917 lands. + [ + "middle-1.0.0.crate", + "middle-1.0.0.crate", + "new-1.0.0.crate", + "new-1.0.0.crate", + "old-1.0.0.crate" + ] ); // T-0 months: Current version, make sure it can read data from stable, @@ -2027,7 +2036,10 @@ fn compatible_with_older_cargo() { assert_eq!(get_registry_names("src"), ["new-1.0.0"]); assert_eq!( get_registry_names("cache"), - ["middle-1.0.0.crate", "new-1.0.0.crate"] + // Duplicate crates from two different cache location + // because we're changing how SourceId is hashed. + // This change should be reverted once rust-lang/cargo#14917 lands. + ["middle-1.0.0.crate", "new-1.0.0.crate", "new-1.0.0.crate"] ); }