Vendor dependencies for 0.3.0 release

vendor/blake3/src/ffi_avx2.rs

@@ -0,0 +1,65 @@
+use crate::{CVWords, IncrementCounter, BLOCK_LEN, OUT_LEN};
+
+// Note that there is no AVX2 implementation of compress_in_place or
+// compress_xof.
+
+// Unsafe because this may only be called on platforms supporting AVX2.
+pub unsafe fn hash_many<const N: usize>(
+    inputs: &[&[u8; N]],
+    key: &CVWords,
+    counter: u64,
+    increment_counter: IncrementCounter,
+    flags: u8,
+    flags_start: u8,
+    flags_end: u8,
+    out: &mut [u8],
+) {
+    unsafe {
+        // The Rust hash_many implementations do bounds checking on the `out`
+        // array, but the C implementations don't. Even though this is an unsafe
+        // function, assert the bounds here.
+        assert!(out.len() >= inputs.len() * OUT_LEN);
+        ffi::blake3_hash_many_avx2(
+            inputs.as_ptr() as *const *const u8,
+            inputs.len(),
+            N / BLOCK_LEN,
+            key.as_ptr(),
+            counter,
+            increment_counter.yes(),
+            flags,
+            flags_start,
+            flags_end,
+            out.as_mut_ptr(),
+        )
+    }
+}
+
+pub mod ffi {
+    extern "C" {
+        pub fn blake3_hash_many_avx2(
+            inputs: *const *const u8,
+            num_inputs: usize,
+            blocks: usize,
+            key: *const u32,
+            counter: u64,
+            increment_counter: bool,
+            flags: u8,
+            flags_start: u8,
+            flags_end: u8,
+            out: *mut u8,
+        );
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn test_hash_many() {
+        if !crate::platform::avx2_detected() {
+            return;
+        }
+        crate::test::test_hash_many_fn(hash_many, hash_many);
+    }
+}

vendor/blake3/src/ffi_avx512.rs

@@ -0,0 +1,169 @@
+use crate::{CVWords, IncrementCounter, BLOCK_LEN, OUT_LEN};
+
+// Unsafe because this may only be called on platforms supporting AVX-512.
+pub unsafe fn compress_in_place(
+    cv: &mut CVWords,
+    block: &[u8; BLOCK_LEN],
+    block_len: u8,
+    counter: u64,
+    flags: u8,
+) {
+    unsafe {
+        ffi::blake3_compress_in_place_avx512(
+            cv.as_mut_ptr(),
+            block.as_ptr(),
+            block_len,
+            counter,
+            flags,
+        )
+    }
+}
+
+// Unsafe because this may only be called on platforms supporting AVX-512.
+pub unsafe fn compress_xof(
+    cv: &CVWords,
+    block: &[u8; BLOCK_LEN],
+    block_len: u8,
+    counter: u64,
+    flags: u8,
+) -> [u8; 64] {
+    unsafe {
+        let mut out = [0u8; 64];
+        ffi::blake3_compress_xof_avx512(
+            cv.as_ptr(),
+            block.as_ptr(),
+            block_len,
+            counter,
+            flags,
+            out.as_mut_ptr(),
+        );
+        out
+    }
+}
+
+// Unsafe because this may only be called on platforms supporting AVX-512.
+pub unsafe fn hash_many<const N: usize>(
+    inputs: &[&[u8; N]],
+    key: &CVWords,
+    counter: u64,
+    increment_counter: IncrementCounter,
+    flags: u8,
+    flags_start: u8,
+    flags_end: u8,
+    out: &mut [u8],
+) {
+    unsafe {
+        // The Rust hash_many implementations do bounds checking on the `out`
+        // array, but the C implementations don't. Even though this is an unsafe
+        // function, assert the bounds here.
+        assert!(out.len() >= inputs.len() * OUT_LEN);
+        ffi::blake3_hash_many_avx512(
+            inputs.as_ptr() as *const *const u8,
+            inputs.len(),
+            N / BLOCK_LEN,
+            key.as_ptr(),
+            counter,
+            increment_counter.yes(),
+            flags,
+            flags_start,
+            flags_end,
+            out.as_mut_ptr(),
+        )
+    }
+}
+
+// Unsafe because this may only be called on platforms supporting AVX-512.
+#[cfg(unix)]
+pub unsafe fn xof_many(
+    cv: &CVWords,
+    block: &[u8; BLOCK_LEN],
+    block_len: u8,
+    counter: u64,
+    flags: u8,
+    out: &mut [u8],
+) {
+    unsafe {
+        debug_assert_eq!(0, out.len() % BLOCK_LEN, "whole blocks only");
+        ffi::blake3_xof_many_avx512(
+            cv.as_ptr(),
+            block.as_ptr(),
+            block_len,
+            counter,
+            flags,
+            out.as_mut_ptr(),
+            out.len() / BLOCK_LEN,
+        );
+    }
+}
+
+pub mod ffi {
+    extern "C" {
+        pub fn blake3_compress_in_place_avx512(
+            cv: *mut u32,
+            block: *const u8,
+            block_len: u8,
+            counter: u64,
+            flags: u8,
+        );
+        pub fn blake3_compress_xof_avx512(
+            cv: *const u32,
+            block: *const u8,
+            block_len: u8,
+            counter: u64,
+            flags: u8,
+            out: *mut u8,
+        );
+        pub fn blake3_hash_many_avx512(
+            inputs: *const *const u8,
+            num_inputs: usize,
+            blocks: usize,
+            key: *const u32,
+            counter: u64,
+            increment_counter: bool,
+            flags: u8,
+            flags_start: u8,
+            flags_end: u8,
+            out: *mut u8,
+        );
+        #[cfg(unix)]
+        pub fn blake3_xof_many_avx512(
+            cv: *const u32,
+            block: *const u8,
+            block_len: u8,
+            counter: u64,
+            flags: u8,
+            out: *mut u8,
+            outblocks: usize,
+        );
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn test_compress() {
+        if !crate::platform::avx512_detected() {
+            return;
+        }
+        crate::test::test_compress_fn(compress_in_place, compress_xof);
+    }
+
+    #[test]
+    fn test_hash_many() {
+        if !crate::platform::avx512_detected() {
+            return;
+        }
+        crate::test::test_hash_many_fn(hash_many, hash_many);
+    }
+
+    #[cfg(unix)]
+    #[test]
+    fn test_xof_many() {
+        if !crate::platform::avx512_detected() {
+            return;
+        }
+        crate::test::test_xof_many_fn(xof_many);
+    }
+}

vendor/blake3/src/ffi_neon.rs

@@ -0,0 +1,82 @@
+use crate::{CVWords, IncrementCounter, BLOCK_LEN, OUT_LEN};
+
+// Unsafe because this may only be called on platforms supporting NEON.
+pub unsafe fn hash_many<const N: usize>(
+    inputs: &[&[u8; N]],
+    key: &CVWords,
+    counter: u64,
+    increment_counter: IncrementCounter,
+    flags: u8,
+    flags_start: u8,
+    flags_end: u8,
+    out: &mut [u8],
+) {
+    // The Rust hash_many implementations do bounds checking on the `out`
+    // array, but the C implementations don't. Even though this is an unsafe
+    // function, assert the bounds here.
+    assert!(out.len() >= inputs.len() * OUT_LEN);
+    ffi::blake3_hash_many_neon(
+        inputs.as_ptr() as *const *const u8,
+        inputs.len(),
+        N / BLOCK_LEN,
+        key.as_ptr(),
+        counter,
+        increment_counter.yes(),
+        flags,
+        flags_start,
+        flags_end,
+        out.as_mut_ptr(),
+    )
+}
+
+// blake3_neon.c normally depends on blake3_portable.c, because the NEON
+// implementation only provides 4x compression, and it relies on the portable
+// implementation for 1x compression. However, we expose the portable Rust
+// implementation here instead, to avoid linking in unnecessary code.
+#[no_mangle]
+pub extern "C" fn blake3_compress_in_place_portable(
+    cv: *mut u32,
+    block: *const u8,
+    block_len: u8,
+    counter: u64,
+    flags: u8,
+) {
+    unsafe {
+        crate::portable::compress_in_place(
+            &mut *(cv as *mut [u32; 8]),
+            &*(block as *const [u8; 64]),
+            block_len,
+            counter,
+            flags,
+        )
+    }
+}
+
+pub mod ffi {
+    extern "C" {
+        pub fn blake3_hash_many_neon(
+            inputs: *const *const u8,
+            num_inputs: usize,
+            blocks: usize,
+            key: *const u32,
+            counter: u64,
+            increment_counter: bool,
+            flags: u8,
+            flags_start: u8,
+            flags_end: u8,
+            out: *mut u8,
+        );
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn test_hash_many() {
+        // This entire file is gated on feature="neon", so NEON support is
+        // assumed here.
+        crate::test::test_hash_many_fn(hash_many, hash_many);
+    }
+}

vendor/blake3/src/ffi_sse2.rs

@@ -0,0 +1,126 @@
+use crate::{CVWords, IncrementCounter, BLOCK_LEN, OUT_LEN};
+
+// Unsafe because this may only be called on platforms supporting SSE2.
+pub unsafe fn compress_in_place(
+    cv: &mut CVWords,
+    block: &[u8; BLOCK_LEN],
+    block_len: u8,
+    counter: u64,
+    flags: u8,
+) {
+    unsafe {
+        ffi::blake3_compress_in_place_sse2(
+            cv.as_mut_ptr(),
+            block.as_ptr(),
+            block_len,
+            counter,
+            flags,
+        )
+    }
+}
+
+// Unsafe because this may only be called on platforms supporting SSE2.
+pub unsafe fn compress_xof(
+    cv: &CVWords,
+    block: &[u8; BLOCK_LEN],
+    block_len: u8,
+    counter: u64,
+    flags: u8,
+) -> [u8; 64] {
+    unsafe {
+        let mut out = [0u8; 64];
+        ffi::blake3_compress_xof_sse2(
+            cv.as_ptr(),
+            block.as_ptr(),
+            block_len,
+            counter,
+            flags,
+            out.as_mut_ptr(),
+        );
+        out
+    }
+}
+
+// Unsafe because this may only be called on platforms supporting SSE2.
+pub unsafe fn hash_many<const N: usize>(
+    inputs: &[&[u8; N]],
+    key: &CVWords,
+    counter: u64,
+    increment_counter: IncrementCounter,
+    flags: u8,
+    flags_start: u8,
+    flags_end: u8,
+    out: &mut [u8],
+) {
+    unsafe {
+        // The Rust hash_many implementations do bounds checking on the `out`
+        // array, but the C implementations don't. Even though this is an unsafe
+        // function, assert the bounds here.
+        assert!(out.len() >= inputs.len() * OUT_LEN);
+        ffi::blake3_hash_many_sse2(
+            inputs.as_ptr() as *const *const u8,
+            inputs.len(),
+            N / BLOCK_LEN,
+            key.as_ptr(),
+            counter,
+            increment_counter.yes(),
+            flags,
+            flags_start,
+            flags_end,
+            out.as_mut_ptr(),
+        )
+    }
+}
+
+pub mod ffi {
+    extern "C" {
+        pub fn blake3_compress_in_place_sse2(
+            cv: *mut u32,
+            block: *const u8,
+            block_len: u8,
+            counter: u64,
+            flags: u8,
+        );
+        pub fn blake3_compress_xof_sse2(
+            cv: *const u32,
+            block: *const u8,
+            block_len: u8,
+            counter: u64,
+            flags: u8,
+            out: *mut u8,
+        );
+        pub fn blake3_hash_many_sse2(
+            inputs: *const *const u8,
+            num_inputs: usize,
+            blocks: usize,
+            key: *const u32,
+            counter: u64,
+            increment_counter: bool,
+            flags: u8,
+            flags_start: u8,
+            flags_end: u8,
+            out: *mut u8,
+        );
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn test_compress() {
+        if !crate::platform::sse2_detected() {
+            return;
+        }
+        crate::test::test_compress_fn(compress_in_place, compress_xof);
+    }
+
+    #[test]
+    fn test_hash_many() {
+        if !crate::platform::sse2_detected() {
+            return;
+        }
+        crate::test::test_hash_many_fn(hash_many, hash_many);
+    }
+}

vendor/blake3/src/ffi_sse41.rs

@@ -0,0 +1,126 @@
+use crate::{CVWords, IncrementCounter, BLOCK_LEN, OUT_LEN};
+
+// Unsafe because this may only be called on platforms supporting SSE4.1.
+pub unsafe fn compress_in_place(
+    cv: &mut CVWords,
+    block: &[u8; BLOCK_LEN],
+    block_len: u8,
+    counter: u64,
+    flags: u8,
+) {
+    unsafe {
+        ffi::blake3_compress_in_place_sse41(
+            cv.as_mut_ptr(),
+            block.as_ptr(),
+            block_len,
+            counter,
+            flags,
+        )
+    }
+}
+
+// Unsafe because this may only be called on platforms supporting SSE4.1.
+pub unsafe fn compress_xof(
+    cv: &CVWords,
+    block: &[u8; BLOCK_LEN],
+    block_len: u8,
+    counter: u64,
+    flags: u8,
+) -> [u8; 64] {
+    unsafe {
+        let mut out = [0u8; 64];
+        ffi::blake3_compress_xof_sse41(
+            cv.as_ptr(),
+            block.as_ptr(),
+            block_len,
+            counter,
+            flags,
+            out.as_mut_ptr(),
+        );
+        out
+    }
+}
+
+// Unsafe because this may only be called on platforms supporting SSE4.1.
+pub unsafe fn hash_many<const N: usize>(
+    inputs: &[&[u8; N]],
+    key: &CVWords,
+    counter: u64,
+    increment_counter: IncrementCounter,
+    flags: u8,
+    flags_start: u8,
+    flags_end: u8,
+    out: &mut [u8],
+) {
+    unsafe {
+        // The Rust hash_many implementations do bounds checking on the `out`
+        // array, but the C implementations don't. Even though this is an unsafe
+        // function, assert the bounds here.
+        assert!(out.len() >= inputs.len() * OUT_LEN);
+        ffi::blake3_hash_many_sse41(
+            inputs.as_ptr() as *const *const u8,
+            inputs.len(),
+            N / BLOCK_LEN,
+            key.as_ptr(),
+            counter,
+            increment_counter.yes(),
+            flags,
+            flags_start,
+            flags_end,
+            out.as_mut_ptr(),
+        )
+    }
+}
+
+pub mod ffi {
+    extern "C" {
+        pub fn blake3_compress_in_place_sse41(
+            cv: *mut u32,
+            block: *const u8,
+            block_len: u8,
+            counter: u64,
+            flags: u8,
+        );
+        pub fn blake3_compress_xof_sse41(
+            cv: *const u32,
+            block: *const u8,
+            block_len: u8,
+            counter: u64,
+            flags: u8,
+            out: *mut u8,
+        );
+        pub fn blake3_hash_many_sse41(
+            inputs: *const *const u8,
+            num_inputs: usize,
+            blocks: usize,
+            key: *const u32,
+            counter: u64,
+            increment_counter: bool,
+            flags: u8,
+            flags_start: u8,
+            flags_end: u8,
+            out: *mut u8,
+        );
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn test_compress() {
+        if !crate::platform::sse41_detected() {
+            return;
+        }
+        crate::test::test_compress_fn(compress_in_place, compress_xof);
+    }
+
+    #[test]
+    fn test_hash_many() {
+        if !crate::platform::sse41_detected() {
+            return;
+        }
+        crate::test::test_hash_many_fn(hash_many, hash_many);
+    }
+}

vendor/blake3/src/guts.rs

@@ -0,0 +1,60 @@
+//! Deprecated in favor of [`hazmat`](crate::hazmat)
+
+pub use crate::{BLOCK_LEN, CHUNK_LEN};
+
+#[derive(Clone, Debug)]
+pub struct ChunkState(crate::ChunkState);
+
+impl ChunkState {
+    // Currently this type only supports the regular hash mode. If an
+    // incremental user needs keyed_hash or derive_key, we can add that.
+    pub fn new(chunk_counter: u64) -> Self {
+        Self(crate::ChunkState::new(
+            crate::IV,
+            chunk_counter,
+            0,
+            crate::platform::Platform::detect(),
+        ))
+    }
+
+    #[inline]
+    pub fn len(&self) -> usize {
+        self.0.count()
+    }
+
+    #[inline]
+    pub fn update(&mut self, input: &[u8]) -> &mut Self {
+        self.0.update(input);
+        self
+    }
+
+    pub fn finalize(&self, is_root: bool) -> crate::Hash {
+        let output = self.0.output();
+        if is_root {
+            output.root_hash()
+        } else {
+            output.chaining_value().into()
+        }
+    }
+}
+
+// As above, this currently assumes the regular hash mode. If an incremental
+// user needs keyed_hash or derive_key, we can add that.
+pub fn parent_cv(
+    left_child: &crate::Hash,
+    right_child: &crate::Hash,
+    is_root: bool,
+) -> crate::Hash {
+    let output = crate::parent_node_output(
+        left_child.as_bytes(),
+        right_child.as_bytes(),
+        crate::IV,
+        0,
+        crate::platform::Platform::detect(),
+    );
+    if is_root {
+        output.root_hash()
+    } else {
+        output.chaining_value().into()
+    }
+}

vendor/blake3/src/hazmat.rs

@@ -0,0 +1,704 @@
+//! Low-level tree manipulations and other sharp tools
+//!
+//! The target audience for this module is projects like [Bao](https://github.com/oconnor663/bao),
+//! which work directly with the interior hashes ("chaining values") of BLAKE3 chunks and subtrees.
+//! For example, you could use these functions to implement a BitTorrent-like protocol using the
+//! BLAKE3 tree structure, or to hash an input that's distributed across different machines. These
+//! use cases are advanced, and most applications don't need this module. Also:
+//!
+//! <div class="warning">
+//!
+//! **Warning:** This module is *hazardous material*. If you've heard folks say *don't roll your
+//! own crypto,* this is the sort of thing they're talking about. These functions have complicated
+//! requirements, and any mistakes will give you garbage output and/or break the security
+//! properties that BLAKE3 is supposed to have. Read section 2.1 of [the BLAKE3
+//! paper](https://github.com/BLAKE3-team/BLAKE3-specs/blob/master/blake3.pdf) to understand the
+//! tree structure you need to maintain. Test your code against [`blake3::hash`](../fn.hash.html)
+//! and make sure you can get the same outputs for [lots of different
+//! inputs](https://github.com/BLAKE3-team/BLAKE3/blob/master/test_vectors/test_vectors.json).
+//!
+//! </div>
+//!
+//! On the other hand:
+//!
+//! <div class="warning">
+//!
+//! **Encouragement:** Playing with these functions is a great way to learn how BLAKE3 works on the
+//! inside. Have fun!
+//!
+//! </div>
+//!
+//! The main entrypoint for this module is the [`HasherExt`] trait, particularly the
+//! [`set_input_offset`](HasherExt::set_input_offset) and
+//! [`finalize_non_root`](HasherExt::finalize_non_root) methods. These let you compute the chaining
+//! values of individual chunks or subtrees. You then combine these chaining values into larger
+//! subtrees using [`merge_subtrees_non_root`] and finally (once at the very top)
+//! [`merge_subtrees_root`] or [`merge_subtrees_root_xof`].
+//!
+//! # Examples
+//!
+//! Here's an example of computing all the interior hashes in a 3-chunk tree:
+//!
+//! ```text
+//!            root
+//!          /      \
+//!      parent      \
+//!    /       \      \
+//! chunk0  chunk1  chunk2
+//! ```
+//!
+//! ```
+//! # fn main() {
+//! use blake3::{Hasher, CHUNK_LEN};
+//! use blake3::hazmat::{merge_subtrees_non_root, merge_subtrees_root, Mode};
+//! use blake3::hazmat::HasherExt; // an extension trait for Hasher
+//!
+//! let chunk0 = [b'a'; CHUNK_LEN];
+//! let chunk1 = [b'b'; CHUNK_LEN];
+//! let chunk2 = [b'c'; 42]; // The final chunk can be short.
+//!
+//! // Compute the non-root hashes ("chaining values") of all three chunks. Chunks or subtrees
+//! // that don't begin at the start of the input use `set_input_offset` to say where they begin.
+//! let chunk0_cv = Hasher::new()
+//!     // .set_input_offset(0) is the default.
+//!     .update(&chunk0)
+//!     .finalize_non_root();
+//! let chunk1_cv = Hasher::new()
+//!     .set_input_offset(CHUNK_LEN as u64)
+//!     .update(&chunk1)
+//!     .finalize_non_root();
+//! let chunk2_cv = Hasher::new()
+//!     .set_input_offset(2 * CHUNK_LEN as u64)
+//!     .update(&chunk2)
+//!     .finalize_non_root();
+//!
+//! // Join the first two chunks with a non-root parent node and compute its chaining value.
+//! let parent_cv = merge_subtrees_non_root(&chunk0_cv, &chunk1_cv, Mode::Hash);
+//!
+//! // Join that parent node and the third chunk with a root parent node and compute the hash.
+//! let root_hash = merge_subtrees_root(&parent_cv, &chunk2_cv, Mode::Hash);
+//!
+//! // Double check that we got the right answer.
+//! let mut combined_input = Vec::new();
+//! combined_input.extend_from_slice(&chunk0);
+//! combined_input.extend_from_slice(&chunk1);
+//! combined_input.extend_from_slice(&chunk2);
+//! assert_eq!(root_hash, blake3::hash(&combined_input));
+//! # }
+//! ```
+//!
+//! Hashing many chunks together is important for performance, because it allows the implementation
+//! to use SIMD parallelism internally. ([AVX-512](https://en.wikipedia.org/wiki/AVX-512) for
+//! example needs 16 chunks to really get going.) We can reproduce `parent_cv` by hashing `chunk0`
+//! and `chunk1` at the same time:
+//!
+//! ```
+//! # fn main() {
+//! # use blake3::{Hasher, CHUNK_LEN};
+//! # use blake3::hazmat::{Mode, HasherExt, merge_subtrees_non_root, merge_subtrees_root};
+//! # let chunk0 = [b'a'; CHUNK_LEN];
+//! # let chunk1 = [b'b'; CHUNK_LEN];
+//! # let chunk0_cv = Hasher::new().update(&chunk0).finalize_non_root();
+//! # let chunk1_cv = Hasher::new().set_input_offset(CHUNK_LEN as u64).update(&chunk1).finalize_non_root();
+//! # let parent_cv = merge_subtrees_non_root(&chunk0_cv, &chunk1_cv, Mode::Hash);
+//! # let mut combined_input = Vec::new();
+//! # combined_input.extend_from_slice(&chunk0);
+//! # combined_input.extend_from_slice(&chunk1);
+//! let left_subtree_cv = Hasher::new()
+//!     // .set_input_offset(0) is the default.
+//!     .update(&combined_input[..2 * CHUNK_LEN])
+//!     .finalize_non_root();
+//! assert_eq!(left_subtree_cv, parent_cv);
+//!
+//! // Using multiple updates gives the same answer, though it's not as efficient.
+//! let mut subtree_hasher = Hasher::new();
+//! // Again, .set_input_offset(0) is the default.
+//! subtree_hasher.update(&chunk0);
+//! subtree_hasher.update(&chunk1);
+//! assert_eq!(left_subtree_cv, subtree_hasher.finalize_non_root());
+//! # }
+//! ```
+//!
+//! However, hashing multiple chunks together **must** respect the overall tree structure. Hashing
+//! `chunk0` and `chunk1` together is valid, but hashing `chunk1` and `chunk2` together is
+//! incorrect and gives a garbage result that will never match a standard BLAKE3 hash. The
+//! implementation includes a few best-effort asserts to catch some of these mistakes, but these
+//! checks aren't guaranteed. For example, this second call to `update` currently panics:
+//!
+//! ```should_panic
+//! # fn main() {
+//! # use blake3::{Hasher, CHUNK_LEN};
+//! # use blake3::hazmat::HasherExt;
+//! # let chunk0 = [b'a'; CHUNK_LEN];
+//! # let chunk1 = [b'b'; CHUNK_LEN];
+//! # let chunk2 = [b'c'; 42];
+//! let oops = Hasher::new()
+//!     .set_input_offset(CHUNK_LEN as u64)
+//!     .update(&chunk1)
+//!     // PANIC: "the subtree starting at 1024 contains at most 1024 bytes"
+//!     .update(&chunk2)
+//!     .finalize_non_root();
+//! # }
+//! ```
+//!
+//! For more on valid tree structures, see the docs for and [`left_subtree_len`] and
+//! [`max_subtree_len`], and see section 2.1 of [the BLAKE3
+//! paper](https://github.com/BLAKE3-team/BLAKE3-specs/blob/master/blake3.pdf). Note that the
+//! merging functions ([`merge_subtrees_root`] and friends) don't know the shape of the left and
+//! right subtrees you're giving them, and they can't help you catch mistakes. The best way to
+//! catch mistakes with these is to compare your root output to the [`blake3::hash`](crate::hash)
+//! of the same input.
+
+use crate::platform::Platform;
+use crate::{CVWords, Hasher, CHUNK_LEN, IV, KEY_LEN, OUT_LEN};
+
+/// Extension methods for [`Hasher`]. This is the main entrypoint to the `hazmat` module.
+pub trait HasherExt {
+    /// Similar to [`Hasher::new_derive_key`] but using a pre-hashed [`ContextKey`] from
+    /// [`hash_derive_key_context`].
+    ///
+    /// The [`hash_derive_key_context`] function is _only_ valid source of the [`ContextKey`]
+    ///
+    /// # Example
+    ///
+    /// ```
+    /// use blake3::Hasher;
+    /// use blake3::hazmat::HasherExt;
+    ///
+    /// let context_key = blake3::hazmat::hash_derive_key_context("foo");
+    /// let mut hasher = Hasher::new_from_context_key(&context_key);
+    /// hasher.update(b"bar");
+    /// let derived_key = *hasher.finalize().as_bytes();
+    ///
+    /// assert_eq!(derived_key, blake3::derive_key("foo", b"bar"));
+    /// ```
+    fn new_from_context_key(context_key: &ContextKey) -> Self;
+
+    /// Configure the `Hasher` to process a chunk or subtree starting at `offset` bytes into the
+    /// whole input.
+    ///
+    /// You must call this function before processing any input with [`update`](Hasher::update) or
+    /// similar. This step isn't required for the first chunk, or for a subtree that includes the
+    /// first chunk (i.e. when the `offset` is zero), but it's required for all other chunks and
+    /// subtrees.
+    ///
+    /// The starting input offset of a subtree implies a maximum possible length for that subtree.
+    /// See [`max_subtree_len`] and section 2.1 of [the BLAKE3
+    /// paper](https://github.com/BLAKE3-team/BLAKE3-specs/blob/master/blake3.pdf). Note that only
+    /// subtrees along the right edge of the whole tree can have a length less than their maximum
+    /// possible length.
+    ///
+    /// See the [module level examples](index.html#examples).
+    ///
+    /// # Panics
+    ///
+    /// This function panics if the `Hasher` has already accepted any input with
+    /// [`update`](Hasher::update) or similar.
+    ///
+    /// This should always be paired with [`finalize_non_root`](HasherExt::finalize_non_root). It's
+    /// never correct to use a non-zero input offset with [`finalize`](Hasher::finalize) or
+    /// [`finalize_xof`](Hasher::finalize_xof). The `offset` must also be a multiple of
+    /// `CHUNK_LEN`. Violating either of these rules will currently fail an assertion and panic,
+    /// but this is not guaranteed.
+    fn set_input_offset(&mut self, offset: u64) -> &mut Self;
+
+    /// Finalize the non-root hash ("chaining value") of the current chunk or subtree.
+    ///
+    /// Afterwards you can merge subtree chaining values into parent nodes using
+    /// [`merge_subtrees_non_root`] and ultimately into the root node with either
+    /// [`merge_subtrees_root`] (similar to [`Hasher::finalize`]) or [`merge_subtrees_root_xof`]
+    /// (similar to [`Hasher::finalize_xof`]).
+    ///
+    /// See the [module level examples](index.html#examples), particularly the discussion of valid
+    /// tree structures.
+    fn finalize_non_root(&self) -> ChainingValue;
+}
+
+impl HasherExt for Hasher {
+    fn new_from_context_key(context_key: &[u8; KEY_LEN]) -> Hasher {
+        let context_key_words = crate::platform::words_from_le_bytes_32(context_key);
+        Hasher::new_internal(&context_key_words, crate::DERIVE_KEY_MATERIAL)
+    }
+
+    fn set_input_offset(&mut self, offset: u64) -> &mut Hasher {
+        assert_eq!(self.count(), 0, "hasher has already accepted input");
+        assert_eq!(
+            offset % CHUNK_LEN as u64,
+            0,
+            "offset ({offset}) must be a chunk boundary (divisible by {CHUNK_LEN})",
+        );
+        let counter = offset / CHUNK_LEN as u64;
+        self.chunk_state.chunk_counter = counter;
+        self.initial_chunk_counter = counter;
+        self
+    }
+
+    fn finalize_non_root(&self) -> ChainingValue {
+        assert_ne!(self.count(), 0, "empty subtrees are never valid");
+        self.final_output().chaining_value()
+    }
+}
+
+/// The maximum length of a subtree in bytes, given its starting offset in bytes
+///
+/// If you try to hash more than this many bytes as one subtree, you'll end up merging parent nodes
+/// that shouldn't be merged, and your output will be garbage. [`Hasher::update`] will currently
+/// panic in this case, but this is not guaranteed.
+///
+/// For input offset zero (the default), there is no maximum length, and this function returns
+/// `None`. For all other offsets it returns `Some`. Note that valid offsets must be a multiple of
+/// [`CHUNK_LEN`] (1024); it's not possible to start hashing a chunk in the middle.
+///
+/// In the example tree below, chunks are numbered by their _0-based index_. The subtree that
+/// _starts_ with chunk 3, i.e. `input_offset = 3 * CHUNK_LEN`, includes only that one chunk, so
+/// its max length is `Some(CHUNK_LEN)`. The subtree that starts with chunk 6 includes chunk 7 but
+/// not chunk 8, so its max length is `Some(2 * CHUNK_LEN)`. The subtree that starts with chunk 12
+/// includes chunks 13, 14, and 15, but if the tree were bigger it would not include chunk 16, so
+/// its max length is `Some(4 * CHUNK_LEN)`. One way to think about the rule here is that, if you
+/// go beyond the max subtree length from a given starting offset, you start dealing with subtrees
+/// that include chunks _to the left_ of where you started.
+///
+/// ```text
+///                           root
+///                 /                       \
+///              .                             .
+///        /           \                 /           \
+///       .             .               .             .
+///    /    \         /    \         /    \         /    \
+///   .      .       .      .       .      .       .      .
+///  / \    / \     / \    / \     / \    / \     / \    / \
+/// 0  1   2  3    4  5   6  7    8  9   10 11   12 13  14 15
+/// ```
+///
+/// The general rule turns out to be that for a subtree starting at a 0-based chunk index N greater
+/// than zero, the maximum number of chunks in that subtree is the largest power-of-two that
+/// divides N, which is given by `1 << N.trailing_zeros()`.
+///
+/// This function can be useful for writing tests or debug assertions, but it's actually rare to
+/// use this for real control flow. Callers who split their input recursively using
+/// [`left_subtree_len`] will automatically satisfy the `max_subtree_len` bound and don't
+/// necessarily need to check. It's also common to choose some fixed power-of-two subtree size, say
+/// 64 chunks, and divide your input up into slices of that fixed length (with the final slice
+/// possibly short). This approach also automatically satisfies the `max_subtree_len` bound and
+/// doesn't need to check. Proving that this is true can be an interesting exercise. Note that
+/// chunks 0, 4, 8, and 12 all begin subtrees of at least 4 chunks in the example tree above.
+///
+/// # Panics
+///
+/// This function currently panics if `input_offset` is not a multiple of `CHUNK_LEN`. This is not
+/// guaranteed.
+#[inline(always)]
+pub fn max_subtree_len(input_offset: u64) -> Option<u64> {
+    if input_offset == 0 {
+        return None;
+    }
+    assert_eq!(input_offset % CHUNK_LEN as u64, 0);
+    let counter = input_offset / CHUNK_LEN as u64;
+    let max_chunks = 1 << counter.trailing_zeros();
+    Some(max_chunks * CHUNK_LEN as u64)
+}
+
+#[test]
+fn test_max_subtree_len() {
+    assert_eq!(max_subtree_len(0), None);
+    // (chunk index, max chunks)
+    let cases = [
+        (1, 1),
+        (2, 2),
+        (3, 1),
+        (4, 4),
+        (5, 1),
+        (6, 2),
+        (7, 1),
+        (8, 8),
+    ];
+    for (chunk_index, max_chunks) in cases {
+        let input_offset = chunk_index * CHUNK_LEN as u64;
+        assert_eq!(
+            max_subtree_len(input_offset),
+            Some(max_chunks * CHUNK_LEN as u64),
+        );
+    }
+}
+
+/// Given the length in bytes of either a complete input or a subtree input, return the number of
+/// bytes that belong to its left child subtree. The rest belong to its right child subtree.
+///
+/// Concretely, this function returns the largest power-of-two number of bytes that's strictly less
+/// than `input_len`. This leads to a tree where all left subtrees are "complete" and at least as
+/// large as their sibling right subtrees, as specified in section 2.1 of [the BLAKE3
+/// paper](https://github.com/BLAKE3-team/BLAKE3-specs/blob/master/blake3.pdf). For example, if an
+/// input is exactly two chunks, its left and right subtrees both get one chunk. But if an input is
+/// two chunks plus one more byte, then its left subtree gets two chunks, and its right subtree
+/// only gets one byte.
+///
+/// This function isn't meaningful for one chunk of input, because chunks don't have children. It
+/// currently panics in debug mode if `input_len <= CHUNK_LEN`.
+///
+/// # Example
+///
+/// Hash a input of random length as two subtrees:
+///
+/// ```
+/// # #[cfg(feature = "std")] {
+/// use blake3::hazmat::{left_subtree_len, merge_subtrees_root, HasherExt, Mode};
+/// use blake3::{Hasher, CHUNK_LEN};
+///
+/// // Generate a random-length input. Note that to be split into two subtrees, the input length
+/// // must be greater than CHUNK_LEN.
+/// let input_len = rand::random_range(CHUNK_LEN + 1..1_000_000);
+/// let mut input = vec![0; input_len];
+/// rand::fill(&mut input[..]);
+///
+/// // Compute the left and right subtree hashes and then the root hash. left_subtree_len() tells
+/// // us exactly where to split the input. Any other split would either panic (if we're lucky) or
+/// // lead to an incorrect root hash.
+/// let left_len = left_subtree_len(input_len as u64) as usize;
+/// let left_subtree_cv = Hasher::new()
+///     .update(&input[..left_len])
+///     .finalize_non_root();
+/// let right_subtree_cv = Hasher::new()
+///     .set_input_offset(left_len as u64)
+///     .update(&input[left_len..])
+///     .finalize_non_root();
+/// let root_hash = merge_subtrees_root(&left_subtree_cv, &right_subtree_cv, Mode::Hash);
+///
+/// // Double check the answer.
+/// assert_eq!(root_hash, blake3::hash(&input));
+/// # }
+/// ```
+#[inline(always)]
+pub fn left_subtree_len(input_len: u64) -> u64 {
+    debug_assert!(input_len > CHUNK_LEN as u64);
+    // Note that .next_power_of_two() is greater than *or equal*.
+    ((input_len + 1) / 2).next_power_of_two()
+}
+
+#[test]
+fn test_left_subtree_len() {
+    assert_eq!(left_subtree_len(1025), 1024);
+    for boundary_case in [2, 4, 8, 16, 32, 64] {
+        let input_len = boundary_case * CHUNK_LEN as u64;
+        assert_eq!(left_subtree_len(input_len - 1), input_len / 2);
+        assert_eq!(left_subtree_len(input_len), input_len / 2);
+        assert_eq!(left_subtree_len(input_len + 1), input_len);
+    }
+}
+
+/// The `mode` argument to [`merge_subtrees_root`] and friends
+///
+/// See the [module level examples](index.html#examples).
+#[derive(Copy, Clone, Debug)]
+pub enum Mode<'a> {
+    /// Corresponding to [`hash`](crate::hash)
+    Hash,
+
+    /// Corresponding to [`keyed_hash`](crate::hash)
+    KeyedHash(&'a [u8; KEY_LEN]),
+
+    /// Corresponding to [`derive_key`](crate::hash)
+    ///
+    /// The [`ContextKey`] comes from [`hash_derive_key_context`].
+    DeriveKeyMaterial(&'a ContextKey),
+}
+
+impl<'a> Mode<'a> {
+    fn key_words(&self) -> CVWords {
+        match self {
+            Mode::Hash => *IV,
+            Mode::KeyedHash(key) => crate::platform::words_from_le_bytes_32(key),
+            Mode::DeriveKeyMaterial(cx_key) => crate::platform::words_from_le_bytes_32(cx_key),
+        }
+    }
+
+    fn flags_byte(&self) -> u8 {
+        match self {
+            Mode::Hash => 0,
+            Mode::KeyedHash(_) => crate::KEYED_HASH,
+            Mode::DeriveKeyMaterial(_) => crate::DERIVE_KEY_MATERIAL,
+        }
+    }
+}
+
+/// "Chaining value" is the academic term for a non-root or non-final hash.
+///
+/// Besides just sounding fancy, it turns out there are [security
+/// reasons](https://jacko.io/tree_hashing.html) to be careful about the difference between
+/// (root/final) hashes and (non-root/non-final) chaining values.
+pub type ChainingValue = [u8; OUT_LEN];
+
+fn merge_subtrees_inner(
+    left_child: &ChainingValue,
+    right_child: &ChainingValue,
+    mode: Mode,
+) -> crate::Output {
+    crate::parent_node_output(
+        &left_child,
+        &right_child,
+        &mode.key_words(),
+        mode.flags_byte(),
+        Platform::detect(),
+    )
+}
+
+/// Compute a non-root parent node chaining value from two child chaining values.
+///
+/// See the [module level examples](index.html#examples), particularly the discussion of valid tree
+/// structures. The left and right child chaining values can come from either
+/// [`Hasher::finalize_non_root`](HasherExt::finalize_non_root) or other calls to
+/// `merge_subtrees_non_root`. "Chaining value" is the academic term for a non-root or non-final
+/// hash.
+pub fn merge_subtrees_non_root(
+    left_child: &ChainingValue,
+    right_child: &ChainingValue,
+    mode: Mode,
+) -> ChainingValue {
+    merge_subtrees_inner(left_child, right_child, mode).chaining_value()
+}
+
+/// Compute a root hash from two child chaining values.
+///
+/// See the [module level examples](index.html#examples), particularly the discussion of valid tree
+/// structures. The left and right child chaining values can come from either
+/// [`Hasher::finalize_non_root`](HasherExt::finalize_non_root) or [`merge_subtrees_non_root`].
+/// "Chaining value" is the academic term for a non-root or non-final hash.
+///
+/// Note that inputs of [`CHUNK_LEN`] or less don't produce any parent nodes and can't be hashed
+/// using this function. In that case you must get the root hash from [`Hasher::finalize`] (or just
+/// [`blake3::hash`](crate::hash)).
+pub fn merge_subtrees_root(
+    left_child: &ChainingValue,
+    right_child: &ChainingValue,
+    mode: Mode,
+) -> crate::Hash {
+    merge_subtrees_inner(left_child, right_child, mode).root_hash()
+}
+
+/// Build a root [`OutputReader`](crate::OutputReader) from two child chaining values.
+///
+/// See also the [module level examples](index.html#examples), particularly the discussion of valid
+/// tree structures. The left and right child chaining values can come from either
+/// [`Hasher::finalize_non_root`](HasherExt::finalize_non_root) or [`merge_subtrees_non_root`].
+/// "Chaining value" is the academic term for a non-root or non-final hash.
+///
+/// Note that inputs of [`CHUNK_LEN`] or less don't produce any parent nodes and can't be hashed
+/// using this function. In that case you must get the `OutputReader` from
+/// [`Hasher::finalize_xof`].
+///
+/// # Example
+///
+/// ```
+/// use blake3::hazmat::{merge_subtrees_root_xof, HasherExt, Mode};
+/// use blake3::{Hasher, CHUNK_LEN};
+///
+/// // Hash a 2-chunk subtree in steps. Note that only
+/// // the final chunk can be shorter than CHUNK_LEN.
+/// let chunk0 = &[42; CHUNK_LEN];
+/// let chunk1 = b"hello world";
+/// let chunk0_cv = Hasher::new()
+///     .update(chunk0)
+///     .finalize_non_root();
+/// let chunk1_cv = Hasher::new()
+///     .set_input_offset(CHUNK_LEN as u64)
+///     .update(chunk1)
+///     .finalize_non_root();
+///
+/// // Obtain a blake3::OutputReader at the root and extract 1000 bytes.
+/// let mut output_reader = merge_subtrees_root_xof(&chunk0_cv, &chunk1_cv, Mode::Hash);
+/// let mut output_bytes = [0; 1_000];
+/// output_reader.fill(&mut output_bytes);
+///
+/// // Double check the answer.
+/// let mut hasher = Hasher::new();
+/// hasher.update(chunk0);
+/// hasher.update(chunk1);
+/// let mut expected = [0; 1_000];
+/// hasher.finalize_xof().fill(&mut expected);
+/// assert_eq!(output_bytes, expected);
+/// ```
+pub fn merge_subtrees_root_xof(
+    left_child: &ChainingValue,
+    right_child: &ChainingValue,
+    mode: Mode,
+) -> crate::OutputReader {
+    crate::OutputReader::new(merge_subtrees_inner(left_child, right_child, mode))
+}
+
+/// An alias to distinguish [`hash_derive_key_context`] outputs from other keys.
+pub type ContextKey = [u8; KEY_LEN];
+
+/// Hash a [`derive_key`](crate::derive_key) context string and return a [`ContextKey`].
+///
+/// The _only_ valid uses for the returned [`ContextKey`] are [`Hasher::new_from_context_key`] and
+/// [`Mode::DeriveKeyMaterial`] (together with the merge subtree functions).
+///
+/// # Example
+///
+/// ```
+/// use blake3::Hasher;
+/// use blake3::hazmat::HasherExt;
+///
+/// let context_key = blake3::hazmat::hash_derive_key_context("foo");
+/// let mut hasher = Hasher::new_from_context_key(&context_key);
+/// hasher.update(b"bar");
+/// let derived_key = *hasher.finalize().as_bytes();
+///
+/// assert_eq!(derived_key, blake3::derive_key("foo", b"bar"));
+/// ```
+pub fn hash_derive_key_context(context: &str) -> ContextKey {
+    crate::hash_all_at_once::<crate::join::SerialJoin>(
+        context.as_bytes(),
+        IV,
+        crate::DERIVE_KEY_CONTEXT,
+    )
+    .root_hash()
+    .0
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    #[should_panic]
+    fn test_empty_subtree_should_panic() {
+        Hasher::new().finalize_non_root();
+    }
+
+    #[test]
+    #[should_panic]
+    fn test_unaligned_offset_should_panic() {
+        Hasher::new().set_input_offset(1);
+    }
+
+    #[test]
+    #[should_panic]
+    fn test_hasher_already_accepted_input_should_panic() {
+        Hasher::new().update(b"x").set_input_offset(0);
+    }
+
+    #[test]
+    #[should_panic]
+    fn test_too_much_input_should_panic() {
+        Hasher::new()
+            .set_input_offset(CHUNK_LEN as u64)
+            .update(&[0; CHUNK_LEN + 1]);
+    }
+
+    #[test]
+    #[should_panic]
+    fn test_set_input_offset_cant_finalize() {
+        Hasher::new().set_input_offset(CHUNK_LEN as u64).finalize();
+    }
+
+    #[test]
+    #[should_panic]
+    fn test_set_input_offset_cant_finalize_xof() {
+        Hasher::new()
+            .set_input_offset(CHUNK_LEN as u64)
+            .finalize_xof();
+    }
+
+    #[test]
+    fn test_grouped_hash() {
+        const MAX_CHUNKS: usize = (crate::test::TEST_CASES_MAX + 1) / CHUNK_LEN;
+        let mut input_buf = [0; crate::test::TEST_CASES_MAX];
+        crate::test::paint_test_input(&mut input_buf);
+        for subtree_chunks in [1, 2, 4, 8, 16, 32] {
+            #[cfg(feature = "std")]
+            dbg!(subtree_chunks);
+            let subtree_len = subtree_chunks * CHUNK_LEN;
+            for &case in crate::test::TEST_CASES {
+                if case <= subtree_len {
+                    continue;
+                }
+                #[cfg(feature = "std")]
+                dbg!(case);
+                let input = &input_buf[..case];
+                let expected_hash = crate::hash(input);
+
+                // Collect all the group chaining values.
+                let mut chaining_values = arrayvec::ArrayVec::<ChainingValue, MAX_CHUNKS>::new();
+                let mut subtree_offset = 0;
+                while subtree_offset < input.len() {
+                    let take = core::cmp::min(subtree_len, input.len() - subtree_offset);
+                    let subtree_input = &input[subtree_offset..][..take];
+                    let subtree_cv = Hasher::new()
+                        .set_input_offset(subtree_offset as u64)
+                        .update(subtree_input)
+                        .finalize_non_root();
+                    chaining_values.push(subtree_cv);
+                    subtree_offset += take;
+                }
+
+                // Compress all the chaining_values together, layer by layer.
+                assert!(chaining_values.len() >= 2);
+                while chaining_values.len() > 2 {
+                    let n = chaining_values.len();
+                    // Merge each side-by-side pair in place, overwriting the front half of the
+                    // array with the merged results. This moves us "up one level" in the tree.
+                    for i in 0..(n / 2) {
+                        chaining_values[i] = merge_subtrees_non_root(
+                            &chaining_values[2 * i],
+                            &chaining_values[2 * i + 1],
+                            Mode::Hash,
+                        );
+                    }
+                    // If there's an odd CV out, it moves up.
+                    if n % 2 == 1 {
+                        chaining_values[n / 2] = chaining_values[n - 1];
+                    }
+                    chaining_values.truncate(n / 2 + n % 2);
+                }
+                assert_eq!(chaining_values.len(), 2);
+                let root_hash =
+                    merge_subtrees_root(&chaining_values[0], &chaining_values[1], Mode::Hash);
+                assert_eq!(expected_hash, root_hash);
+            }
+        }
+    }
+
+    #[test]
+    fn test_keyed_hash_xof() {
+        let group0 = &[42; 4096];
+        let group1 = &[43; 4095];
+        let mut input = [0; 8191];
+        input[..4096].copy_from_slice(group0);
+        input[4096..].copy_from_slice(group1);
+        let key = &[44; 32];
+
+        let mut expected_output = [0; 100];
+        Hasher::new_keyed(&key)
+            .update(&input)
+            .finalize_xof()
+            .fill(&mut expected_output);
+
+        let mut hazmat_output = [0; 100];
+        let left = Hasher::new_keyed(key).update(group0).finalize_non_root();
+        let right = Hasher::new_keyed(key)
+            .set_input_offset(group0.len() as u64)
+            .update(group1)
+            .finalize_non_root();
+        merge_subtrees_root_xof(&left, &right, Mode::KeyedHash(&key)).fill(&mut hazmat_output);
+        assert_eq!(expected_output, hazmat_output);
+    }
+
+    #[test]
+    fn test_derive_key() {
+        let context = "foo";
+        let mut input = [0; 1025];
+        crate::test::paint_test_input(&mut input);
+        let expected = crate::derive_key(context, &input);
+
+        let cx_key = hash_derive_key_context(context);
+        let left = Hasher::new_from_context_key(&cx_key)
+            .update(&input[..1024])
+            .finalize_non_root();
+        let right = Hasher::new_from_context_key(&cx_key)
+            .set_input_offset(1024)
+            .update(&input[1024..])
+            .finalize_non_root();
+        let derived_key = merge_subtrees_root(&left, &right, Mode::DeriveKeyMaterial(&cx_key)).0;
+        assert_eq!(expected, derived_key);
+    }
+}

vendor/blake3/src/io.rs

@@ -0,0 +1,64 @@
+//! Helper functions for efficient IO.
+
+#[cfg(feature = "std")]
+pub(crate) fn copy_wide(
+    mut reader: impl std::io::Read,
+    hasher: &mut crate::Hasher,
+) -> std::io::Result<u64> {
+    let mut buffer = [0; 65536];
+    let mut total = 0;
+    loop {
+        match reader.read(&mut buffer) {
+            Ok(0) => return Ok(total),
+            Ok(n) => {
+                hasher.update(&buffer[..n]);
+                total += n as u64;
+            }
+            // see test_update_reader_interrupted
+            Err(e) if e.kind() == std::io::ErrorKind::Interrupted => continue,
+            Err(e) => return Err(e),
+        }
+    }
+}
+
+// Mmap a file, if it looks like a good idea. Return None in cases where we know mmap will fail, or
+// if the file is short enough that mmapping isn't worth it. However, if we do try to mmap and it
+// fails, return the error.
+//
+// SAFETY: Mmaps are fundamentally unsafe, because you can call invariant-checking functions like
+// str::from_utf8 on them and then have them change out from under you. Letting a safe caller get
+// their hands on an mmap, or even a &[u8] that's backed by an mmap, is unsound. However, because
+// this function is crate-private, we can guarantee that all can ever happen in the event of a race
+// condition is that we either hash nonsense bytes or crash with SIGBUS or similar, neither of
+// which should risk memory corruption in a safe caller.
+//
+// PARANOIA: But a data race...is a data race...is a data race...right? Even if we know that no
+// platform in the "real world" is ever going to do anything other than compute the "wrong answer"
+// if we race on this mmap while we hash it, aren't we still supposed to feel bad about doing this?
+// Well, maybe. This is IO, and IO gets special carve-outs in the memory model. Consider a
+// memory-mapped register that returns random 32-bit words. (This is actually realistic if you have
+// a hardware RNG.) It's probably sound to construct a *const i32 pointing to that register and do
+// some raw pointer reads from it. Those reads should be volatile if you don't want the compiler to
+// coalesce them, but either way the compiler isn't allowed to just _go nuts_ and insert
+// should-never-happen branches to wipe your hard drive if two adjacent reads happen to give
+// different values. As far as I'm aware, there's no such thing as a read that's allowed if it's
+// volatile but prohibited if it's not (unlike atomics). As mentioned above, it's not ok to
+// construct a safe &i32 to the register if you're going to leak that reference to unknown callers.
+// But if you "know what you're doing," I don't think *const i32 and &i32 are fundamentally
+// different here. Feedback needed.
+#[cfg(feature = "mmap")]
+pub(crate) fn maybe_mmap_file(file: &std::fs::File) -> std::io::Result<Option<memmap2::Mmap>> {
+    let metadata = file.metadata()?;
+    let file_size = metadata.len();
+    if !metadata.is_file() {
+        // Not a real file.
+        Ok(None)
+    } else if file_size < 16 * 1024 {
+        // Mapping small files is not worth it, and some special files that can't be mapped report
+        // a size of zero.
+        Ok(None)
+    } else {
+        let map = unsafe { memmap2::Mmap::map(file)? };
+        Ok(Some(map))
+    }
+}

vendor/blake3/src/join.rs

@@ -0,0 +1,92 @@
+//! The multi-threading abstractions used by `Hasher::update_with_join`.
+//!
+//! Different implementations of the `Join` trait determine whether
+//! `Hasher::update_with_join` performs multi-threading on sufficiently large
+//! inputs. The `SerialJoin` implementation is single-threaded, and the
+//! `RayonJoin` implementation (gated by the `rayon` feature) is multi-threaded.
+//! Interfaces other than `Hasher::update_with_join`, like [`hash`](crate::hash)
+//! and [`Hasher::update`](crate::Hasher::update), always use `SerialJoin`
+//! internally.
+//!
+//! The `Join` trait is an almost exact copy of the [`rayon::join`] API, and
+//! `RayonJoin` is the only non-trivial implementation. Previously this trait
+//! was public, but currently it's been re-privatized, as it's both 1) of no
+//! value to most callers and 2) a pretty big implementation detail to commit
+//! to.
+//!
+//! [`rayon::join`]: https://docs.rs/rayon/1.3.0/rayon/fn.join.html
+
+/// The trait that abstracts over single-threaded and multi-threaded recursion.
+///
+/// See the [`join` module docs](index.html) for more details.
+pub trait Join {
+    fn join<A, B, RA, RB>(oper_a: A, oper_b: B) -> (RA, RB)
+    where
+        A: FnOnce() -> RA + Send,
+        B: FnOnce() -> RB + Send,
+        RA: Send,
+        RB: Send;
+}
+
+/// The trivial, serial implementation of `Join`. The left and right sides are
+/// executed one after the other, on the calling thread. The standalone hashing
+/// functions and the `Hasher::update` method use this implementation
+/// internally.
+///
+/// See the [`join` module docs](index.html) for more details.
+pub enum SerialJoin {}
+
+impl Join for SerialJoin {
+    #[inline]
+    fn join<A, B, RA, RB>(oper_a: A, oper_b: B) -> (RA, RB)
+    where
+        A: FnOnce() -> RA + Send,
+        B: FnOnce() -> RB + Send,
+        RA: Send,
+        RB: Send,
+    {
+        (oper_a(), oper_b())
+    }
+}
+
+/// The Rayon-based implementation of `Join`. The left and right sides are
+/// executed on the Rayon thread pool, potentially in parallel. This
+/// implementation is gated by the `rayon` feature, which is off by default.
+///
+/// See the [`join` module docs](index.html) for more details.
+#[cfg(feature = "rayon")]
+pub enum RayonJoin {}
+
+#[cfg(feature = "rayon")]
+impl Join for RayonJoin {
+    #[inline]
+    fn join<A, B, RA, RB>(oper_a: A, oper_b: B) -> (RA, RB)
+    where
+        A: FnOnce() -> RA + Send,
+        B: FnOnce() -> RB + Send,
+        RA: Send,
+        RB: Send,
+    {
+        rayon_core::join(oper_a, oper_b)
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn test_serial_join() {
+        let oper_a = || 1 + 1;
+        let oper_b = || 2 + 2;
+        assert_eq!((2, 4), SerialJoin::join(oper_a, oper_b));
+    }
+
+    #[test]
+    #[cfg(feature = "rayon")]
+    fn test_rayon_join() {
+        let oper_a = || 1 + 1;
+        let oper_b = || 2 + 2;
+        assert_eq!((2, 4), RayonJoin::join(oper_a, oper_b));
+    }
+}

vendor/blake3/src/platform.rs

@@ -0,0 +1,587 @@
+use crate::{portable, CVWords, IncrementCounter, BLOCK_LEN};
+use arrayref::{array_mut_ref, array_ref};
+
+cfg_if::cfg_if! {
+    if #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] {
+        cfg_if::cfg_if! {
+            if #[cfg(blake3_avx512_ffi)] {
+                pub const MAX_SIMD_DEGREE: usize = 16;
+            } else {
+                pub const MAX_SIMD_DEGREE: usize = 8;
+            }
+        }
+    } else if #[cfg(blake3_neon)] {
+        pub const MAX_SIMD_DEGREE: usize = 4;
+    } else if #[cfg(blake3_wasm32_simd)] {
+        pub const MAX_SIMD_DEGREE: usize = 4;
+    } else {
+        pub const MAX_SIMD_DEGREE: usize = 1;
+    }
+}
+
+// There are some places where we want a static size that's equal to the
+// MAX_SIMD_DEGREE, but also at least 2. Constant contexts aren't currently
+// allowed to use cmp::max, so we have to hardcode this additional constant
+// value. Get rid of this once cmp::max is a const fn.
+cfg_if::cfg_if! {
+    if #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] {
+        cfg_if::cfg_if! {
+            if #[cfg(blake3_avx512_ffi)] {
+                pub const MAX_SIMD_DEGREE_OR_2: usize = 16;
+            } else {
+                pub const MAX_SIMD_DEGREE_OR_2: usize = 8;
+            }
+        }
+    } else if #[cfg(blake3_neon)] {
+        pub const MAX_SIMD_DEGREE_OR_2: usize = 4;
+    } else if #[cfg(blake3_wasm32_simd)] {
+        pub const MAX_SIMD_DEGREE_OR_2: usize = 4;
+    } else {
+        pub const MAX_SIMD_DEGREE_OR_2: usize = 2;
+    }
+}
+
+#[derive(Clone, Copy, Debug)]
+pub enum Platform {
+    Portable,
+    #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+    SSE2,
+    #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+    SSE41,
+    #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+    AVX2,
+    #[cfg(blake3_avx512_ffi)]
+    #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+    AVX512,
+    #[cfg(blake3_neon)]
+    NEON,
+    #[cfg(blake3_wasm32_simd)]
+    #[allow(non_camel_case_types)]
+    WASM32_SIMD,
+}
+
+impl Platform {
+    #[allow(unreachable_code)]
+    pub fn detect() -> Self {
+        #[cfg(miri)]
+        {
+            return Platform::Portable;
+        }
+
+        #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+        {
+            #[cfg(blake3_avx512_ffi)]
+            {
+                if avx512_detected() {
+                    return Platform::AVX512;
+                }
+            }
+            if avx2_detected() {
+                return Platform::AVX2;
+            }
+            if sse41_detected() {
+                return Platform::SSE41;
+            }
+            if sse2_detected() {
+                return Platform::SSE2;
+            }
+        }
+        // We don't use dynamic feature detection for NEON. If the "neon"
+        // feature is on, NEON is assumed to be supported.
+        #[cfg(blake3_neon)]
+        {
+            return Platform::NEON;
+        }
+        #[cfg(blake3_wasm32_simd)]
+        {
+            return Platform::WASM32_SIMD;
+        }
+        Platform::Portable
+    }
+
+    pub fn simd_degree(&self) -> usize {
+        let degree = match self {
+            Platform::Portable => 1,
+            #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+            Platform::SSE2 => 4,
+            #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+            Platform::SSE41 => 4,
+            #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+            Platform::AVX2 => 8,
+            #[cfg(blake3_avx512_ffi)]
+            #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+            Platform::AVX512 => 16,
+            #[cfg(blake3_neon)]
+            Platform::NEON => 4,
+            #[cfg(blake3_wasm32_simd)]
+            Platform::WASM32_SIMD => 4,
+        };
+        debug_assert!(degree <= MAX_SIMD_DEGREE);
+        degree
+    }
+
+    pub fn compress_in_place(
+        &self,
+        cv: &mut CVWords,
+        block: &[u8; BLOCK_LEN],
+        block_len: u8,
+        counter: u64,
+        flags: u8,
+    ) {
+        match self {
+            Platform::Portable => portable::compress_in_place(cv, block, block_len, counter, flags),
+            // Safe because detect() checked for platform support.
+            #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+            Platform::SSE2 => unsafe {
+                crate::sse2::compress_in_place(cv, block, block_len, counter, flags)
+            },
+            // Safe because detect() checked for platform support.
+            #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+            Platform::SSE41 | Platform::AVX2 => unsafe {
+                crate::sse41::compress_in_place(cv, block, block_len, counter, flags)
+            },
+            // Safe because detect() checked for platform support.
+            #[cfg(blake3_avx512_ffi)]
+            #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+            Platform::AVX512 => unsafe {
+                crate::avx512::compress_in_place(cv, block, block_len, counter, flags)
+            },
+            // No NEON compress_in_place() implementation yet.
+            #[cfg(blake3_neon)]
+            Platform::NEON => portable::compress_in_place(cv, block, block_len, counter, flags),
+            #[cfg(blake3_wasm32_simd)]
+            Platform::WASM32_SIMD => {
+                crate::wasm32_simd::compress_in_place(cv, block, block_len, counter, flags)
+            }
+        }
+    }
+
+    pub fn compress_xof(
+        &self,
+        cv: &CVWords,
+        block: &[u8; BLOCK_LEN],
+        block_len: u8,
+        counter: u64,
+        flags: u8,
+    ) -> [u8; 64] {
+        match self {
+            Platform::Portable => portable::compress_xof(cv, block, block_len, counter, flags),
+            // Safe because detect() checked for platform support.
+            #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+            Platform::SSE2 => unsafe {
+                crate::sse2::compress_xof(cv, block, block_len, counter, flags)
+            },
+            // Safe because detect() checked for platform support.
+            #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+            Platform::SSE41 | Platform::AVX2 => unsafe {
+                crate::sse41::compress_xof(cv, block, block_len, counter, flags)
+            },
+            // Safe because detect() checked for platform support.
+            #[cfg(blake3_avx512_ffi)]
+            #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+            Platform::AVX512 => unsafe {
+                crate::avx512::compress_xof(cv, block, block_len, counter, flags)
+            },
+            // No NEON compress_xof() implementation yet.
+            #[cfg(blake3_neon)]
+            Platform::NEON => portable::compress_xof(cv, block, block_len, counter, flags),
+            #[cfg(blake3_wasm32_simd)]
+            Platform::WASM32_SIMD => {
+                crate::wasm32_simd::compress_xof(cv, block, block_len, counter, flags)
+            }
+        }
+    }
+
+    // IMPLEMENTATION NOTE
+    // ===================
+    // hash_many() applies two optimizations. The critically important
+    // optimization is the high-performance parallel SIMD hashing mode,
+    // described in detail in the spec. This more than doubles throughput per
+    // thread. Another optimization is keeping the state vectors transposed
+    // from block to block within a chunk. When state vectors are transposed
+    // after every block, there's a small but measurable performance loss.
+    // Compressing chunks with a dedicated loop avoids this.
+
+    pub fn hash_many<const N: usize>(
+        &self,
+        inputs: &[&[u8; N]],
+        key: &CVWords,
+        counter: u64,
+        increment_counter: IncrementCounter,
+        flags: u8,
+        flags_start: u8,
+        flags_end: u8,
+        out: &mut [u8],
+    ) {
+        match self {
+            Platform::Portable => portable::hash_many(
+                inputs,
+                key,
+                counter,
+                increment_counter,
+                flags,
+                flags_start,
+                flags_end,
+                out,
+            ),
+            // Safe because detect() checked for platform support.
+            #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+            Platform::SSE2 => unsafe {
+                crate::sse2::hash_many(
+                    inputs,
+                    key,
+                    counter,
+                    increment_counter,
+                    flags,
+                    flags_start,
+                    flags_end,
+                    out,
+                )
+            },
+            // Safe because detect() checked for platform support.
+            #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+            Platform::SSE41 => unsafe {
+                crate::sse41::hash_many(
+                    inputs,
+                    key,
+                    counter,
+                    increment_counter,
+                    flags,
+                    flags_start,
+                    flags_end,
+                    out,
+                )
+            },
+            // Safe because detect() checked for platform support.
+            #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+            Platform::AVX2 => unsafe {
+                crate::avx2::hash_many(
+                    inputs,
+                    key,
+                    counter,
+                    increment_counter,
+                    flags,
+                    flags_start,
+                    flags_end,
+                    out,
+                )
+            },
+            // Safe because detect() checked for platform support.
+            #[cfg(blake3_avx512_ffi)]
+            #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+            Platform::AVX512 => unsafe {
+                crate::avx512::hash_many(
+                    inputs,
+                    key,
+                    counter,
+                    increment_counter,
+                    flags,
+                    flags_start,
+                    flags_end,
+                    out,
+                )
+            },
+            // Assumed to be safe if the "neon" feature is on.
+            #[cfg(blake3_neon)]
+            Platform::NEON => unsafe {
+                crate::neon::hash_many(
+                    inputs,
+                    key,
+                    counter,
+                    increment_counter,
+                    flags,
+                    flags_start,
+                    flags_end,
+                    out,
+                )
+            },
+            // Assumed to be safe if the "wasm32_simd" feature is on.
+            #[cfg(blake3_wasm32_simd)]
+            Platform::WASM32_SIMD => unsafe {
+                crate::wasm32_simd::hash_many(
+                    inputs,
+                    key,
+                    counter,
+                    increment_counter,
+                    flags,
+                    flags_start,
+                    flags_end,
+                    out,
+                )
+            },
+        }
+    }
+
+    pub fn xof_many(
+        &self,
+        cv: &CVWords,
+        block: &[u8; BLOCK_LEN],
+        block_len: u8,
+        mut counter: u64,
+        flags: u8,
+        out: &mut [u8],
+    ) {
+        debug_assert_eq!(0, out.len() % BLOCK_LEN, "whole blocks only");
+        if out.is_empty() {
+            // The current assembly implementation always outputs at least 1 block.
+            return;
+        }
+        match self {
+            // Safe because detect() checked for platform support.
+            #[cfg(blake3_avx512_ffi)]
+            #[cfg(unix)]
+            #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+            Platform::AVX512 => unsafe {
+                crate::avx512::xof_many(cv, block, block_len, counter, flags, out)
+            },
+            _ => {
+                // For platforms without an optimized xof_many, fall back to a loop over
+                // compress_xof. This is still faster than portable code.
+                for out_block in out.chunks_exact_mut(BLOCK_LEN) {
+                    // TODO: Use array_chunks_mut here once that's stable.
+                    let out_array: &mut [u8; BLOCK_LEN] = out_block.try_into().unwrap();
+                    *out_array = self.compress_xof(cv, block, block_len, counter, flags);
+                    counter += 1;
+                }
+            }
+        }
+    }
+
+    // Explicit platform constructors, for benchmarks.
+
+    pub fn portable() -> Self {
+        Self::Portable
+    }
+
+    #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+    pub fn sse2() -> Option<Self> {
+        if sse2_detected() {
+            Some(Self::SSE2)
+        } else {
+            None
+        }
+    }
+
+    #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+    pub fn sse41() -> Option<Self> {
+        if sse41_detected() {
+            Some(Self::SSE41)
+        } else {
+            None
+        }
+    }
+
+    #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+    pub fn avx2() -> Option<Self> {
+        if avx2_detected() {
+            Some(Self::AVX2)
+        } else {
+            None
+        }
+    }
+
+    #[cfg(blake3_avx512_ffi)]
+    #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+    pub fn avx512() -> Option<Self> {
+        if avx512_detected() {
+            Some(Self::AVX512)
+        } else {
+            None
+        }
+    }
+
+    #[cfg(blake3_neon)]
+    pub fn neon() -> Option<Self> {
+        // Assumed to be safe if the "neon" feature is on.
+        Some(Self::NEON)
+    }
+
+    #[cfg(blake3_wasm32_simd)]
+    pub fn wasm32_simd() -> Option<Self> {
+        // Assumed to be safe if the "wasm32_simd" feature is on.
+        Some(Self::WASM32_SIMD)
+    }
+}
+
+// Note that AVX-512 is divided into multiple featuresets, and we use two of
+// them, F and VL.
+#[cfg(blake3_avx512_ffi)]
+#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+#[inline(always)]
+#[allow(unreachable_code)]
+pub fn avx512_detected() -> bool {
+    if cfg!(miri) {
+        return false;
+    }
+
+    // A testing-only short-circuit.
+    if cfg!(feature = "no_avx512") {
+        return false;
+    }
+    // Static check, e.g. for building with target-cpu=native.
+    #[cfg(all(target_feature = "avx512f", target_feature = "avx512vl"))]
+    {
+        return true;
+    }
+    // Dynamic check, if std is enabled.
+    #[cfg(feature = "std")]
+    {
+        if is_x86_feature_detected!("avx512f") && is_x86_feature_detected!("avx512vl") {
+            return true;
+        }
+    }
+    false
+}
+
+#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+#[inline(always)]
+#[allow(unreachable_code)]
+pub fn avx2_detected() -> bool {
+    if cfg!(miri) {
+        return false;
+    }
+
+    // A testing-only short-circuit.
+    if cfg!(feature = "no_avx2") {
+        return false;
+    }
+    // Static check, e.g. for building with target-cpu=native.
+    #[cfg(target_feature = "avx2")]
+    {
+        return true;
+    }
+    // Dynamic check, if std is enabled.
+    #[cfg(feature = "std")]
+    {
+        if is_x86_feature_detected!("avx2") {
+            return true;
+        }
+    }
+    false
+}
+
+#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+#[inline(always)]
+#[allow(unreachable_code)]
+pub fn sse41_detected() -> bool {
+    if cfg!(miri) {
+        return false;
+    }
+
+    // A testing-only short-circuit.
+    if cfg!(feature = "no_sse41") {
+        return false;
+    }
+    // Static check, e.g. for building with target-cpu=native.
+    #[cfg(target_feature = "sse4.1")]
+    {
+        return true;
+    }
+    // Dynamic check, if std is enabled.
+    #[cfg(feature = "std")]
+    {
+        if is_x86_feature_detected!("sse4.1") {
+            return true;
+        }
+    }
+    false
+}
+
+#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+#[inline(always)]
+#[allow(unreachable_code)]
+pub fn sse2_detected() -> bool {
+    if cfg!(miri) {
+        return false;
+    }
+
+    // A testing-only short-circuit.
+    if cfg!(feature = "no_sse2") {
+        return false;
+    }
+    // Static check, e.g. for building with target-cpu=native.
+    #[cfg(target_feature = "sse2")]
+    {
+        return true;
+    }
+    // Dynamic check, if std is enabled.
+    #[cfg(feature = "std")]
+    {
+        if is_x86_feature_detected!("sse2") {
+            return true;
+        }
+    }
+    false
+}
+
+#[inline(always)]
+pub fn words_from_le_bytes_32(bytes: &[u8; 32]) -> [u32; 8] {
+    let mut out = [0; 8];
+    out[0] = u32::from_le_bytes(*array_ref!(bytes, 0 * 4, 4));
+    out[1] = u32::from_le_bytes(*array_ref!(bytes, 1 * 4, 4));
+    out[2] = u32::from_le_bytes(*array_ref!(bytes, 2 * 4, 4));
+    out[3] = u32::from_le_bytes(*array_ref!(bytes, 3 * 4, 4));
+    out[4] = u32::from_le_bytes(*array_ref!(bytes, 4 * 4, 4));
+    out[5] = u32::from_le_bytes(*array_ref!(bytes, 5 * 4, 4));
+    out[6] = u32::from_le_bytes(*array_ref!(bytes, 6 * 4, 4));
+    out[7] = u32::from_le_bytes(*array_ref!(bytes, 7 * 4, 4));
+    out
+}
+
+#[inline(always)]
+pub fn words_from_le_bytes_64(bytes: &[u8; 64]) -> [u32; 16] {
+    let mut out = [0; 16];
+    out[0] = u32::from_le_bytes(*array_ref!(bytes, 0 * 4, 4));
+    out[1] = u32::from_le_bytes(*array_ref!(bytes, 1 * 4, 4));
+    out[2] = u32::from_le_bytes(*array_ref!(bytes, 2 * 4, 4));
+    out[3] = u32::from_le_bytes(*array_ref!(bytes, 3 * 4, 4));
+    out[4] = u32::from_le_bytes(*array_ref!(bytes, 4 * 4, 4));
+    out[5] = u32::from_le_bytes(*array_ref!(bytes, 5 * 4, 4));
+    out[6] = u32::from_le_bytes(*array_ref!(bytes, 6 * 4, 4));
+    out[7] = u32::from_le_bytes(*array_ref!(bytes, 7 * 4, 4));
+    out[8] = u32::from_le_bytes(*array_ref!(bytes, 8 * 4, 4));
+    out[9] = u32::from_le_bytes(*array_ref!(bytes, 9 * 4, 4));
+    out[10] = u32::from_le_bytes(*array_ref!(bytes, 10 * 4, 4));
+    out[11] = u32::from_le_bytes(*array_ref!(bytes, 11 * 4, 4));
+    out[12] = u32::from_le_bytes(*array_ref!(bytes, 12 * 4, 4));
+    out[13] = u32::from_le_bytes(*array_ref!(bytes, 13 * 4, 4));
+    out[14] = u32::from_le_bytes(*array_ref!(bytes, 14 * 4, 4));
+    out[15] = u32::from_le_bytes(*array_ref!(bytes, 15 * 4, 4));
+    out
+}
+
+#[inline(always)]
+pub fn le_bytes_from_words_32(words: &[u32; 8]) -> [u8; 32] {
+    let mut out = [0; 32];
+    *array_mut_ref!(out, 0 * 4, 4) = words[0].to_le_bytes();
+    *array_mut_ref!(out, 1 * 4, 4) = words[1].to_le_bytes();
+    *array_mut_ref!(out, 2 * 4, 4) = words[2].to_le_bytes();
+    *array_mut_ref!(out, 3 * 4, 4) = words[3].to_le_bytes();
+    *array_mut_ref!(out, 4 * 4, 4) = words[4].to_le_bytes();
+    *array_mut_ref!(out, 5 * 4, 4) = words[5].to_le_bytes();
+    *array_mut_ref!(out, 6 * 4, 4) = words[6].to_le_bytes();
+    *array_mut_ref!(out, 7 * 4, 4) = words[7].to_le_bytes();
+    out
+}
+
+#[inline(always)]
+pub fn le_bytes_from_words_64(words: &[u32; 16]) -> [u8; 64] {
+    let mut out = [0; 64];
+    *array_mut_ref!(out, 0 * 4, 4) = words[0].to_le_bytes();
+    *array_mut_ref!(out, 1 * 4, 4) = words[1].to_le_bytes();
+    *array_mut_ref!(out, 2 * 4, 4) = words[2].to_le_bytes();
+    *array_mut_ref!(out, 3 * 4, 4) = words[3].to_le_bytes();
+    *array_mut_ref!(out, 4 * 4, 4) = words[4].to_le_bytes();
+    *array_mut_ref!(out, 5 * 4, 4) = words[5].to_le_bytes();
+    *array_mut_ref!(out, 6 * 4, 4) = words[6].to_le_bytes();
+    *array_mut_ref!(out, 7 * 4, 4) = words[7].to_le_bytes();
+    *array_mut_ref!(out, 8 * 4, 4) = words[8].to_le_bytes();
+    *array_mut_ref!(out, 9 * 4, 4) = words[9].to_le_bytes();
+    *array_mut_ref!(out, 10 * 4, 4) = words[10].to_le_bytes();
+    *array_mut_ref!(out, 11 * 4, 4) = words[11].to_le_bytes();
+    *array_mut_ref!(out, 12 * 4, 4) = words[12].to_le_bytes();
+    *array_mut_ref!(out, 13 * 4, 4) = words[13].to_le_bytes();
+    *array_mut_ref!(out, 14 * 4, 4) = words[14].to_le_bytes();
+    *array_mut_ref!(out, 15 * 4, 4) = words[15].to_le_bytes();
+    out
+}

vendor/blake3/src/portable.rs

@@ -0,0 +1,198 @@
+use crate::{
+    counter_high, counter_low, CVBytes, CVWords, IncrementCounter, BLOCK_LEN, IV, MSG_SCHEDULE,
+    OUT_LEN,
+};
+use arrayref::{array_mut_ref, array_ref};
+
+#[inline(always)]
+fn g(state: &mut [u32; 16], a: usize, b: usize, c: usize, d: usize, x: u32, y: u32) {
+    state[a] = state[a].wrapping_add(state[b]).wrapping_add(x);
+    state[d] = (state[d] ^ state[a]).rotate_right(16);
+    state[c] = state[c].wrapping_add(state[d]);
+    state[b] = (state[b] ^ state[c]).rotate_right(12);
+    state[a] = state[a].wrapping_add(state[b]).wrapping_add(y);
+    state[d] = (state[d] ^ state[a]).rotate_right(8);
+    state[c] = state[c].wrapping_add(state[d]);
+    state[b] = (state[b] ^ state[c]).rotate_right(7);
+}
+
+#[inline(always)]
+fn round(state: &mut [u32; 16], msg: &[u32; 16], round: usize) {
+    // Select the message schedule based on the round.
+    let schedule = MSG_SCHEDULE[round];
+
+    // Mix the columns.
+    g(state, 0, 4, 8, 12, msg[schedule[0]], msg[schedule[1]]);
+    g(state, 1, 5, 9, 13, msg[schedule[2]], msg[schedule[3]]);
+    g(state, 2, 6, 10, 14, msg[schedule[4]], msg[schedule[5]]);
+    g(state, 3, 7, 11, 15, msg[schedule[6]], msg[schedule[7]]);
+
+    // Mix the diagonals.
+    g(state, 0, 5, 10, 15, msg[schedule[8]], msg[schedule[9]]);
+    g(state, 1, 6, 11, 12, msg[schedule[10]], msg[schedule[11]]);
+    g(state, 2, 7, 8, 13, msg[schedule[12]], msg[schedule[13]]);
+    g(state, 3, 4, 9, 14, msg[schedule[14]], msg[schedule[15]]);
+}
+
+#[inline(always)]
+fn compress_pre(
+    cv: &CVWords,
+    block: &[u8; BLOCK_LEN],
+    block_len: u8,
+    counter: u64,
+    flags: u8,
+) -> [u32; 16] {
+    let block_words = crate::platform::words_from_le_bytes_64(block);
+
+    let mut state = [
+        cv[0],
+        cv[1],
+        cv[2],
+        cv[3],
+        cv[4],
+        cv[5],
+        cv[6],
+        cv[7],
+        IV[0],
+        IV[1],
+        IV[2],
+        IV[3],
+        counter_low(counter),
+        counter_high(counter),
+        block_len as u32,
+        flags as u32,
+    ];
+
+    round(&mut state, &block_words, 0);
+    round(&mut state, &block_words, 1);
+    round(&mut state, &block_words, 2);
+    round(&mut state, &block_words, 3);
+    round(&mut state, &block_words, 4);
+    round(&mut state, &block_words, 5);
+    round(&mut state, &block_words, 6);
+
+    state
+}
+
+pub fn compress_in_place(
+    cv: &mut CVWords,
+    block: &[u8; BLOCK_LEN],
+    block_len: u8,
+    counter: u64,
+    flags: u8,
+) {
+    let state = compress_pre(cv, block, block_len, counter, flags);
+
+    cv[0] = state[0] ^ state[8];
+    cv[1] = state[1] ^ state[9];
+    cv[2] = state[2] ^ state[10];
+    cv[3] = state[3] ^ state[11];
+    cv[4] = state[4] ^ state[12];
+    cv[5] = state[5] ^ state[13];
+    cv[6] = state[6] ^ state[14];
+    cv[7] = state[7] ^ state[15];
+}
+
+pub fn compress_xof(
+    cv: &CVWords,
+    block: &[u8; BLOCK_LEN],
+    block_len: u8,
+    counter: u64,
+    flags: u8,
+) -> [u8; 64] {
+    let mut state = compress_pre(cv, block, block_len, counter, flags);
+    state[0] ^= state[8];
+    state[1] ^= state[9];
+    state[2] ^= state[10];
+    state[3] ^= state[11];
+    state[4] ^= state[12];
+    state[5] ^= state[13];
+    state[6] ^= state[14];
+    state[7] ^= state[15];
+    state[8] ^= cv[0];
+    state[9] ^= cv[1];
+    state[10] ^= cv[2];
+    state[11] ^= cv[3];
+    state[12] ^= cv[4];
+    state[13] ^= cv[5];
+    state[14] ^= cv[6];
+    state[15] ^= cv[7];
+    crate::platform::le_bytes_from_words_64(&state)
+}
+
+pub fn hash1<const N: usize>(
+    input: &[u8; N],
+    key: &CVWords,
+    counter: u64,
+    flags: u8,
+    flags_start: u8,
+    flags_end: u8,
+    out: &mut CVBytes,
+) {
+    debug_assert_eq!(N % BLOCK_LEN, 0, "uneven blocks");
+    let mut cv = *key;
+    let mut block_flags = flags | flags_start;
+    let mut slice = &input[..];
+    while slice.len() >= BLOCK_LEN {
+        if slice.len() == BLOCK_LEN {
+            block_flags |= flags_end;
+        }
+        compress_in_place(
+            &mut cv,
+            array_ref!(slice, 0, BLOCK_LEN),
+            BLOCK_LEN as u8,
+            counter,
+            block_flags,
+        );
+        block_flags = flags;
+        slice = &slice[BLOCK_LEN..];
+    }
+    *out = crate::platform::le_bytes_from_words_32(&cv);
+}
+
+pub fn hash_many<const N: usize>(
+    inputs: &[&[u8; N]],
+    key: &CVWords,
+    mut counter: u64,
+    increment_counter: IncrementCounter,
+    flags: u8,
+    flags_start: u8,
+    flags_end: u8,
+    out: &mut [u8],
+) {
+    debug_assert!(out.len() >= inputs.len() * OUT_LEN, "out too short");
+    for (&input, output) in inputs.iter().zip(out.chunks_exact_mut(OUT_LEN)) {
+        hash1(
+            input,
+            key,
+            counter,
+            flags,
+            flags_start,
+            flags_end,
+            array_mut_ref!(output, 0, OUT_LEN),
+        );
+        if increment_counter.yes() {
+            counter += 1;
+        }
+    }
+}
+
+#[cfg(test)]
+pub mod test {
+    use super::*;
+
+    // This is basically testing the portable implementation against itself,
+    // but it also checks that compress_in_place and compress_xof are
+    // consistent. And there are tests against the reference implementation and
+    // against hardcoded test vectors elsewhere.
+    #[test]
+    fn test_compress() {
+        crate::test::test_compress_fn(compress_in_place, compress_xof);
+    }
+
+    // Ditto.
+    #[test]
+    fn test_hash_many() {
+        crate::test::test_hash_many_fn(hash_many, hash_many);
+    }
+}

vendor/blake3/src/rust_avx2.rs

@@ -0,0 +1,474 @@
+#[cfg(target_arch = "x86")]
+use core::arch::x86::*;
+#[cfg(target_arch = "x86_64")]
+use core::arch::x86_64::*;
+
+use crate::{
+    counter_high, counter_low, CVWords, IncrementCounter, BLOCK_LEN, IV, MSG_SCHEDULE, OUT_LEN,
+};
+use arrayref::{array_mut_ref, mut_array_refs};
+
+pub const DEGREE: usize = 8;
+
+#[inline(always)]
+unsafe fn loadu(src: *const u8) -> __m256i {
+    // This is an unaligned load, so the pointer cast is allowed.
+    _mm256_loadu_si256(src as *const __m256i)
+}
+
+#[inline(always)]
+unsafe fn storeu(src: __m256i, dest: *mut u8) {
+    // This is an unaligned store, so the pointer cast is allowed.
+    _mm256_storeu_si256(dest as *mut __m256i, src)
+}
+
+#[inline(always)]
+unsafe fn add(a: __m256i, b: __m256i) -> __m256i {
+    _mm256_add_epi32(a, b)
+}
+
+#[inline(always)]
+unsafe fn xor(a: __m256i, b: __m256i) -> __m256i {
+    _mm256_xor_si256(a, b)
+}
+
+#[inline(always)]
+unsafe fn set1(x: u32) -> __m256i {
+    _mm256_set1_epi32(x as i32)
+}
+
+#[inline(always)]
+unsafe fn set8(a: u32, b: u32, c: u32, d: u32, e: u32, f: u32, g: u32, h: u32) -> __m256i {
+    _mm256_setr_epi32(
+        a as i32, b as i32, c as i32, d as i32, e as i32, f as i32, g as i32, h as i32,
+    )
+}
+
+// These rotations are the "simple/shifts version". For the
+// "complicated/shuffles version", see
+// https://github.com/sneves/blake2-avx2/blob/b3723921f668df09ece52dcd225a36d4a4eea1d9/blake2s-common.h#L63-L66.
+// For a discussion of the tradeoffs, see
+// https://github.com/sneves/blake2-avx2/pull/5. Due to an LLVM bug
+// (https://bugs.llvm.org/show_bug.cgi?id=44379), this version performs better
+// on recent x86 chips.
+
+#[inline(always)]
+unsafe fn rot16(x: __m256i) -> __m256i {
+    _mm256_or_si256(_mm256_srli_epi32(x, 16), _mm256_slli_epi32(x, 32 - 16))
+}
+
+#[inline(always)]
+unsafe fn rot12(x: __m256i) -> __m256i {
+    _mm256_or_si256(_mm256_srli_epi32(x, 12), _mm256_slli_epi32(x, 32 - 12))
+}
+
+#[inline(always)]
+unsafe fn rot8(x: __m256i) -> __m256i {
+    _mm256_or_si256(_mm256_srli_epi32(x, 8), _mm256_slli_epi32(x, 32 - 8))
+}
+
+#[inline(always)]
+unsafe fn rot7(x: __m256i) -> __m256i {
+    _mm256_or_si256(_mm256_srli_epi32(x, 7), _mm256_slli_epi32(x, 32 - 7))
+}
+
+#[inline(always)]
+unsafe fn round(v: &mut [__m256i; 16], m: &[__m256i; 16], r: usize) {
+    v[0] = add(v[0], m[MSG_SCHEDULE[r][0] as usize]);
+    v[1] = add(v[1], m[MSG_SCHEDULE[r][2] as usize]);
+    v[2] = add(v[2], m[MSG_SCHEDULE[r][4] as usize]);
+    v[3] = add(v[3], m[MSG_SCHEDULE[r][6] as usize]);
+    v[0] = add(v[0], v[4]);
+    v[1] = add(v[1], v[5]);
+    v[2] = add(v[2], v[6]);
+    v[3] = add(v[3], v[7]);
+    v[12] = xor(v[12], v[0]);
+    v[13] = xor(v[13], v[1]);
+    v[14] = xor(v[14], v[2]);
+    v[15] = xor(v[15], v[3]);
+    v[12] = rot16(v[12]);
+    v[13] = rot16(v[13]);
+    v[14] = rot16(v[14]);
+    v[15] = rot16(v[15]);
+    v[8] = add(v[8], v[12]);
+    v[9] = add(v[9], v[13]);
+    v[10] = add(v[10], v[14]);
+    v[11] = add(v[11], v[15]);
+    v[4] = xor(v[4], v[8]);
+    v[5] = xor(v[5], v[9]);
+    v[6] = xor(v[6], v[10]);
+    v[7] = xor(v[7], v[11]);
+    v[4] = rot12(v[4]);
+    v[5] = rot12(v[5]);
+    v[6] = rot12(v[6]);
+    v[7] = rot12(v[7]);
+    v[0] = add(v[0], m[MSG_SCHEDULE[r][1] as usize]);
+    v[1] = add(v[1], m[MSG_SCHEDULE[r][3] as usize]);
+    v[2] = add(v[2], m[MSG_SCHEDULE[r][5] as usize]);
+    v[3] = add(v[3], m[MSG_SCHEDULE[r][7] as usize]);
+    v[0] = add(v[0], v[4]);
+    v[1] = add(v[1], v[5]);
+    v[2] = add(v[2], v[6]);
+    v[3] = add(v[3], v[7]);
+    v[12] = xor(v[12], v[0]);
+    v[13] = xor(v[13], v[1]);
+    v[14] = xor(v[14], v[2]);
+    v[15] = xor(v[15], v[3]);
+    v[12] = rot8(v[12]);
+    v[13] = rot8(v[13]);
+    v[14] = rot8(v[14]);
+    v[15] = rot8(v[15]);
+    v[8] = add(v[8], v[12]);
+    v[9] = add(v[9], v[13]);
+    v[10] = add(v[10], v[14]);
+    v[11] = add(v[11], v[15]);
+    v[4] = xor(v[4], v[8]);
+    v[5] = xor(v[5], v[9]);
+    v[6] = xor(v[6], v[10]);
+    v[7] = xor(v[7], v[11]);
+    v[4] = rot7(v[4]);
+    v[5] = rot7(v[5]);
+    v[6] = rot7(v[6]);
+    v[7] = rot7(v[7]);
+
+    v[0] = add(v[0], m[MSG_SCHEDULE[r][8] as usize]);
+    v[1] = add(v[1], m[MSG_SCHEDULE[r][10] as usize]);
+    v[2] = add(v[2], m[MSG_SCHEDULE[r][12] as usize]);
+    v[3] = add(v[3], m[MSG_SCHEDULE[r][14] as usize]);
+    v[0] = add(v[0], v[5]);
+    v[1] = add(v[1], v[6]);
+    v[2] = add(v[2], v[7]);
+    v[3] = add(v[3], v[4]);
+    v[15] = xor(v[15], v[0]);
+    v[12] = xor(v[12], v[1]);
+    v[13] = xor(v[13], v[2]);
+    v[14] = xor(v[14], v[3]);
+    v[15] = rot16(v[15]);
+    v[12] = rot16(v[12]);
+    v[13] = rot16(v[13]);
+    v[14] = rot16(v[14]);
+    v[10] = add(v[10], v[15]);
+    v[11] = add(v[11], v[12]);
+    v[8] = add(v[8], v[13]);
+    v[9] = add(v[9], v[14]);
+    v[5] = xor(v[5], v[10]);
+    v[6] = xor(v[6], v[11]);
+    v[7] = xor(v[7], v[8]);
+    v[4] = xor(v[4], v[9]);
+    v[5] = rot12(v[5]);
+    v[6] = rot12(v[6]);
+    v[7] = rot12(v[7]);
+    v[4] = rot12(v[4]);
+    v[0] = add(v[0], m[MSG_SCHEDULE[r][9] as usize]);
+    v[1] = add(v[1], m[MSG_SCHEDULE[r][11] as usize]);
+    v[2] = add(v[2], m[MSG_SCHEDULE[r][13] as usize]);
+    v[3] = add(v[3], m[MSG_SCHEDULE[r][15] as usize]);
+    v[0] = add(v[0], v[5]);
+    v[1] = add(v[1], v[6]);
+    v[2] = add(v[2], v[7]);
+    v[3] = add(v[3], v[4]);
+    v[15] = xor(v[15], v[0]);
+    v[12] = xor(v[12], v[1]);
+    v[13] = xor(v[13], v[2]);
+    v[14] = xor(v[14], v[3]);
+    v[15] = rot8(v[15]);
+    v[12] = rot8(v[12]);
+    v[13] = rot8(v[13]);
+    v[14] = rot8(v[14]);
+    v[10] = add(v[10], v[15]);
+    v[11] = add(v[11], v[12]);
+    v[8] = add(v[8], v[13]);
+    v[9] = add(v[9], v[14]);
+    v[5] = xor(v[5], v[10]);
+    v[6] = xor(v[6], v[11]);
+    v[7] = xor(v[7], v[8]);
+    v[4] = xor(v[4], v[9]);
+    v[5] = rot7(v[5]);
+    v[6] = rot7(v[6]);
+    v[7] = rot7(v[7]);
+    v[4] = rot7(v[4]);
+}
+
+#[inline(always)]
+unsafe fn interleave128(a: __m256i, b: __m256i) -> (__m256i, __m256i) {
+    (
+        _mm256_permute2x128_si256(a, b, 0x20),
+        _mm256_permute2x128_si256(a, b, 0x31),
+    )
+}
+
+// There are several ways to do a transposition. We could do it naively, with 8 separate
+// _mm256_set_epi32 instructions, referencing each of the 32 words explicitly. Or we could copy
+// the vecs into contiguous storage and then use gather instructions. This third approach is to use
+// a series of unpack instructions to interleave the vectors. In my benchmarks, interleaving is the
+// fastest approach. To test this, run `cargo +nightly bench --bench libtest load_8` in the
+// https://github.com/oconnor663/bao_experiments repo.
+#[inline(always)]
+unsafe fn transpose_vecs(vecs: &mut [__m256i; DEGREE]) {
+    // Interleave 32-bit lanes. The low unpack is lanes 00/11/44/55, and the high is 22/33/66/77.
+    let ab_0145 = _mm256_unpacklo_epi32(vecs[0], vecs[1]);
+    let ab_2367 = _mm256_unpackhi_epi32(vecs[0], vecs[1]);
+    let cd_0145 = _mm256_unpacklo_epi32(vecs[2], vecs[3]);
+    let cd_2367 = _mm256_unpackhi_epi32(vecs[2], vecs[3]);
+    let ef_0145 = _mm256_unpacklo_epi32(vecs[4], vecs[5]);
+    let ef_2367 = _mm256_unpackhi_epi32(vecs[4], vecs[5]);
+    let gh_0145 = _mm256_unpacklo_epi32(vecs[6], vecs[7]);
+    let gh_2367 = _mm256_unpackhi_epi32(vecs[6], vecs[7]);
+
+    // Interleave 64-bit lanes. The low unpack is lanes 00/22 and the high is 11/33.
+    let abcd_04 = _mm256_unpacklo_epi64(ab_0145, cd_0145);
+    let abcd_15 = _mm256_unpackhi_epi64(ab_0145, cd_0145);
+    let abcd_26 = _mm256_unpacklo_epi64(ab_2367, cd_2367);
+    let abcd_37 = _mm256_unpackhi_epi64(ab_2367, cd_2367);
+    let efgh_04 = _mm256_unpacklo_epi64(ef_0145, gh_0145);
+    let efgh_15 = _mm256_unpackhi_epi64(ef_0145, gh_0145);
+    let efgh_26 = _mm256_unpacklo_epi64(ef_2367, gh_2367);
+    let efgh_37 = _mm256_unpackhi_epi64(ef_2367, gh_2367);
+
+    // Interleave 128-bit lanes.
+    let (abcdefgh_0, abcdefgh_4) = interleave128(abcd_04, efgh_04);
+    let (abcdefgh_1, abcdefgh_5) = interleave128(abcd_15, efgh_15);
+    let (abcdefgh_2, abcdefgh_6) = interleave128(abcd_26, efgh_26);
+    let (abcdefgh_3, abcdefgh_7) = interleave128(abcd_37, efgh_37);
+
+    vecs[0] = abcdefgh_0;
+    vecs[1] = abcdefgh_1;
+    vecs[2] = abcdefgh_2;
+    vecs[3] = abcdefgh_3;
+    vecs[4] = abcdefgh_4;
+    vecs[5] = abcdefgh_5;
+    vecs[6] = abcdefgh_6;
+    vecs[7] = abcdefgh_7;
+}
+
+#[inline(always)]
+unsafe fn transpose_msg_vecs(inputs: &[*const u8; DEGREE], block_offset: usize) -> [__m256i; 16] {
+    let mut vecs = [
+        loadu(inputs[0].add(block_offset + 0 * 4 * DEGREE)),
+        loadu(inputs[1].add(block_offset + 0 * 4 * DEGREE)),
+        loadu(inputs[2].add(block_offset + 0 * 4 * DEGREE)),
+        loadu(inputs[3].add(block_offset + 0 * 4 * DEGREE)),
+        loadu(inputs[4].add(block_offset + 0 * 4 * DEGREE)),
+        loadu(inputs[5].add(block_offset + 0 * 4 * DEGREE)),
+        loadu(inputs[6].add(block_offset + 0 * 4 * DEGREE)),
+        loadu(inputs[7].add(block_offset + 0 * 4 * DEGREE)),
+        loadu(inputs[0].add(block_offset + 1 * 4 * DEGREE)),
+        loadu(inputs[1].add(block_offset + 1 * 4 * DEGREE)),
+        loadu(inputs[2].add(block_offset + 1 * 4 * DEGREE)),
+        loadu(inputs[3].add(block_offset + 1 * 4 * DEGREE)),
+        loadu(inputs[4].add(block_offset + 1 * 4 * DEGREE)),
+        loadu(inputs[5].add(block_offset + 1 * 4 * DEGREE)),
+        loadu(inputs[6].add(block_offset + 1 * 4 * DEGREE)),
+        loadu(inputs[7].add(block_offset + 1 * 4 * DEGREE)),
+    ];
+    for i in 0..DEGREE {
+        _mm_prefetch(inputs[i].add(block_offset + 256) as *const i8, _MM_HINT_T0);
+    }
+    let squares = mut_array_refs!(&mut vecs, DEGREE, DEGREE);
+    transpose_vecs(squares.0);
+    transpose_vecs(squares.1);
+    vecs
+}
+
+#[inline(always)]
+unsafe fn load_counters(counter: u64, increment_counter: IncrementCounter) -> (__m256i, __m256i) {
+    let mask = if increment_counter.yes() { !0 } else { 0 };
+    (
+        set8(
+            counter_low(counter + (mask & 0)),
+            counter_low(counter + (mask & 1)),
+            counter_low(counter + (mask & 2)),
+            counter_low(counter + (mask & 3)),
+            counter_low(counter + (mask & 4)),
+            counter_low(counter + (mask & 5)),
+            counter_low(counter + (mask & 6)),
+            counter_low(counter + (mask & 7)),
+        ),
+        set8(
+            counter_high(counter + (mask & 0)),
+            counter_high(counter + (mask & 1)),
+            counter_high(counter + (mask & 2)),
+            counter_high(counter + (mask & 3)),
+            counter_high(counter + (mask & 4)),
+            counter_high(counter + (mask & 5)),
+            counter_high(counter + (mask & 6)),
+            counter_high(counter + (mask & 7)),
+        ),
+    )
+}
+
+#[target_feature(enable = "avx2")]
+pub unsafe fn hash8(
+    inputs: &[*const u8; DEGREE],
+    blocks: usize,
+    key: &CVWords,
+    counter: u64,
+    increment_counter: IncrementCounter,
+    flags: u8,
+    flags_start: u8,
+    flags_end: u8,
+    out: &mut [u8; DEGREE * OUT_LEN],
+) {
+    let mut h_vecs = [
+        set1(key[0]),
+        set1(key[1]),
+        set1(key[2]),
+        set1(key[3]),
+        set1(key[4]),
+        set1(key[5]),
+        set1(key[6]),
+        set1(key[7]),
+    ];
+    let (counter_low_vec, counter_high_vec) = load_counters(counter, increment_counter);
+    let mut block_flags = flags | flags_start;
+
+    for block in 0..blocks {
+        if block + 1 == blocks {
+            block_flags |= flags_end;
+        }
+        let block_len_vec = set1(BLOCK_LEN as u32); // full blocks only
+        let block_flags_vec = set1(block_flags as u32);
+        let msg_vecs = transpose_msg_vecs(inputs, block * BLOCK_LEN);
+
+        // The transposed compression function. Note that inlining this
+        // manually here improves compile times by a lot, compared to factoring
+        // it out into its own function and making it #[inline(always)]. Just
+        // guessing, it might have something to do with loop unrolling.
+        let mut v = [
+            h_vecs[0],
+            h_vecs[1],
+            h_vecs[2],
+            h_vecs[3],
+            h_vecs[4],
+            h_vecs[5],
+            h_vecs[6],
+            h_vecs[7],
+            set1(IV[0]),
+            set1(IV[1]),
+            set1(IV[2]),
+            set1(IV[3]),
+            counter_low_vec,
+            counter_high_vec,
+            block_len_vec,
+            block_flags_vec,
+        ];
+        round(&mut v, &msg_vecs, 0);
+        round(&mut v, &msg_vecs, 1);
+        round(&mut v, &msg_vecs, 2);
+        round(&mut v, &msg_vecs, 3);
+        round(&mut v, &msg_vecs, 4);
+        round(&mut v, &msg_vecs, 5);
+        round(&mut v, &msg_vecs, 6);
+        h_vecs[0] = xor(v[0], v[8]);
+        h_vecs[1] = xor(v[1], v[9]);
+        h_vecs[2] = xor(v[2], v[10]);
+        h_vecs[3] = xor(v[3], v[11]);
+        h_vecs[4] = xor(v[4], v[12]);
+        h_vecs[5] = xor(v[5], v[13]);
+        h_vecs[6] = xor(v[6], v[14]);
+        h_vecs[7] = xor(v[7], v[15]);
+
+        block_flags = flags;
+    }
+
+    transpose_vecs(&mut h_vecs);
+    storeu(h_vecs[0], out.as_mut_ptr().add(0 * 4 * DEGREE));
+    storeu(h_vecs[1], out.as_mut_ptr().add(1 * 4 * DEGREE));
+    storeu(h_vecs[2], out.as_mut_ptr().add(2 * 4 * DEGREE));
+    storeu(h_vecs[3], out.as_mut_ptr().add(3 * 4 * DEGREE));
+    storeu(h_vecs[4], out.as_mut_ptr().add(4 * 4 * DEGREE));
+    storeu(h_vecs[5], out.as_mut_ptr().add(5 * 4 * DEGREE));
+    storeu(h_vecs[6], out.as_mut_ptr().add(6 * 4 * DEGREE));
+    storeu(h_vecs[7], out.as_mut_ptr().add(7 * 4 * DEGREE));
+}
+
+#[target_feature(enable = "avx2")]
+pub unsafe fn hash_many<const N: usize>(
+    mut inputs: &[&[u8; N]],
+    key: &CVWords,
+    mut counter: u64,
+    increment_counter: IncrementCounter,
+    flags: u8,
+    flags_start: u8,
+    flags_end: u8,
+    mut out: &mut [u8],
+) {
+    debug_assert!(out.len() >= inputs.len() * OUT_LEN, "out too short");
+    while inputs.len() >= DEGREE && out.len() >= DEGREE * OUT_LEN {
+        // Safe because the layout of arrays is guaranteed, and because the
+        // `blocks` count is determined statically from the argument type.
+        let input_ptrs: &[*const u8; DEGREE] = &*(inputs.as_ptr() as *const [*const u8; DEGREE]);
+        let blocks = N / BLOCK_LEN;
+        hash8(
+            input_ptrs,
+            blocks,
+            key,
+            counter,
+            increment_counter,
+            flags,
+            flags_start,
+            flags_end,
+            array_mut_ref!(out, 0, DEGREE * OUT_LEN),
+        );
+        if increment_counter.yes() {
+            counter += DEGREE as u64;
+        }
+        inputs = &inputs[DEGREE..];
+        out = &mut out[DEGREE * OUT_LEN..];
+    }
+    crate::sse41::hash_many(
+        inputs,
+        key,
+        counter,
+        increment_counter,
+        flags,
+        flags_start,
+        flags_end,
+        out,
+    );
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn test_transpose() {
+        if !crate::platform::avx2_detected() {
+            return;
+        }
+
+        #[target_feature(enable = "avx2")]
+        unsafe fn transpose_wrapper(vecs: &mut [__m256i; DEGREE]) {
+            transpose_vecs(vecs);
+        }
+
+        let mut matrix = [[0 as u32; DEGREE]; DEGREE];
+        for i in 0..DEGREE {
+            for j in 0..DEGREE {
+                matrix[i][j] = (i * DEGREE + j) as u32;
+            }
+        }
+
+        unsafe {
+            let mut vecs: [__m256i; DEGREE] = core::mem::transmute(matrix);
+            transpose_wrapper(&mut vecs);
+            matrix = core::mem::transmute(vecs);
+        }
+
+        for i in 0..DEGREE {
+            for j in 0..DEGREE {
+                // Reversed indexes from above.
+                assert_eq!(matrix[j][i], (i * DEGREE + j) as u32);
+            }
+        }
+    }
+
+    #[test]
+    fn test_hash_many() {
+        if !crate::platform::avx2_detected() {
+            return;
+        }
+        crate::test::test_hash_many_fn(hash_many, hash_many);
+    }
+}

vendor/blake3/src/rust_sse2.rs

@@ -0,0 +1,775 @@
+#[cfg(target_arch = "x86")]
+use core::arch::x86::*;
+#[cfg(target_arch = "x86_64")]
+use core::arch::x86_64::*;
+
+use crate::{
+    counter_high, counter_low, CVBytes, CVWords, IncrementCounter, BLOCK_LEN, IV, MSG_SCHEDULE,
+    OUT_LEN,
+};
+use arrayref::{array_mut_ref, array_ref, mut_array_refs};
+
+pub const DEGREE: usize = 4;
+
+#[inline(always)]
+unsafe fn loadu(src: *const u8) -> __m128i {
+    // This is an unaligned load, so the pointer cast is allowed.
+    _mm_loadu_si128(src as *const __m128i)
+}
+
+#[inline(always)]
+unsafe fn storeu(src: __m128i, dest: *mut u8) {
+    // This is an unaligned store, so the pointer cast is allowed.
+    _mm_storeu_si128(dest as *mut __m128i, src)
+}
+
+#[inline(always)]
+unsafe fn add(a: __m128i, b: __m128i) -> __m128i {
+    _mm_add_epi32(a, b)
+}
+
+#[inline(always)]
+unsafe fn xor(a: __m128i, b: __m128i) -> __m128i {
+    _mm_xor_si128(a, b)
+}
+
+#[inline(always)]
+unsafe fn set1(x: u32) -> __m128i {
+    _mm_set1_epi32(x as i32)
+}
+
+#[inline(always)]
+unsafe fn set4(a: u32, b: u32, c: u32, d: u32) -> __m128i {
+    _mm_setr_epi32(a as i32, b as i32, c as i32, d as i32)
+}
+
+// These rotations are the "simple/shifts version". For the
+// "complicated/shuffles version", see
+// https://github.com/sneves/blake2-avx2/blob/b3723921f668df09ece52dcd225a36d4a4eea1d9/blake2s-common.h#L63-L66.
+// For a discussion of the tradeoffs, see
+// https://github.com/sneves/blake2-avx2/pull/5. Due to an LLVM bug
+// (https://bugs.llvm.org/show_bug.cgi?id=44379), this version performs better
+// on recent x86 chips.
+
+#[inline(always)]
+unsafe fn rot16(a: __m128i) -> __m128i {
+    _mm_or_si128(_mm_srli_epi32(a, 16), _mm_slli_epi32(a, 32 - 16))
+}
+
+#[inline(always)]
+unsafe fn rot12(a: __m128i) -> __m128i {
+    _mm_or_si128(_mm_srli_epi32(a, 12), _mm_slli_epi32(a, 32 - 12))
+}
+
+#[inline(always)]
+unsafe fn rot8(a: __m128i) -> __m128i {
+    _mm_or_si128(_mm_srli_epi32(a, 8), _mm_slli_epi32(a, 32 - 8))
+}
+
+#[inline(always)]
+unsafe fn rot7(a: __m128i) -> __m128i {
+    _mm_or_si128(_mm_srli_epi32(a, 7), _mm_slli_epi32(a, 32 - 7))
+}
+
+#[inline(always)]
+unsafe fn g1(
+    row0: &mut __m128i,
+    row1: &mut __m128i,
+    row2: &mut __m128i,
+    row3: &mut __m128i,
+    m: __m128i,
+) {
+    *row0 = add(add(*row0, m), *row1);
+    *row3 = xor(*row3, *row0);
+    *row3 = rot16(*row3);
+    *row2 = add(*row2, *row3);
+    *row1 = xor(*row1, *row2);
+    *row1 = rot12(*row1);
+}
+
+#[inline(always)]
+unsafe fn g2(
+    row0: &mut __m128i,
+    row1: &mut __m128i,
+    row2: &mut __m128i,
+    row3: &mut __m128i,
+    m: __m128i,
+) {
+    *row0 = add(add(*row0, m), *row1);
+    *row3 = xor(*row3, *row0);
+    *row3 = rot8(*row3);
+    *row2 = add(*row2, *row3);
+    *row1 = xor(*row1, *row2);
+    *row1 = rot7(*row1);
+}
+
+// Adapted from https://github.com/rust-lang-nursery/stdsimd/pull/479.
+macro_rules! _MM_SHUFFLE {
+    ($z:expr, $y:expr, $x:expr, $w:expr) => {
+        ($z << 6) | ($y << 4) | ($x << 2) | $w
+    };
+}
+
+macro_rules! shuffle2 {
+    ($a:expr, $b:expr, $c:expr) => {
+        _mm_castps_si128(_mm_shuffle_ps(
+            _mm_castsi128_ps($a),
+            _mm_castsi128_ps($b),
+            $c,
+        ))
+    };
+}
+
+// Note the optimization here of leaving row1 as the unrotated row, rather than
+// row0. All the message loads below are adjusted to compensate for this. See
+// discussion at https://github.com/sneves/blake2-avx2/pull/4
+#[inline(always)]
+unsafe fn diagonalize(row0: &mut __m128i, row2: &mut __m128i, row3: &mut __m128i) {
+    *row0 = _mm_shuffle_epi32(*row0, _MM_SHUFFLE!(2, 1, 0, 3));
+    *row3 = _mm_shuffle_epi32(*row3, _MM_SHUFFLE!(1, 0, 3, 2));
+    *row2 = _mm_shuffle_epi32(*row2, _MM_SHUFFLE!(0, 3, 2, 1));
+}
+
+#[inline(always)]
+unsafe fn undiagonalize(row0: &mut __m128i, row2: &mut __m128i, row3: &mut __m128i) {
+    *row0 = _mm_shuffle_epi32(*row0, _MM_SHUFFLE!(0, 3, 2, 1));
+    *row3 = _mm_shuffle_epi32(*row3, _MM_SHUFFLE!(1, 0, 3, 2));
+    *row2 = _mm_shuffle_epi32(*row2, _MM_SHUFFLE!(2, 1, 0, 3));
+}
+
+#[inline(always)]
+unsafe fn blend_epi16(a: __m128i, b: __m128i, imm8: i32) -> __m128i {
+    let bits = _mm_set_epi16(0x80, 0x40, 0x20, 0x10, 0x08, 0x04, 0x02, 0x01);
+    let mut mask = _mm_set1_epi16(imm8 as i16);
+    mask = _mm_and_si128(mask, bits);
+    mask = _mm_cmpeq_epi16(mask, bits);
+    _mm_or_si128(_mm_and_si128(mask, b), _mm_andnot_si128(mask, a))
+}
+
+#[inline(always)]
+unsafe fn compress_pre(
+    cv: &CVWords,
+    block: &[u8; BLOCK_LEN],
+    block_len: u8,
+    counter: u64,
+    flags: u8,
+) -> [__m128i; 4] {
+    let row0 = &mut loadu(cv.as_ptr().add(0) as *const u8);
+    let row1 = &mut loadu(cv.as_ptr().add(4) as *const u8);
+    let row2 = &mut set4(IV[0], IV[1], IV[2], IV[3]);
+    let row3 = &mut set4(
+        counter_low(counter),
+        counter_high(counter),
+        block_len as u32,
+        flags as u32,
+    );
+
+    let mut m0 = loadu(block.as_ptr().add(0 * 4 * DEGREE));
+    let mut m1 = loadu(block.as_ptr().add(1 * 4 * DEGREE));
+    let mut m2 = loadu(block.as_ptr().add(2 * 4 * DEGREE));
+    let mut m3 = loadu(block.as_ptr().add(3 * 4 * DEGREE));
+
+    let mut t0;
+    let mut t1;
+    let mut t2;
+    let mut t3;
+    let mut tt;
+
+    // Round 1. The first round permutes the message words from the original
+    // input order, into the groups that get mixed in parallel.
+    t0 = shuffle2!(m0, m1, _MM_SHUFFLE!(2, 0, 2, 0)); //  6  4  2  0
+    g1(row0, row1, row2, row3, t0);
+    t1 = shuffle2!(m0, m1, _MM_SHUFFLE!(3, 1, 3, 1)); //  7  5  3  1
+    g2(row0, row1, row2, row3, t1);
+    diagonalize(row0, row2, row3);
+    t2 = shuffle2!(m2, m3, _MM_SHUFFLE!(2, 0, 2, 0)); // 14 12 10  8
+    t2 = _mm_shuffle_epi32(t2, _MM_SHUFFLE!(2, 1, 0, 3)); // 12 10  8 14
+    g1(row0, row1, row2, row3, t2);
+    t3 = shuffle2!(m2, m3, _MM_SHUFFLE!(3, 1, 3, 1)); // 15 13 11  9
+    t3 = _mm_shuffle_epi32(t3, _MM_SHUFFLE!(2, 1, 0, 3)); // 13 11  9 15
+    g2(row0, row1, row2, row3, t3);
+    undiagonalize(row0, row2, row3);
+    m0 = t0;
+    m1 = t1;
+    m2 = t2;
+    m3 = t3;
+
+    // Round 2. This round and all following rounds apply a fixed permutation
+    // to the message words from the round before.
+    t0 = shuffle2!(m0, m1, _MM_SHUFFLE!(3, 1, 1, 2));
+    t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE!(0, 3, 2, 1));
+    g1(row0, row1, row2, row3, t0);
+    t1 = shuffle2!(m2, m3, _MM_SHUFFLE!(3, 3, 2, 2));
+    tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE!(0, 0, 3, 3));
+    t1 = blend_epi16(tt, t1, 0xCC);
+    g2(row0, row1, row2, row3, t1);
+    diagonalize(row0, row2, row3);
+    t2 = _mm_unpacklo_epi64(m3, m1);
+    tt = blend_epi16(t2, m2, 0xC0);
+    t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE!(1, 3, 2, 0));
+    g1(row0, row1, row2, row3, t2);
+    t3 = _mm_unpackhi_epi32(m1, m3);
+    tt = _mm_unpacklo_epi32(m2, t3);
+    t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE!(0, 1, 3, 2));
+    g2(row0, row1, row2, row3, t3);
+    undiagonalize(row0, row2, row3);
+    m0 = t0;
+    m1 = t1;
+    m2 = t2;
+    m3 = t3;
+
+    // Round 3
+    t0 = shuffle2!(m0, m1, _MM_SHUFFLE!(3, 1, 1, 2));
+    t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE!(0, 3, 2, 1));
+    g1(row0, row1, row2, row3, t0);
+    t1 = shuffle2!(m2, m3, _MM_SHUFFLE!(3, 3, 2, 2));
+    tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE!(0, 0, 3, 3));
+    t1 = blend_epi16(tt, t1, 0xCC);
+    g2(row0, row1, row2, row3, t1);
+    diagonalize(row0, row2, row3);
+    t2 = _mm_unpacklo_epi64(m3, m1);
+    tt = blend_epi16(t2, m2, 0xC0);
+    t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE!(1, 3, 2, 0));
+    g1(row0, row1, row2, row3, t2);
+    t3 = _mm_unpackhi_epi32(m1, m3);
+    tt = _mm_unpacklo_epi32(m2, t3);
+    t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE!(0, 1, 3, 2));
+    g2(row0, row1, row2, row3, t3);
+    undiagonalize(row0, row2, row3);
+    m0 = t0;
+    m1 = t1;
+    m2 = t2;
+    m3 = t3;
+
+    // Round 4
+    t0 = shuffle2!(m0, m1, _MM_SHUFFLE!(3, 1, 1, 2));
+    t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE!(0, 3, 2, 1));
+    g1(row0, row1, row2, row3, t0);
+    t1 = shuffle2!(m2, m3, _MM_SHUFFLE!(3, 3, 2, 2));
+    tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE!(0, 0, 3, 3));
+    t1 = blend_epi16(tt, t1, 0xCC);
+    g2(row0, row1, row2, row3, t1);
+    diagonalize(row0, row2, row3);
+    t2 = _mm_unpacklo_epi64(m3, m1);
+    tt = blend_epi16(t2, m2, 0xC0);
+    t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE!(1, 3, 2, 0));
+    g1(row0, row1, row2, row3, t2);
+    t3 = _mm_unpackhi_epi32(m1, m3);
+    tt = _mm_unpacklo_epi32(m2, t3);
+    t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE!(0, 1, 3, 2));
+    g2(row0, row1, row2, row3, t3);
+    undiagonalize(row0, row2, row3);
+    m0 = t0;
+    m1 = t1;
+    m2 = t2;
+    m3 = t3;
+
+    // Round 5
+    t0 = shuffle2!(m0, m1, _MM_SHUFFLE!(3, 1, 1, 2));
+    t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE!(0, 3, 2, 1));
+    g1(row0, row1, row2, row3, t0);
+    t1 = shuffle2!(m2, m3, _MM_SHUFFLE!(3, 3, 2, 2));
+    tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE!(0, 0, 3, 3));
+    t1 = blend_epi16(tt, t1, 0xCC);
+    g2(row0, row1, row2, row3, t1);
+    diagonalize(row0, row2, row3);
+    t2 = _mm_unpacklo_epi64(m3, m1);
+    tt = blend_epi16(t2, m2, 0xC0);
+    t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE!(1, 3, 2, 0));
+    g1(row0, row1, row2, row3, t2);
+    t3 = _mm_unpackhi_epi32(m1, m3);
+    tt = _mm_unpacklo_epi32(m2, t3);
+    t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE!(0, 1, 3, 2));
+    g2(row0, row1, row2, row3, t3);
+    undiagonalize(row0, row2, row3);
+    m0 = t0;
+    m1 = t1;
+    m2 = t2;
+    m3 = t3;
+
+    // Round 6
+    t0 = shuffle2!(m0, m1, _MM_SHUFFLE!(3, 1, 1, 2));
+    t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE!(0, 3, 2, 1));
+    g1(row0, row1, row2, row3, t0);
+    t1 = shuffle2!(m2, m3, _MM_SHUFFLE!(3, 3, 2, 2));
+    tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE!(0, 0, 3, 3));
+    t1 = blend_epi16(tt, t1, 0xCC);
+    g2(row0, row1, row2, row3, t1);
+    diagonalize(row0, row2, row3);
+    t2 = _mm_unpacklo_epi64(m3, m1);
+    tt = blend_epi16(t2, m2, 0xC0);
+    t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE!(1, 3, 2, 0));
+    g1(row0, row1, row2, row3, t2);
+    t3 = _mm_unpackhi_epi32(m1, m3);
+    tt = _mm_unpacklo_epi32(m2, t3);
+    t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE!(0, 1, 3, 2));
+    g2(row0, row1, row2, row3, t3);
+    undiagonalize(row0, row2, row3);
+    m0 = t0;
+    m1 = t1;
+    m2 = t2;
+    m3 = t3;
+
+    // Round 7
+    t0 = shuffle2!(m0, m1, _MM_SHUFFLE!(3, 1, 1, 2));
+    t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE!(0, 3, 2, 1));
+    g1(row0, row1, row2, row3, t0);
+    t1 = shuffle2!(m2, m3, _MM_SHUFFLE!(3, 3, 2, 2));
+    tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE!(0, 0, 3, 3));
+    t1 = blend_epi16(tt, t1, 0xCC);
+    g2(row0, row1, row2, row3, t1);
+    diagonalize(row0, row2, row3);
+    t2 = _mm_unpacklo_epi64(m3, m1);
+    tt = blend_epi16(t2, m2, 0xC0);
+    t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE!(1, 3, 2, 0));
+    g1(row0, row1, row2, row3, t2);
+    t3 = _mm_unpackhi_epi32(m1, m3);
+    tt = _mm_unpacklo_epi32(m2, t3);
+    t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE!(0, 1, 3, 2));
+    g2(row0, row1, row2, row3, t3);
+    undiagonalize(row0, row2, row3);
+
+    [*row0, *row1, *row2, *row3]
+}
+
+#[target_feature(enable = "sse2")]
+pub unsafe fn compress_in_place(
+    cv: &mut CVWords,
+    block: &[u8; BLOCK_LEN],
+    block_len: u8,
+    counter: u64,
+    flags: u8,
+) {
+    let [row0, row1, row2, row3] = compress_pre(cv, block, block_len, counter, flags);
+    storeu(xor(row0, row2), cv.as_mut_ptr().add(0) as *mut u8);
+    storeu(xor(row1, row3), cv.as_mut_ptr().add(4) as *mut u8);
+}
+
+#[target_feature(enable = "sse2")]
+pub unsafe fn compress_xof(
+    cv: &CVWords,
+    block: &[u8; BLOCK_LEN],
+    block_len: u8,
+    counter: u64,
+    flags: u8,
+) -> [u8; 64] {
+    let [mut row0, mut row1, mut row2, mut row3] =
+        compress_pre(cv, block, block_len, counter, flags);
+    row0 = xor(row0, row2);
+    row1 = xor(row1, row3);
+    row2 = xor(row2, loadu(cv.as_ptr().add(0) as *const u8));
+    row3 = xor(row3, loadu(cv.as_ptr().add(4) as *const u8));
+    core::mem::transmute([row0, row1, row2, row3])
+}
+
+#[inline(always)]
+unsafe fn round(v: &mut [__m128i; 16], m: &[__m128i; 16], r: usize) {
+    v[0] = add(v[0], m[MSG_SCHEDULE[r][0] as usize]);
+    v[1] = add(v[1], m[MSG_SCHEDULE[r][2] as usize]);
+    v[2] = add(v[2], m[MSG_SCHEDULE[r][4] as usize]);
+    v[3] = add(v[3], m[MSG_SCHEDULE[r][6] as usize]);
+    v[0] = add(v[0], v[4]);
+    v[1] = add(v[1], v[5]);
+    v[2] = add(v[2], v[6]);
+    v[3] = add(v[3], v[7]);
+    v[12] = xor(v[12], v[0]);
+    v[13] = xor(v[13], v[1]);
+    v[14] = xor(v[14], v[2]);
+    v[15] = xor(v[15], v[3]);
+    v[12] = rot16(v[12]);
+    v[13] = rot16(v[13]);
+    v[14] = rot16(v[14]);
+    v[15] = rot16(v[15]);
+    v[8] = add(v[8], v[12]);
+    v[9] = add(v[9], v[13]);
+    v[10] = add(v[10], v[14]);
+    v[11] = add(v[11], v[15]);
+    v[4] = xor(v[4], v[8]);
+    v[5] = xor(v[5], v[9]);
+    v[6] = xor(v[6], v[10]);
+    v[7] = xor(v[7], v[11]);
+    v[4] = rot12(v[4]);
+    v[5] = rot12(v[5]);
+    v[6] = rot12(v[6]);
+    v[7] = rot12(v[7]);
+    v[0] = add(v[0], m[MSG_SCHEDULE[r][1] as usize]);
+    v[1] = add(v[1], m[MSG_SCHEDULE[r][3] as usize]);
+    v[2] = add(v[2], m[MSG_SCHEDULE[r][5] as usize]);
+    v[3] = add(v[3], m[MSG_SCHEDULE[r][7] as usize]);
+    v[0] = add(v[0], v[4]);
+    v[1] = add(v[1], v[5]);
+    v[2] = add(v[2], v[6]);
+    v[3] = add(v[3], v[7]);
+    v[12] = xor(v[12], v[0]);
+    v[13] = xor(v[13], v[1]);
+    v[14] = xor(v[14], v[2]);
+    v[15] = xor(v[15], v[3]);
+    v[12] = rot8(v[12]);
+    v[13] = rot8(v[13]);
+    v[14] = rot8(v[14]);
+    v[15] = rot8(v[15]);
+    v[8] = add(v[8], v[12]);
+    v[9] = add(v[9], v[13]);
+    v[10] = add(v[10], v[14]);
+    v[11] = add(v[11], v[15]);
+    v[4] = xor(v[4], v[8]);
+    v[5] = xor(v[5], v[9]);
+    v[6] = xor(v[6], v[10]);
+    v[7] = xor(v[7], v[11]);
+    v[4] = rot7(v[4]);
+    v[5] = rot7(v[5]);
+    v[6] = rot7(v[6]);
+    v[7] = rot7(v[7]);
+
+    v[0] = add(v[0], m[MSG_SCHEDULE[r][8] as usize]);
+    v[1] = add(v[1], m[MSG_SCHEDULE[r][10] as usize]);
+    v[2] = add(v[2], m[MSG_SCHEDULE[r][12] as usize]);
+    v[3] = add(v[3], m[MSG_SCHEDULE[r][14] as usize]);
+    v[0] = add(v[0], v[5]);
+    v[1] = add(v[1], v[6]);
+    v[2] = add(v[2], v[7]);
+    v[3] = add(v[3], v[4]);
+    v[15] = xor(v[15], v[0]);
+    v[12] = xor(v[12], v[1]);
+    v[13] = xor(v[13], v[2]);
+    v[14] = xor(v[14], v[3]);
+    v[15] = rot16(v[15]);
+    v[12] = rot16(v[12]);
+    v[13] = rot16(v[13]);
+    v[14] = rot16(v[14]);
+    v[10] = add(v[10], v[15]);
+    v[11] = add(v[11], v[12]);
+    v[8] = add(v[8], v[13]);
+    v[9] = add(v[9], v[14]);
+    v[5] = xor(v[5], v[10]);
+    v[6] = xor(v[6], v[11]);
+    v[7] = xor(v[7], v[8]);
+    v[4] = xor(v[4], v[9]);
+    v[5] = rot12(v[5]);
+    v[6] = rot12(v[6]);
+    v[7] = rot12(v[7]);
+    v[4] = rot12(v[4]);
+    v[0] = add(v[0], m[MSG_SCHEDULE[r][9] as usize]);
+    v[1] = add(v[1], m[MSG_SCHEDULE[r][11] as usize]);
+    v[2] = add(v[2], m[MSG_SCHEDULE[r][13] as usize]);
+    v[3] = add(v[3], m[MSG_SCHEDULE[r][15] as usize]);
+    v[0] = add(v[0], v[5]);
+    v[1] = add(v[1], v[6]);
+    v[2] = add(v[2], v[7]);
+    v[3] = add(v[3], v[4]);
+    v[15] = xor(v[15], v[0]);
+    v[12] = xor(v[12], v[1]);
+    v[13] = xor(v[13], v[2]);
+    v[14] = xor(v[14], v[3]);
+    v[15] = rot8(v[15]);
+    v[12] = rot8(v[12]);
+    v[13] = rot8(v[13]);
+    v[14] = rot8(v[14]);
+    v[10] = add(v[10], v[15]);
+    v[11] = add(v[11], v[12]);
+    v[8] = add(v[8], v[13]);
+    v[9] = add(v[9], v[14]);
+    v[5] = xor(v[5], v[10]);
+    v[6] = xor(v[6], v[11]);
+    v[7] = xor(v[7], v[8]);
+    v[4] = xor(v[4], v[9]);
+    v[5] = rot7(v[5]);
+    v[6] = rot7(v[6]);
+    v[7] = rot7(v[7]);
+    v[4] = rot7(v[4]);
+}
+
+#[inline(always)]
+unsafe fn transpose_vecs(vecs: &mut [__m128i; DEGREE]) {
+    // Interleave 32-bit lanes. The low unpack is lanes 00/11 and the high is
+    // 22/33. Note that this doesn't split the vector into two lanes, as the
+    // AVX2 counterparts do.
+    let ab_01 = _mm_unpacklo_epi32(vecs[0], vecs[1]);
+    let ab_23 = _mm_unpackhi_epi32(vecs[0], vecs[1]);
+    let cd_01 = _mm_unpacklo_epi32(vecs[2], vecs[3]);
+    let cd_23 = _mm_unpackhi_epi32(vecs[2], vecs[3]);
+
+    // Interleave 64-bit lanes.
+    let abcd_0 = _mm_unpacklo_epi64(ab_01, cd_01);
+    let abcd_1 = _mm_unpackhi_epi64(ab_01, cd_01);
+    let abcd_2 = _mm_unpacklo_epi64(ab_23, cd_23);
+    let abcd_3 = _mm_unpackhi_epi64(ab_23, cd_23);
+
+    vecs[0] = abcd_0;
+    vecs[1] = abcd_1;
+    vecs[2] = abcd_2;
+    vecs[3] = abcd_3;
+}
+
+#[inline(always)]
+unsafe fn transpose_msg_vecs(inputs: &[*const u8; DEGREE], block_offset: usize) -> [__m128i; 16] {
+    let mut vecs = [
+        loadu(inputs[0].add(block_offset + 0 * 4 * DEGREE)),
+        loadu(inputs[1].add(block_offset + 0 * 4 * DEGREE)),
+        loadu(inputs[2].add(block_offset + 0 * 4 * DEGREE)),
+        loadu(inputs[3].add(block_offset + 0 * 4 * DEGREE)),
+        loadu(inputs[0].add(block_offset + 1 * 4 * DEGREE)),
+        loadu(inputs[1].add(block_offset + 1 * 4 * DEGREE)),
+        loadu(inputs[2].add(block_offset + 1 * 4 * DEGREE)),
+        loadu(inputs[3].add(block_offset + 1 * 4 * DEGREE)),
+        loadu(inputs[0].add(block_offset + 2 * 4 * DEGREE)),
+        loadu(inputs[1].add(block_offset + 2 * 4 * DEGREE)),
+        loadu(inputs[2].add(block_offset + 2 * 4 * DEGREE)),
+        loadu(inputs[3].add(block_offset + 2 * 4 * DEGREE)),
+        loadu(inputs[0].add(block_offset + 3 * 4 * DEGREE)),
+        loadu(inputs[1].add(block_offset + 3 * 4 * DEGREE)),
+        loadu(inputs[2].add(block_offset + 3 * 4 * DEGREE)),
+        loadu(inputs[3].add(block_offset + 3 * 4 * DEGREE)),
+    ];
+    for i in 0..DEGREE {
+        _mm_prefetch(inputs[i].add(block_offset + 256) as *const i8, _MM_HINT_T0);
+    }
+    let squares = mut_array_refs!(&mut vecs, DEGREE, DEGREE, DEGREE, DEGREE);
+    transpose_vecs(squares.0);
+    transpose_vecs(squares.1);
+    transpose_vecs(squares.2);
+    transpose_vecs(squares.3);
+    vecs
+}
+
+#[inline(always)]
+unsafe fn load_counters(counter: u64, increment_counter: IncrementCounter) -> (__m128i, __m128i) {
+    let mask = if increment_counter.yes() { !0 } else { 0 };
+    (
+        set4(
+            counter_low(counter + (mask & 0)),
+            counter_low(counter + (mask & 1)),
+            counter_low(counter + (mask & 2)),
+            counter_low(counter + (mask & 3)),
+        ),
+        set4(
+            counter_high(counter + (mask & 0)),
+            counter_high(counter + (mask & 1)),
+            counter_high(counter + (mask & 2)),
+            counter_high(counter + (mask & 3)),
+        ),
+    )
+}
+
+#[target_feature(enable = "sse2")]
+pub unsafe fn hash4(
+    inputs: &[*const u8; DEGREE],
+    blocks: usize,
+    key: &CVWords,
+    counter: u64,
+    increment_counter: IncrementCounter,
+    flags: u8,
+    flags_start: u8,
+    flags_end: u8,
+    out: &mut [u8; DEGREE * OUT_LEN],
+) {
+    let mut h_vecs = [
+        set1(key[0]),
+        set1(key[1]),
+        set1(key[2]),
+        set1(key[3]),
+        set1(key[4]),
+        set1(key[5]),
+        set1(key[6]),
+        set1(key[7]),
+    ];
+    let (counter_low_vec, counter_high_vec) = load_counters(counter, increment_counter);
+    let mut block_flags = flags | flags_start;
+
+    for block in 0..blocks {
+        if block + 1 == blocks {
+            block_flags |= flags_end;
+        }
+        let block_len_vec = set1(BLOCK_LEN as u32); // full blocks only
+        let block_flags_vec = set1(block_flags as u32);
+        let msg_vecs = transpose_msg_vecs(inputs, block * BLOCK_LEN);
+
+        // The transposed compression function. Note that inlining this
+        // manually here improves compile times by a lot, compared to factoring
+        // it out into its own function and making it #[inline(always)]. Just
+        // guessing, it might have something to do with loop unrolling.
+        let mut v = [
+            h_vecs[0],
+            h_vecs[1],
+            h_vecs[2],
+            h_vecs[3],
+            h_vecs[4],
+            h_vecs[5],
+            h_vecs[6],
+            h_vecs[7],
+            set1(IV[0]),
+            set1(IV[1]),
+            set1(IV[2]),
+            set1(IV[3]),
+            counter_low_vec,
+            counter_high_vec,
+            block_len_vec,
+            block_flags_vec,
+        ];
+        round(&mut v, &msg_vecs, 0);
+        round(&mut v, &msg_vecs, 1);
+        round(&mut v, &msg_vecs, 2);
+        round(&mut v, &msg_vecs, 3);
+        round(&mut v, &msg_vecs, 4);
+        round(&mut v, &msg_vecs, 5);
+        round(&mut v, &msg_vecs, 6);
+        h_vecs[0] = xor(v[0], v[8]);
+        h_vecs[1] = xor(v[1], v[9]);
+        h_vecs[2] = xor(v[2], v[10]);
+        h_vecs[3] = xor(v[3], v[11]);
+        h_vecs[4] = xor(v[4], v[12]);
+        h_vecs[5] = xor(v[5], v[13]);
+        h_vecs[6] = xor(v[6], v[14]);
+        h_vecs[7] = xor(v[7], v[15]);
+
+        block_flags = flags;
+    }
+
+    let squares = mut_array_refs!(&mut h_vecs, DEGREE, DEGREE);
+    transpose_vecs(squares.0);
+    transpose_vecs(squares.1);
+    // The first four vecs now contain the first half of each output, and the
+    // second four vecs contain the second half of each output.
+    storeu(h_vecs[0], out.as_mut_ptr().add(0 * 4 * DEGREE));
+    storeu(h_vecs[4], out.as_mut_ptr().add(1 * 4 * DEGREE));
+    storeu(h_vecs[1], out.as_mut_ptr().add(2 * 4 * DEGREE));
+    storeu(h_vecs[5], out.as_mut_ptr().add(3 * 4 * DEGREE));
+    storeu(h_vecs[2], out.as_mut_ptr().add(4 * 4 * DEGREE));
+    storeu(h_vecs[6], out.as_mut_ptr().add(5 * 4 * DEGREE));
+    storeu(h_vecs[3], out.as_mut_ptr().add(6 * 4 * DEGREE));
+    storeu(h_vecs[7], out.as_mut_ptr().add(7 * 4 * DEGREE));
+}
+
+#[target_feature(enable = "sse2")]
+unsafe fn hash1<const N: usize>(
+    input: &[u8; N],
+    key: &CVWords,
+    counter: u64,
+    flags: u8,
+    flags_start: u8,
+    flags_end: u8,
+    out: &mut CVBytes,
+) {
+    debug_assert_eq!(N % BLOCK_LEN, 0, "uneven blocks");
+    let mut cv = *key;
+    let mut block_flags = flags | flags_start;
+    let mut slice = &input[..];
+    while slice.len() >= BLOCK_LEN {
+        if slice.len() == BLOCK_LEN {
+            block_flags |= flags_end;
+        }
+        compress_in_place(
+            &mut cv,
+            array_ref!(slice, 0, BLOCK_LEN),
+            BLOCK_LEN as u8,
+            counter,
+            block_flags,
+        );
+        block_flags = flags;
+        slice = &slice[BLOCK_LEN..];
+    }
+    *out = core::mem::transmute(cv); // x86 is little-endian
+}
+
+#[target_feature(enable = "sse2")]
+pub unsafe fn hash_many<const N: usize>(
+    mut inputs: &[&[u8; N]],
+    key: &CVWords,
+    mut counter: u64,
+    increment_counter: IncrementCounter,
+    flags: u8,
+    flags_start: u8,
+    flags_end: u8,
+    mut out: &mut [u8],
+) {
+    debug_assert!(out.len() >= inputs.len() * OUT_LEN, "out too short");
+    while inputs.len() >= DEGREE && out.len() >= DEGREE * OUT_LEN {
+        // Safe because the layout of arrays is guaranteed, and because the
+        // `blocks` count is determined statically from the argument type.
+        let input_ptrs: &[*const u8; DEGREE] = &*(inputs.as_ptr() as *const [*const u8; DEGREE]);
+        let blocks = N / BLOCK_LEN;
+        hash4(
+            input_ptrs,
+            blocks,
+            key,
+            counter,
+            increment_counter,
+            flags,
+            flags_start,
+            flags_end,
+            array_mut_ref!(out, 0, DEGREE * OUT_LEN),
+        );
+        if increment_counter.yes() {
+            counter += DEGREE as u64;
+        }
+        inputs = &inputs[DEGREE..];
+        out = &mut out[DEGREE * OUT_LEN..];
+    }
+    for (&input, output) in inputs.iter().zip(out.chunks_exact_mut(OUT_LEN)) {
+        hash1(
+            input,
+            key,
+            counter,
+            flags,
+            flags_start,
+            flags_end,
+            array_mut_ref!(output, 0, OUT_LEN),
+        );
+        if increment_counter.yes() {
+            counter += 1;
+        }
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn test_transpose() {
+        if !crate::platform::sse2_detected() {
+            return;
+        }
+
+        #[target_feature(enable = "sse2")]
+        unsafe fn transpose_wrapper(vecs: &mut [__m128i; DEGREE]) {
+            transpose_vecs(vecs);
+        }
+
+        let mut matrix = [[0 as u32; DEGREE]; DEGREE];
+        for i in 0..DEGREE {
+            for j in 0..DEGREE {
+                matrix[i][j] = (i * DEGREE + j) as u32;
+            }
+        }
+
+        unsafe {
+            let mut vecs: [__m128i; DEGREE] = core::mem::transmute(matrix);
+            transpose_wrapper(&mut vecs);
+            matrix = core::mem::transmute(vecs);
+        }
+
+        for i in 0..DEGREE {
+            for j in 0..DEGREE {
+                // Reversed indexes from above.
+                assert_eq!(matrix[j][i], (i * DEGREE + j) as u32);
+            }
+        }
+    }
+
+    #[test]
+    fn test_compress() {
+        if !crate::platform::sse2_detected() {
+            return;
+        }
+        crate::test::test_compress_fn(compress_in_place, compress_xof);
+    }
+
+    #[test]
+    fn test_hash_many() {
+        if !crate::platform::sse2_detected() {
+            return;
+        }
+        crate::test::test_hash_many_fn(hash_many, hash_many);
+    }
+}

vendor/blake3/src/rust_sse41.rs

@@ -0,0 +1,766 @@
+#[cfg(target_arch = "x86")]
+use core::arch::x86::*;
+#[cfg(target_arch = "x86_64")]
+use core::arch::x86_64::*;
+
+use crate::{
+    counter_high, counter_low, CVBytes, CVWords, IncrementCounter, BLOCK_LEN, IV, MSG_SCHEDULE,
+    OUT_LEN,
+};
+use arrayref::{array_mut_ref, array_ref, mut_array_refs};
+
+pub const DEGREE: usize = 4;
+
+#[inline(always)]
+unsafe fn loadu(src: *const u8) -> __m128i {
+    // This is an unaligned load, so the pointer cast is allowed.
+    _mm_loadu_si128(src as *const __m128i)
+}
+
+#[inline(always)]
+unsafe fn storeu(src: __m128i, dest: *mut u8) {
+    // This is an unaligned store, so the pointer cast is allowed.
+    _mm_storeu_si128(dest as *mut __m128i, src)
+}
+
+#[inline(always)]
+unsafe fn add(a: __m128i, b: __m128i) -> __m128i {
+    _mm_add_epi32(a, b)
+}
+
+#[inline(always)]
+unsafe fn xor(a: __m128i, b: __m128i) -> __m128i {
+    _mm_xor_si128(a, b)
+}
+
+#[inline(always)]
+unsafe fn set1(x: u32) -> __m128i {
+    _mm_set1_epi32(x as i32)
+}
+
+#[inline(always)]
+unsafe fn set4(a: u32, b: u32, c: u32, d: u32) -> __m128i {
+    _mm_setr_epi32(a as i32, b as i32, c as i32, d as i32)
+}
+
+// These rotations are the "simple/shifts version". For the
+// "complicated/shuffles version", see
+// https://github.com/sneves/blake2-avx2/blob/b3723921f668df09ece52dcd225a36d4a4eea1d9/blake2s-common.h#L63-L66.
+// For a discussion of the tradeoffs, see
+// https://github.com/sneves/blake2-avx2/pull/5. Due to an LLVM bug
+// (https://bugs.llvm.org/show_bug.cgi?id=44379), this version performs better
+// on recent x86 chips.
+
+#[inline(always)]
+unsafe fn rot16(a: __m128i) -> __m128i {
+    _mm_or_si128(_mm_srli_epi32(a, 16), _mm_slli_epi32(a, 32 - 16))
+}
+
+#[inline(always)]
+unsafe fn rot12(a: __m128i) -> __m128i {
+    _mm_or_si128(_mm_srli_epi32(a, 12), _mm_slli_epi32(a, 32 - 12))
+}
+
+#[inline(always)]
+unsafe fn rot8(a: __m128i) -> __m128i {
+    _mm_or_si128(_mm_srli_epi32(a, 8), _mm_slli_epi32(a, 32 - 8))
+}
+
+#[inline(always)]
+unsafe fn rot7(a: __m128i) -> __m128i {
+    _mm_or_si128(_mm_srli_epi32(a, 7), _mm_slli_epi32(a, 32 - 7))
+}
+
+#[inline(always)]
+unsafe fn g1(
+    row0: &mut __m128i,
+    row1: &mut __m128i,
+    row2: &mut __m128i,
+    row3: &mut __m128i,
+    m: __m128i,
+) {
+    *row0 = add(add(*row0, m), *row1);
+    *row3 = xor(*row3, *row0);
+    *row3 = rot16(*row3);
+    *row2 = add(*row2, *row3);
+    *row1 = xor(*row1, *row2);
+    *row1 = rot12(*row1);
+}
+
+#[inline(always)]
+unsafe fn g2(
+    row0: &mut __m128i,
+    row1: &mut __m128i,
+    row2: &mut __m128i,
+    row3: &mut __m128i,
+    m: __m128i,
+) {
+    *row0 = add(add(*row0, m), *row1);
+    *row3 = xor(*row3, *row0);
+    *row3 = rot8(*row3);
+    *row2 = add(*row2, *row3);
+    *row1 = xor(*row1, *row2);
+    *row1 = rot7(*row1);
+}
+
+// Adapted from https://github.com/rust-lang-nursery/stdsimd/pull/479.
+macro_rules! _MM_SHUFFLE {
+    ($z:expr, $y:expr, $x:expr, $w:expr) => {
+        ($z << 6) | ($y << 4) | ($x << 2) | $w
+    };
+}
+
+macro_rules! shuffle2 {
+    ($a:expr, $b:expr, $c:expr) => {
+        _mm_castps_si128(_mm_shuffle_ps(
+            _mm_castsi128_ps($a),
+            _mm_castsi128_ps($b),
+            $c,
+        ))
+    };
+}
+
+// Note the optimization here of leaving row1 as the unrotated row, rather than
+// row0. All the message loads below are adjusted to compensate for this. See
+// discussion at https://github.com/sneves/blake2-avx2/pull/4
+#[inline(always)]
+unsafe fn diagonalize(row0: &mut __m128i, row2: &mut __m128i, row3: &mut __m128i) {
+    *row0 = _mm_shuffle_epi32(*row0, _MM_SHUFFLE!(2, 1, 0, 3));
+    *row3 = _mm_shuffle_epi32(*row3, _MM_SHUFFLE!(1, 0, 3, 2));
+    *row2 = _mm_shuffle_epi32(*row2, _MM_SHUFFLE!(0, 3, 2, 1));
+}
+
+#[inline(always)]
+unsafe fn undiagonalize(row0: &mut __m128i, row2: &mut __m128i, row3: &mut __m128i) {
+    *row0 = _mm_shuffle_epi32(*row0, _MM_SHUFFLE!(0, 3, 2, 1));
+    *row3 = _mm_shuffle_epi32(*row3, _MM_SHUFFLE!(1, 0, 3, 2));
+    *row2 = _mm_shuffle_epi32(*row2, _MM_SHUFFLE!(2, 1, 0, 3));
+}
+
+#[inline(always)]
+unsafe fn compress_pre(
+    cv: &CVWords,
+    block: &[u8; BLOCK_LEN],
+    block_len: u8,
+    counter: u64,
+    flags: u8,
+) -> [__m128i; 4] {
+    let row0 = &mut loadu(cv.as_ptr().add(0) as *const u8);
+    let row1 = &mut loadu(cv.as_ptr().add(4) as *const u8);
+    let row2 = &mut set4(IV[0], IV[1], IV[2], IV[3]);
+    let row3 = &mut set4(
+        counter_low(counter),
+        counter_high(counter),
+        block_len as u32,
+        flags as u32,
+    );
+
+    let mut m0 = loadu(block.as_ptr().add(0 * 4 * DEGREE));
+    let mut m1 = loadu(block.as_ptr().add(1 * 4 * DEGREE));
+    let mut m2 = loadu(block.as_ptr().add(2 * 4 * DEGREE));
+    let mut m3 = loadu(block.as_ptr().add(3 * 4 * DEGREE));
+
+    let mut t0;
+    let mut t1;
+    let mut t2;
+    let mut t3;
+    let mut tt;
+
+    // Round 1. The first round permutes the message words from the original
+    // input order, into the groups that get mixed in parallel.
+    t0 = shuffle2!(m0, m1, _MM_SHUFFLE!(2, 0, 2, 0)); //  6  4  2  0
+    g1(row0, row1, row2, row3, t0);
+    t1 = shuffle2!(m0, m1, _MM_SHUFFLE!(3, 1, 3, 1)); //  7  5  3  1
+    g2(row0, row1, row2, row3, t1);
+    diagonalize(row0, row2, row3);
+    t2 = shuffle2!(m2, m3, _MM_SHUFFLE!(2, 0, 2, 0)); // 14 12 10  8
+    t2 = _mm_shuffle_epi32(t2, _MM_SHUFFLE!(2, 1, 0, 3)); // 12 10  8 14
+    g1(row0, row1, row2, row3, t2);
+    t3 = shuffle2!(m2, m3, _MM_SHUFFLE!(3, 1, 3, 1)); // 15 13 11  9
+    t3 = _mm_shuffle_epi32(t3, _MM_SHUFFLE!(2, 1, 0, 3)); // 13 11  9 15
+    g2(row0, row1, row2, row3, t3);
+    undiagonalize(row0, row2, row3);
+    m0 = t0;
+    m1 = t1;
+    m2 = t2;
+    m3 = t3;
+
+    // Round 2. This round and all following rounds apply a fixed permutation
+    // to the message words from the round before.
+    t0 = shuffle2!(m0, m1, _MM_SHUFFLE!(3, 1, 1, 2));
+    t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE!(0, 3, 2, 1));
+    g1(row0, row1, row2, row3, t0);
+    t1 = shuffle2!(m2, m3, _MM_SHUFFLE!(3, 3, 2, 2));
+    tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE!(0, 0, 3, 3));
+    t1 = _mm_blend_epi16(tt, t1, 0xCC);
+    g2(row0, row1, row2, row3, t1);
+    diagonalize(row0, row2, row3);
+    t2 = _mm_unpacklo_epi64(m3, m1);
+    tt = _mm_blend_epi16(t2, m2, 0xC0);
+    t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE!(1, 3, 2, 0));
+    g1(row0, row1, row2, row3, t2);
+    t3 = _mm_unpackhi_epi32(m1, m3);
+    tt = _mm_unpacklo_epi32(m2, t3);
+    t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE!(0, 1, 3, 2));
+    g2(row0, row1, row2, row3, t3);
+    undiagonalize(row0, row2, row3);
+    m0 = t0;
+    m1 = t1;
+    m2 = t2;
+    m3 = t3;
+
+    // Round 3
+    t0 = shuffle2!(m0, m1, _MM_SHUFFLE!(3, 1, 1, 2));
+    t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE!(0, 3, 2, 1));
+    g1(row0, row1, row2, row3, t0);
+    t1 = shuffle2!(m2, m3, _MM_SHUFFLE!(3, 3, 2, 2));
+    tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE!(0, 0, 3, 3));
+    t1 = _mm_blend_epi16(tt, t1, 0xCC);
+    g2(row0, row1, row2, row3, t1);
+    diagonalize(row0, row2, row3);
+    t2 = _mm_unpacklo_epi64(m3, m1);
+    tt = _mm_blend_epi16(t2, m2, 0xC0);
+    t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE!(1, 3, 2, 0));
+    g1(row0, row1, row2, row3, t2);
+    t3 = _mm_unpackhi_epi32(m1, m3);
+    tt = _mm_unpacklo_epi32(m2, t3);
+    t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE!(0, 1, 3, 2));
+    g2(row0, row1, row2, row3, t3);
+    undiagonalize(row0, row2, row3);
+    m0 = t0;
+    m1 = t1;
+    m2 = t2;
+    m3 = t3;
+
+    // Round 4
+    t0 = shuffle2!(m0, m1, _MM_SHUFFLE!(3, 1, 1, 2));
+    t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE!(0, 3, 2, 1));
+    g1(row0, row1, row2, row3, t0);
+    t1 = shuffle2!(m2, m3, _MM_SHUFFLE!(3, 3, 2, 2));
+    tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE!(0, 0, 3, 3));
+    t1 = _mm_blend_epi16(tt, t1, 0xCC);
+    g2(row0, row1, row2, row3, t1);
+    diagonalize(row0, row2, row3);
+    t2 = _mm_unpacklo_epi64(m3, m1);
+    tt = _mm_blend_epi16(t2, m2, 0xC0);
+    t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE!(1, 3, 2, 0));
+    g1(row0, row1, row2, row3, t2);
+    t3 = _mm_unpackhi_epi32(m1, m3);
+    tt = _mm_unpacklo_epi32(m2, t3);
+    t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE!(0, 1, 3, 2));
+    g2(row0, row1, row2, row3, t3);
+    undiagonalize(row0, row2, row3);
+    m0 = t0;
+    m1 = t1;
+    m2 = t2;
+    m3 = t3;
+
+    // Round 5
+    t0 = shuffle2!(m0, m1, _MM_SHUFFLE!(3, 1, 1, 2));
+    t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE!(0, 3, 2, 1));
+    g1(row0, row1, row2, row3, t0);
+    t1 = shuffle2!(m2, m3, _MM_SHUFFLE!(3, 3, 2, 2));
+    tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE!(0, 0, 3, 3));
+    t1 = _mm_blend_epi16(tt, t1, 0xCC);
+    g2(row0, row1, row2, row3, t1);
+    diagonalize(row0, row2, row3);
+    t2 = _mm_unpacklo_epi64(m3, m1);
+    tt = _mm_blend_epi16(t2, m2, 0xC0);
+    t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE!(1, 3, 2, 0));
+    g1(row0, row1, row2, row3, t2);
+    t3 = _mm_unpackhi_epi32(m1, m3);
+    tt = _mm_unpacklo_epi32(m2, t3);
+    t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE!(0, 1, 3, 2));
+    g2(row0, row1, row2, row3, t3);
+    undiagonalize(row0, row2, row3);
+    m0 = t0;
+    m1 = t1;
+    m2 = t2;
+    m3 = t3;
+
+    // Round 6
+    t0 = shuffle2!(m0, m1, _MM_SHUFFLE!(3, 1, 1, 2));
+    t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE!(0, 3, 2, 1));
+    g1(row0, row1, row2, row3, t0);
+    t1 = shuffle2!(m2, m3, _MM_SHUFFLE!(3, 3, 2, 2));
+    tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE!(0, 0, 3, 3));
+    t1 = _mm_blend_epi16(tt, t1, 0xCC);
+    g2(row0, row1, row2, row3, t1);
+    diagonalize(row0, row2, row3);
+    t2 = _mm_unpacklo_epi64(m3, m1);
+    tt = _mm_blend_epi16(t2, m2, 0xC0);
+    t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE!(1, 3, 2, 0));
+    g1(row0, row1, row2, row3, t2);
+    t3 = _mm_unpackhi_epi32(m1, m3);
+    tt = _mm_unpacklo_epi32(m2, t3);
+    t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE!(0, 1, 3, 2));
+    g2(row0, row1, row2, row3, t3);
+    undiagonalize(row0, row2, row3);
+    m0 = t0;
+    m1 = t1;
+    m2 = t2;
+    m3 = t3;
+
+    // Round 7
+    t0 = shuffle2!(m0, m1, _MM_SHUFFLE!(3, 1, 1, 2));
+    t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE!(0, 3, 2, 1));
+    g1(row0, row1, row2, row3, t0);
+    t1 = shuffle2!(m2, m3, _MM_SHUFFLE!(3, 3, 2, 2));
+    tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE!(0, 0, 3, 3));
+    t1 = _mm_blend_epi16(tt, t1, 0xCC);
+    g2(row0, row1, row2, row3, t1);
+    diagonalize(row0, row2, row3);
+    t2 = _mm_unpacklo_epi64(m3, m1);
+    tt = _mm_blend_epi16(t2, m2, 0xC0);
+    t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE!(1, 3, 2, 0));
+    g1(row0, row1, row2, row3, t2);
+    t3 = _mm_unpackhi_epi32(m1, m3);
+    tt = _mm_unpacklo_epi32(m2, t3);
+    t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE!(0, 1, 3, 2));
+    g2(row0, row1, row2, row3, t3);
+    undiagonalize(row0, row2, row3);
+
+    [*row0, *row1, *row2, *row3]
+}
+
+#[target_feature(enable = "sse4.1")]
+pub unsafe fn compress_in_place(
+    cv: &mut CVWords,
+    block: &[u8; BLOCK_LEN],
+    block_len: u8,
+    counter: u64,
+    flags: u8,
+) {
+    let [row0, row1, row2, row3] = compress_pre(cv, block, block_len, counter, flags);
+    storeu(xor(row0, row2), cv.as_mut_ptr().add(0) as *mut u8);
+    storeu(xor(row1, row3), cv.as_mut_ptr().add(4) as *mut u8);
+}
+
+#[target_feature(enable = "sse4.1")]
+pub unsafe fn compress_xof(
+    cv: &CVWords,
+    block: &[u8; BLOCK_LEN],
+    block_len: u8,
+    counter: u64,
+    flags: u8,
+) -> [u8; 64] {
+    let [mut row0, mut row1, mut row2, mut row3] =
+        compress_pre(cv, block, block_len, counter, flags);
+    row0 = xor(row0, row2);
+    row1 = xor(row1, row3);
+    row2 = xor(row2, loadu(cv.as_ptr().add(0) as *const u8));
+    row3 = xor(row3, loadu(cv.as_ptr().add(4) as *const u8));
+    core::mem::transmute([row0, row1, row2, row3])
+}
+
+#[inline(always)]
+unsafe fn round(v: &mut [__m128i; 16], m: &[__m128i; 16], r: usize) {
+    v[0] = add(v[0], m[MSG_SCHEDULE[r][0] as usize]);
+    v[1] = add(v[1], m[MSG_SCHEDULE[r][2] as usize]);
+    v[2] = add(v[2], m[MSG_SCHEDULE[r][4] as usize]);
+    v[3] = add(v[3], m[MSG_SCHEDULE[r][6] as usize]);
+    v[0] = add(v[0], v[4]);
+    v[1] = add(v[1], v[5]);
+    v[2] = add(v[2], v[6]);
+    v[3] = add(v[3], v[7]);
+    v[12] = xor(v[12], v[0]);
+    v[13] = xor(v[13], v[1]);
+    v[14] = xor(v[14], v[2]);
+    v[15] = xor(v[15], v[3]);
+    v[12] = rot16(v[12]);
+    v[13] = rot16(v[13]);
+    v[14] = rot16(v[14]);
+    v[15] = rot16(v[15]);
+    v[8] = add(v[8], v[12]);
+    v[9] = add(v[9], v[13]);
+    v[10] = add(v[10], v[14]);
+    v[11] = add(v[11], v[15]);
+    v[4] = xor(v[4], v[8]);
+    v[5] = xor(v[5], v[9]);
+    v[6] = xor(v[6], v[10]);
+    v[7] = xor(v[7], v[11]);
+    v[4] = rot12(v[4]);
+    v[5] = rot12(v[5]);
+    v[6] = rot12(v[6]);
+    v[7] = rot12(v[7]);
+    v[0] = add(v[0], m[MSG_SCHEDULE[r][1] as usize]);
+    v[1] = add(v[1], m[MSG_SCHEDULE[r][3] as usize]);
+    v[2] = add(v[2], m[MSG_SCHEDULE[r][5] as usize]);
+    v[3] = add(v[3], m[MSG_SCHEDULE[r][7] as usize]);
+    v[0] = add(v[0], v[4]);
+    v[1] = add(v[1], v[5]);
+    v[2] = add(v[2], v[6]);
+    v[3] = add(v[3], v[7]);
+    v[12] = xor(v[12], v[0]);
+    v[13] = xor(v[13], v[1]);
+    v[14] = xor(v[14], v[2]);
+    v[15] = xor(v[15], v[3]);
+    v[12] = rot8(v[12]);
+    v[13] = rot8(v[13]);
+    v[14] = rot8(v[14]);
+    v[15] = rot8(v[15]);
+    v[8] = add(v[8], v[12]);
+    v[9] = add(v[9], v[13]);
+    v[10] = add(v[10], v[14]);
+    v[11] = add(v[11], v[15]);
+    v[4] = xor(v[4], v[8]);
+    v[5] = xor(v[5], v[9]);
+    v[6] = xor(v[6], v[10]);
+    v[7] = xor(v[7], v[11]);
+    v[4] = rot7(v[4]);
+    v[5] = rot7(v[5]);
+    v[6] = rot7(v[6]);
+    v[7] = rot7(v[7]);
+
+    v[0] = add(v[0], m[MSG_SCHEDULE[r][8] as usize]);
+    v[1] = add(v[1], m[MSG_SCHEDULE[r][10] as usize]);
+    v[2] = add(v[2], m[MSG_SCHEDULE[r][12] as usize]);
+    v[3] = add(v[3], m[MSG_SCHEDULE[r][14] as usize]);
+    v[0] = add(v[0], v[5]);
+    v[1] = add(v[1], v[6]);
+    v[2] = add(v[2], v[7]);
+    v[3] = add(v[3], v[4]);
+    v[15] = xor(v[15], v[0]);
+    v[12] = xor(v[12], v[1]);
+    v[13] = xor(v[13], v[2]);
+    v[14] = xor(v[14], v[3]);
+    v[15] = rot16(v[15]);
+    v[12] = rot16(v[12]);
+    v[13] = rot16(v[13]);
+    v[14] = rot16(v[14]);
+    v[10] = add(v[10], v[15]);
+    v[11] = add(v[11], v[12]);
+    v[8] = add(v[8], v[13]);
+    v[9] = add(v[9], v[14]);
+    v[5] = xor(v[5], v[10]);
+    v[6] = xor(v[6], v[11]);
+    v[7] = xor(v[7], v[8]);
+    v[4] = xor(v[4], v[9]);
+    v[5] = rot12(v[5]);
+    v[6] = rot12(v[6]);
+    v[7] = rot12(v[7]);
+    v[4] = rot12(v[4]);
+    v[0] = add(v[0], m[MSG_SCHEDULE[r][9] as usize]);
+    v[1] = add(v[1], m[MSG_SCHEDULE[r][11] as usize]);
+    v[2] = add(v[2], m[MSG_SCHEDULE[r][13] as usize]);
+    v[3] = add(v[3], m[MSG_SCHEDULE[r][15] as usize]);
+    v[0] = add(v[0], v[5]);
+    v[1] = add(v[1], v[6]);
+    v[2] = add(v[2], v[7]);
+    v[3] = add(v[3], v[4]);
+    v[15] = xor(v[15], v[0]);
+    v[12] = xor(v[12], v[1]);
+    v[13] = xor(v[13], v[2]);
+    v[14] = xor(v[14], v[3]);
+    v[15] = rot8(v[15]);
+    v[12] = rot8(v[12]);
+    v[13] = rot8(v[13]);
+    v[14] = rot8(v[14]);
+    v[10] = add(v[10], v[15]);
+    v[11] = add(v[11], v[12]);
+    v[8] = add(v[8], v[13]);
+    v[9] = add(v[9], v[14]);
+    v[5] = xor(v[5], v[10]);
+    v[6] = xor(v[6], v[11]);
+    v[7] = xor(v[7], v[8]);
+    v[4] = xor(v[4], v[9]);
+    v[5] = rot7(v[5]);
+    v[6] = rot7(v[6]);
+    v[7] = rot7(v[7]);
+    v[4] = rot7(v[4]);
+}
+
+#[inline(always)]
+unsafe fn transpose_vecs(vecs: &mut [__m128i; DEGREE]) {
+    // Interleave 32-bit lanes. The low unpack is lanes 00/11 and the high is
+    // 22/33. Note that this doesn't split the vector into two lanes, as the
+    // AVX2 counterparts do.
+    let ab_01 = _mm_unpacklo_epi32(vecs[0], vecs[1]);
+    let ab_23 = _mm_unpackhi_epi32(vecs[0], vecs[1]);
+    let cd_01 = _mm_unpacklo_epi32(vecs[2], vecs[3]);
+    let cd_23 = _mm_unpackhi_epi32(vecs[2], vecs[3]);
+
+    // Interleave 64-bit lanes.
+    let abcd_0 = _mm_unpacklo_epi64(ab_01, cd_01);
+    let abcd_1 = _mm_unpackhi_epi64(ab_01, cd_01);
+    let abcd_2 = _mm_unpacklo_epi64(ab_23, cd_23);
+    let abcd_3 = _mm_unpackhi_epi64(ab_23, cd_23);
+
+    vecs[0] = abcd_0;
+    vecs[1] = abcd_1;
+    vecs[2] = abcd_2;
+    vecs[3] = abcd_3;
+}
+
+#[inline(always)]
+unsafe fn transpose_msg_vecs(inputs: &[*const u8; DEGREE], block_offset: usize) -> [__m128i; 16] {
+    let mut vecs = [
+        loadu(inputs[0].add(block_offset + 0 * 4 * DEGREE)),
+        loadu(inputs[1].add(block_offset + 0 * 4 * DEGREE)),
+        loadu(inputs[2].add(block_offset + 0 * 4 * DEGREE)),
+        loadu(inputs[3].add(block_offset + 0 * 4 * DEGREE)),
+        loadu(inputs[0].add(block_offset + 1 * 4 * DEGREE)),
+        loadu(inputs[1].add(block_offset + 1 * 4 * DEGREE)),
+        loadu(inputs[2].add(block_offset + 1 * 4 * DEGREE)),
+        loadu(inputs[3].add(block_offset + 1 * 4 * DEGREE)),
+        loadu(inputs[0].add(block_offset + 2 * 4 * DEGREE)),
+        loadu(inputs[1].add(block_offset + 2 * 4 * DEGREE)),
+        loadu(inputs[2].add(block_offset + 2 * 4 * DEGREE)),
+        loadu(inputs[3].add(block_offset + 2 * 4 * DEGREE)),
+        loadu(inputs[0].add(block_offset + 3 * 4 * DEGREE)),
+        loadu(inputs[1].add(block_offset + 3 * 4 * DEGREE)),
+        loadu(inputs[2].add(block_offset + 3 * 4 * DEGREE)),
+        loadu(inputs[3].add(block_offset + 3 * 4 * DEGREE)),
+    ];
+    for i in 0..DEGREE {
+        _mm_prefetch(inputs[i].add(block_offset + 256) as *const i8, _MM_HINT_T0);
+    }
+    let squares = mut_array_refs!(&mut vecs, DEGREE, DEGREE, DEGREE, DEGREE);
+    transpose_vecs(squares.0);
+    transpose_vecs(squares.1);
+    transpose_vecs(squares.2);
+    transpose_vecs(squares.3);
+    vecs
+}
+
+#[inline(always)]
+unsafe fn load_counters(counter: u64, increment_counter: IncrementCounter) -> (__m128i, __m128i) {
+    let mask = if increment_counter.yes() { !0 } else { 0 };
+    (
+        set4(
+            counter_low(counter + (mask & 0)),
+            counter_low(counter + (mask & 1)),
+            counter_low(counter + (mask & 2)),
+            counter_low(counter + (mask & 3)),
+        ),
+        set4(
+            counter_high(counter + (mask & 0)),
+            counter_high(counter + (mask & 1)),
+            counter_high(counter + (mask & 2)),
+            counter_high(counter + (mask & 3)),
+        ),
+    )
+}
+
+#[target_feature(enable = "sse4.1")]
+pub unsafe fn hash4(
+    inputs: &[*const u8; DEGREE],
+    blocks: usize,
+    key: &CVWords,
+    counter: u64,
+    increment_counter: IncrementCounter,
+    flags: u8,
+    flags_start: u8,
+    flags_end: u8,
+    out: &mut [u8; DEGREE * OUT_LEN],
+) {
+    let mut h_vecs = [
+        set1(key[0]),
+        set1(key[1]),
+        set1(key[2]),
+        set1(key[3]),
+        set1(key[4]),
+        set1(key[5]),
+        set1(key[6]),
+        set1(key[7]),
+    ];
+    let (counter_low_vec, counter_high_vec) = load_counters(counter, increment_counter);
+    let mut block_flags = flags | flags_start;
+
+    for block in 0..blocks {
+        if block + 1 == blocks {
+            block_flags |= flags_end;
+        }
+        let block_len_vec = set1(BLOCK_LEN as u32); // full blocks only
+        let block_flags_vec = set1(block_flags as u32);
+        let msg_vecs = transpose_msg_vecs(inputs, block * BLOCK_LEN);
+
+        // The transposed compression function. Note that inlining this
+        // manually here improves compile times by a lot, compared to factoring
+        // it out into its own function and making it #[inline(always)]. Just
+        // guessing, it might have something to do with loop unrolling.
+        let mut v = [
+            h_vecs[0],
+            h_vecs[1],
+            h_vecs[2],
+            h_vecs[3],
+            h_vecs[4],
+            h_vecs[5],
+            h_vecs[6],
+            h_vecs[7],
+            set1(IV[0]),
+            set1(IV[1]),
+            set1(IV[2]),
+            set1(IV[3]),
+            counter_low_vec,
+            counter_high_vec,
+            block_len_vec,
+            block_flags_vec,
+        ];
+        round(&mut v, &msg_vecs, 0);
+        round(&mut v, &msg_vecs, 1);
+        round(&mut v, &msg_vecs, 2);
+        round(&mut v, &msg_vecs, 3);
+        round(&mut v, &msg_vecs, 4);
+        round(&mut v, &msg_vecs, 5);
+        round(&mut v, &msg_vecs, 6);
+        h_vecs[0] = xor(v[0], v[8]);
+        h_vecs[1] = xor(v[1], v[9]);
+        h_vecs[2] = xor(v[2], v[10]);
+        h_vecs[3] = xor(v[3], v[11]);
+        h_vecs[4] = xor(v[4], v[12]);
+        h_vecs[5] = xor(v[5], v[13]);
+        h_vecs[6] = xor(v[6], v[14]);
+        h_vecs[7] = xor(v[7], v[15]);
+
+        block_flags = flags;
+    }
+
+    let squares = mut_array_refs!(&mut h_vecs, DEGREE, DEGREE);
+    transpose_vecs(squares.0);
+    transpose_vecs(squares.1);
+    // The first four vecs now contain the first half of each output, and the
+    // second four vecs contain the second half of each output.
+    storeu(h_vecs[0], out.as_mut_ptr().add(0 * 4 * DEGREE));
+    storeu(h_vecs[4], out.as_mut_ptr().add(1 * 4 * DEGREE));
+    storeu(h_vecs[1], out.as_mut_ptr().add(2 * 4 * DEGREE));
+    storeu(h_vecs[5], out.as_mut_ptr().add(3 * 4 * DEGREE));
+    storeu(h_vecs[2], out.as_mut_ptr().add(4 * 4 * DEGREE));
+    storeu(h_vecs[6], out.as_mut_ptr().add(5 * 4 * DEGREE));
+    storeu(h_vecs[3], out.as_mut_ptr().add(6 * 4 * DEGREE));
+    storeu(h_vecs[7], out.as_mut_ptr().add(7 * 4 * DEGREE));
+}
+
+#[target_feature(enable = "sse4.1")]
+unsafe fn hash1<const N: usize>(
+    input: &[u8; N],
+    key: &CVWords,
+    counter: u64,
+    flags: u8,
+    flags_start: u8,
+    flags_end: u8,
+    out: &mut CVBytes,
+) {
+    debug_assert_eq!(N % BLOCK_LEN, 0, "uneven blocks");
+    let mut cv = *key;
+    let mut block_flags = flags | flags_start;
+    let mut slice = &input[..];
+    while slice.len() >= BLOCK_LEN {
+        if slice.len() == BLOCK_LEN {
+            block_flags |= flags_end;
+        }
+        compress_in_place(
+            &mut cv,
+            array_ref!(slice, 0, BLOCK_LEN),
+            BLOCK_LEN as u8,
+            counter,
+            block_flags,
+        );
+        block_flags = flags;
+        slice = &slice[BLOCK_LEN..];
+    }
+    *out = core::mem::transmute(cv); // x86 is little-endian
+}
+
+#[target_feature(enable = "sse4.1")]
+pub unsafe fn hash_many<const N: usize>(
+    mut inputs: &[&[u8; N]],
+    key: &CVWords,
+    mut counter: u64,
+    increment_counter: IncrementCounter,
+    flags: u8,
+    flags_start: u8,
+    flags_end: u8,
+    mut out: &mut [u8],
+) {
+    debug_assert!(out.len() >= inputs.len() * OUT_LEN, "out too short");
+    while inputs.len() >= DEGREE && out.len() >= DEGREE * OUT_LEN {
+        // Safe because the layout of arrays is guaranteed, and because the
+        // `blocks` count is determined statically from the argument type.
+        let input_ptrs: &[*const u8; DEGREE] = &*(inputs.as_ptr() as *const [*const u8; DEGREE]);
+        let blocks = N / BLOCK_LEN;
+        hash4(
+            input_ptrs,
+            blocks,
+            key,
+            counter,
+            increment_counter,
+            flags,
+            flags_start,
+            flags_end,
+            array_mut_ref!(out, 0, DEGREE * OUT_LEN),
+        );
+        if increment_counter.yes() {
+            counter += DEGREE as u64;
+        }
+        inputs = &inputs[DEGREE..];
+        out = &mut out[DEGREE * OUT_LEN..];
+    }
+    for (&input, output) in inputs.iter().zip(out.chunks_exact_mut(OUT_LEN)) {
+        hash1(
+            input,
+            key,
+            counter,
+            flags,
+            flags_start,
+            flags_end,
+            array_mut_ref!(output, 0, OUT_LEN),
+        );
+        if increment_counter.yes() {
+            counter += 1;
+        }
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn test_transpose() {
+        if !crate::platform::sse41_detected() {
+            return;
+        }
+
+        #[target_feature(enable = "sse4.1")]
+        unsafe fn transpose_wrapper(vecs: &mut [__m128i; DEGREE]) {
+            transpose_vecs(vecs);
+        }
+
+        let mut matrix = [[0 as u32; DEGREE]; DEGREE];
+        for i in 0..DEGREE {
+            for j in 0..DEGREE {
+                matrix[i][j] = (i * DEGREE + j) as u32;
+            }
+        }
+
+        unsafe {
+            let mut vecs: [__m128i; DEGREE] = core::mem::transmute(matrix);
+            transpose_wrapper(&mut vecs);
+            matrix = core::mem::transmute(vecs);
+        }
+
+        for i in 0..DEGREE {
+            for j in 0..DEGREE {
+                // Reversed indexes from above.
+                assert_eq!(matrix[j][i], (i * DEGREE + j) as u32);
+            }
+        }
+    }
+
+    #[test]
+    fn test_compress() {
+        if !crate::platform::sse41_detected() {
+            return;
+        }
+        crate::test::test_compress_fn(compress_in_place, compress_xof);
+    }
+
+    #[test]
+    fn test_hash_many() {
+        if !crate::platform::sse41_detected() {
+            return;
+        }
+        crate::test::test_hash_many_fn(hash_many, hash_many);
+    }
+}

vendor/blake3/src/traits.rs

@@ -0,0 +1,227 @@
+//! Implementations of commonly used traits like `Digest` and `Mac` from the
+//! [`digest`](https://crates.io/crates/digest) crate.
+
+pub use digest;
+
+use crate::{Hasher, OutputReader};
+use digest::crypto_common;
+use digest::generic_array::{typenum::U32, typenum::U64, GenericArray};
+
+impl digest::HashMarker for Hasher {}
+
+impl digest::Update for Hasher {
+    #[inline]
+    fn update(&mut self, data: &[u8]) {
+        self.update(data);
+    }
+}
+
+impl digest::Reset for Hasher {
+    #[inline]
+    fn reset(&mut self) {
+        self.reset(); // the inherent method
+    }
+}
+
+impl digest::OutputSizeUser for Hasher {
+    type OutputSize = U32;
+}
+
+impl digest::FixedOutput for Hasher {
+    #[inline]
+    fn finalize_into(self, out: &mut GenericArray<u8, Self::OutputSize>) {
+        out.copy_from_slice(self.finalize().as_bytes());
+    }
+}
+
+impl digest::FixedOutputReset for Hasher {
+    #[inline]
+    fn finalize_into_reset(&mut self, out: &mut GenericArray<u8, Self::OutputSize>) {
+        out.copy_from_slice(self.finalize().as_bytes());
+        self.reset();
+    }
+}
+
+impl digest::ExtendableOutput for Hasher {
+    type Reader = OutputReader;
+
+    #[inline]
+    fn finalize_xof(self) -> Self::Reader {
+        Hasher::finalize_xof(&self)
+    }
+}
+
+impl digest::ExtendableOutputReset for Hasher {
+    #[inline]
+    fn finalize_xof_reset(&mut self) -> Self::Reader {
+        let reader = Hasher::finalize_xof(self);
+        self.reset();
+        reader
+    }
+}
+
+impl digest::XofReader for OutputReader {
+    #[inline]
+    fn read(&mut self, buffer: &mut [u8]) {
+        self.fill(buffer);
+    }
+}
+
+impl crypto_common::KeySizeUser for Hasher {
+    type KeySize = U32;
+}
+
+impl crypto_common::BlockSizeUser for Hasher {
+    type BlockSize = U64;
+}
+
+impl digest::MacMarker for Hasher {}
+
+impl digest::KeyInit for Hasher {
+    #[inline]
+    fn new(key: &digest::Key<Self>) -> Self {
+        let key_bytes: [u8; 32] = (*key).into();
+        Hasher::new_keyed(&key_bytes)
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn test_digest_traits() {
+        // Inherent methods.
+        let mut hasher1 = crate::Hasher::new();
+        hasher1.update(b"foo");
+        hasher1.update(b"bar");
+        hasher1.update(b"baz");
+        let out1 = hasher1.finalize();
+        let mut xof1 = [0; 301];
+        hasher1.finalize_xof().fill(&mut xof1);
+        assert_eq!(out1.as_bytes(), &xof1[..32]);
+
+        // Trait implementations.
+        let mut hasher2: crate::Hasher = digest::Digest::new();
+        digest::Digest::update(&mut hasher2, b"xxx");
+        digest::Digest::reset(&mut hasher2);
+        digest::Digest::update(&mut hasher2, b"foo");
+        digest::Digest::update(&mut hasher2, b"bar");
+        digest::Digest::update(&mut hasher2, b"baz");
+        let out2 = digest::Digest::finalize(hasher2.clone());
+        let mut xof2 = [0; 301];
+        digest::XofReader::read(
+            &mut digest::ExtendableOutput::finalize_xof(hasher2.clone()),
+            &mut xof2,
+        );
+        assert_eq!(out1.as_bytes(), &out2[..]);
+        assert_eq!(xof1[..], xof2[..]);
+
+        // Again with the resetting variants.
+        let mut hasher3: crate::Hasher = digest::Digest::new();
+        digest::Digest::update(&mut hasher3, b"foobarbaz");
+        let mut out3 = [0; 32];
+        digest::FixedOutputReset::finalize_into_reset(
+            &mut hasher3,
+            GenericArray::from_mut_slice(&mut out3),
+        );
+        digest::Digest::update(&mut hasher3, b"foobarbaz");
+        let mut out4 = [0; 32];
+        digest::FixedOutputReset::finalize_into_reset(
+            &mut hasher3,
+            GenericArray::from_mut_slice(&mut out4),
+        );
+        digest::Digest::update(&mut hasher3, b"foobarbaz");
+        let mut xof3 = [0; 301];
+        digest::XofReader::read(
+            &mut digest::ExtendableOutputReset::finalize_xof_reset(&mut hasher3),
+            &mut xof3,
+        );
+        digest::Digest::update(&mut hasher3, b"foobarbaz");
+        let mut xof4 = [0; 301];
+        digest::XofReader::read(
+            &mut digest::ExtendableOutputReset::finalize_xof_reset(&mut hasher3),
+            &mut xof4,
+        );
+        assert_eq!(out1.as_bytes(), &out3[..]);
+        assert_eq!(out1.as_bytes(), &out4[..]);
+        assert_eq!(xof1[..], xof3[..]);
+        assert_eq!(xof1[..], xof4[..]);
+    }
+
+    #[test]
+    fn test_mac_trait() {
+        // Inherent methods.
+        let key = b"some super secret key bytes fooo";
+        let mut hasher1 = crate::Hasher::new_keyed(key);
+        hasher1.update(b"foo");
+        hasher1.update(b"bar");
+        hasher1.update(b"baz");
+        let out1 = hasher1.finalize();
+
+        // Trait implementation.
+        let generic_key = (*key).into();
+        let mut hasher2: crate::Hasher = digest::Mac::new(&generic_key);
+        digest::Mac::update(&mut hasher2, b"xxx");
+        digest::Mac::reset(&mut hasher2);
+        digest::Mac::update(&mut hasher2, b"foo");
+        digest::Mac::update(&mut hasher2, b"bar");
+        digest::Mac::update(&mut hasher2, b"baz");
+        let out2 = digest::Mac::finalize(hasher2);
+        assert_eq!(out1.as_bytes(), out2.into_bytes().as_slice());
+    }
+
+    fn expected_hmac_blake3(key: &[u8], input: &[u8]) -> [u8; 32] {
+        // See https://en.wikipedia.org/wiki/HMAC.
+        let key_hash;
+        let key_prime = if key.len() <= 64 {
+            key
+        } else {
+            key_hash = *crate::hash(key).as_bytes();
+            &key_hash
+        };
+        let mut ipad = [0x36; 64];
+        let mut opad = [0x5c; 64];
+        for i in 0..key_prime.len() {
+            ipad[i] ^= key_prime[i];
+            opad[i] ^= key_prime[i];
+        }
+        let mut inner_state = crate::Hasher::new();
+        inner_state.update(&ipad);
+        inner_state.update(input);
+        let mut outer_state = crate::Hasher::new();
+        outer_state.update(&opad);
+        outer_state.update(inner_state.finalize().as_bytes());
+        outer_state.finalize().into()
+    }
+
+    #[test]
+    fn test_hmac_compatibility() {
+        use hmac::{Mac, SimpleHmac};
+
+        // Test a short key.
+        let mut x = SimpleHmac::<Hasher>::new_from_slice(b"key").unwrap();
+        hmac::digest::Update::update(&mut x, b"data");
+        let output = x.finalize().into_bytes();
+        assert_ne!(output.len(), 0);
+        let expected = expected_hmac_blake3(b"key", b"data");
+        assert_eq!(expected, output.as_ref());
+
+        // Test a range of key and data lengths, particularly to exercise the long-key logic.
+        let mut input_bytes = [0; crate::test::TEST_CASES_MAX];
+        crate::test::paint_test_input(&mut input_bytes);
+        for &input_len in crate::test::TEST_CASES {
+            #[cfg(feature = "std")]
+            dbg!(input_len);
+            let input = &input_bytes[..input_len];
+
+            let mut x = SimpleHmac::<Hasher>::new_from_slice(input).unwrap();
+            hmac::digest::Update::update(&mut x, input);
+            let output = x.finalize().into_bytes();
+            assert_ne!(output.len(), 0);
+
+            let expected = expected_hmac_blake3(input, input);
+            assert_eq!(expected, output.as_ref());
+        }
+    }
+}

vendor/blake3/src/wasm32_simd.rs

@@ -0,0 +1,794 @@
+/*
+ * This code is based on rust_sse2.rs of the same distribution, and is subject to further improvements.
+ * Some comments are left intact even if their applicability is questioned.
+ *
+ * Performance measurements with a primitive benchmark with ~16Kb of data:
+ *
+ * | M1 native     | 11,610 ns |
+ * | M1 Wasm SIMD  | 13,355 ns |
+ * | M1 Wasm       | 22,037 ns |
+ * | x64 native    |  6,713 ns |
+ * | x64 Wasm SIMD | 11,985 ns |
+ * | x64 Wasm      | 25,978 ns |
+ *
+ * wasmtime v12.0.1 was used on both platforms.
+ */
+
+use core::arch::wasm32::*;
+
+use crate::{
+    counter_high, counter_low, CVBytes, CVWords, IncrementCounter, BLOCK_LEN, IV, MSG_SCHEDULE,
+    OUT_LEN,
+};
+use arrayref::{array_mut_ref, array_ref, mut_array_refs};
+
+pub const DEGREE: usize = 4;
+
+#[inline(always)]
+unsafe fn loadu(src: *const u8) -> v128 {
+    // This is an unaligned load, so the pointer cast is allowed.
+    v128_load(src as *const v128)
+}
+
+#[inline(always)]
+unsafe fn storeu(src: v128, dest: *mut u8) {
+    // This is an unaligned store, so the pointer cast is allowed.
+    v128_store(dest as *mut v128, src)
+}
+
+#[inline(always)]
+fn add(a: v128, b: v128) -> v128 {
+    i32x4_add(a, b)
+}
+
+#[inline(always)]
+fn xor(a: v128, b: v128) -> v128 {
+    v128_xor(a, b)
+}
+
+#[inline(always)]
+fn set1(x: u32) -> v128 {
+    i32x4_splat(x as i32)
+}
+
+#[inline(always)]
+fn set4(a: u32, b: u32, c: u32, d: u32) -> v128 {
+    i32x4(a as i32, b as i32, c as i32, d as i32)
+}
+
+// These rotations are the "simple/shifts version". For the
+// "complicated/shuffles version", see
+// https://github.com/sneves/blake2-avx2/blob/b3723921f668df09ece52dcd225a36d4a4eea1d9/blake2s-common.h#L63-L66.
+// For a discussion of the tradeoffs, see
+// https://github.com/sneves/blake2-avx2/pull/5. Due to an LLVM bug
+// (https://bugs.llvm.org/show_bug.cgi?id=44379), this version performs better
+// on recent x86 chips.
+#[inline(always)]
+fn rot16(a: v128) -> v128 {
+    v128_or(u32x4_shr(a, 16), u32x4_shl(a, 32 - 16))
+}
+
+#[inline(always)]
+fn rot12(a: v128) -> v128 {
+    v128_or(u32x4_shr(a, 12), u32x4_shl(a, 32 - 12))
+}
+
+#[inline(always)]
+fn rot8(a: v128) -> v128 {
+    v128_or(u32x4_shr(a, 8), u32x4_shl(a, 32 - 8))
+}
+
+#[inline(always)]
+fn rot7(a: v128) -> v128 {
+    v128_or(u32x4_shr(a, 7), u32x4_shl(a, 32 - 7))
+}
+
+#[inline(always)]
+fn g1(row0: &mut v128, row1: &mut v128, row2: &mut v128, row3: &mut v128, m: v128) {
+    *row0 = add(add(*row0, m), *row1);
+    *row3 = xor(*row3, *row0);
+    *row3 = rot16(*row3);
+    *row2 = add(*row2, *row3);
+    *row1 = xor(*row1, *row2);
+    *row1 = rot12(*row1);
+}
+
+#[inline(always)]
+fn g2(row0: &mut v128, row1: &mut v128, row2: &mut v128, row3: &mut v128, m: v128) {
+    *row0 = add(add(*row0, m), *row1);
+    *row3 = xor(*row3, *row0);
+    *row3 = rot8(*row3);
+    *row2 = add(*row2, *row3);
+    *row1 = xor(*row1, *row2);
+    *row1 = rot7(*row1);
+}
+
+// It could be a function, but artimetics in const generics is too limited yet.
+macro_rules! shuffle {
+    ($a: expr, $b: expr, $z:expr, $y:expr, $x:expr, $w:expr) => {
+        i32x4_shuffle::<{ $w }, { $x }, { $y + 4 }, { $z + 4 }>($a, $b)
+    };
+}
+
+#[inline(always)]
+fn unpacklo_epi64(a: v128, b: v128) -> v128 {
+    i64x2_shuffle::<0, 2>(a, b)
+}
+
+#[inline(always)]
+fn unpackhi_epi64(a: v128, b: v128) -> v128 {
+    i64x2_shuffle::<1, 3>(a, b)
+}
+
+#[inline(always)]
+fn unpacklo_epi32(a: v128, b: v128) -> v128 {
+    i32x4_shuffle::<0, 4, 1, 5>(a, b)
+}
+
+#[inline(always)]
+fn unpackhi_epi32(a: v128, b: v128) -> v128 {
+    i32x4_shuffle::<2, 6, 3, 7>(a, b)
+}
+
+#[inline(always)]
+fn shuffle_epi32<const I3: usize, const I2: usize, const I1: usize, const I0: usize>(
+    a: v128,
+) -> v128 {
+    // Please note that generic arguments in delcaration and imlementation are in
+    // different order.
+    // second arg is actually ignored.
+    i32x4_shuffle::<I0, I1, I2, I3>(a, a)
+}
+
+#[inline(always)]
+fn blend_epi16(a: v128, b: v128, imm8: i32) -> v128 {
+    // imm8 is always constant; it allows to implement this function with
+    // i16x8_shuffle.  However, it is marginally slower on x64.
+    let bits = i16x8(0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80);
+    let mut mask = i16x8_splat(imm8 as i16);
+    mask = v128_and(mask, bits);
+    mask = i16x8_eq(mask, bits);
+    // The swapped argument order is equivalent to mask negation.
+    v128_bitselect(b, a, mask)
+}
+
+// Note the optimization here of leaving row1 as the unrotated row, rather than
+// row0. All the message loads below are adjusted to compensate for this. See
+// discussion at https://github.com/sneves/blake2-avx2/pull/4
+#[inline(always)]
+fn diagonalize(row0: &mut v128, row2: &mut v128, row3: &mut v128) {
+    *row0 = shuffle_epi32::<2, 1, 0, 3>(*row0);
+    *row3 = shuffle_epi32::<1, 0, 3, 2>(*row3);
+    *row2 = shuffle_epi32::<0, 3, 2, 1>(*row2);
+}
+
+#[inline(always)]
+fn undiagonalize(row0: &mut v128, row2: &mut v128, row3: &mut v128) {
+    *row0 = shuffle_epi32::<0, 3, 2, 1>(*row0);
+    *row3 = shuffle_epi32::<1, 0, 3, 2>(*row3);
+    *row2 = shuffle_epi32::<2, 1, 0, 3>(*row2);
+}
+
+#[inline(always)]
+fn compress_pre(
+    cv: &CVWords,
+    block: &[u8; BLOCK_LEN],
+    block_len: u8,
+    counter: u64,
+    flags: u8,
+) -> [v128; 4] {
+    // safe because CVWords is [u32; 8]
+    let row0 = &mut unsafe { loadu(cv.as_ptr().add(0) as *const u8) };
+    let row1 = &mut unsafe { loadu(cv.as_ptr().add(4) as *const u8) };
+    let row2 = &mut set4(IV[0], IV[1], IV[2], IV[3]);
+    let row3 = &mut set4(
+        counter_low(counter),
+        counter_high(counter),
+        block_len as u32,
+        flags as u32,
+    );
+
+    // safe because block is &[u8; 64]
+    let mut m0 = unsafe { loadu(block.as_ptr().add(0 * 4 * DEGREE)) };
+    let mut m1 = unsafe { loadu(block.as_ptr().add(1 * 4 * DEGREE)) };
+    let mut m2 = unsafe { loadu(block.as_ptr().add(2 * 4 * DEGREE)) };
+    let mut m3 = unsafe { loadu(block.as_ptr().add(3 * 4 * DEGREE)) };
+
+    let mut t0;
+    let mut t1;
+    let mut t2;
+    let mut t3;
+    let mut tt;
+
+    // Round 1. The first round permutes the message words from the original
+    // input order, into the groups that get mixed in parallel.
+    t0 = shuffle!(m0, m1, 2, 0, 2, 0); //  6  4  2  0
+    g1(row0, row1, row2, row3, t0);
+    t1 = shuffle!(m0, m1, 3, 1, 3, 1); //  7  5  3  1
+    g2(row0, row1, row2, row3, t1);
+    diagonalize(row0, row2, row3);
+    t2 = shuffle!(m2, m3, 2, 0, 2, 0); // 14 12 10  8
+    t2 = shuffle_epi32::<2, 1, 0, 3>(t2); // 12 10  8 14
+    g1(row0, row1, row2, row3, t2);
+    t3 = shuffle!(m2, m3, 3, 1, 3, 1); // 15 13 11  9
+    t3 = shuffle_epi32::<2, 1, 0, 3>(t3); // 13 11  9 15
+    g2(row0, row1, row2, row3, t3);
+    undiagonalize(row0, row2, row3);
+    m0 = t0;
+    m1 = t1;
+    m2 = t2;
+    m3 = t3;
+
+    // Round 2. This round and all following rounds apply a fixed permutation
+    // to the message words from the round before.
+    t0 = shuffle!(m0, m1, 3, 1, 1, 2);
+    t0 = shuffle_epi32::<0, 3, 2, 1>(t0);
+    g1(row0, row1, row2, row3, t0);
+    t1 = shuffle!(m2, m3, 3, 3, 2, 2);
+    tt = shuffle_epi32::<0, 0, 3, 3>(m0);
+    t1 = blend_epi16(tt, t1, 0xCC);
+    g2(row0, row1, row2, row3, t1);
+    diagonalize(row0, row2, row3);
+    t2 = unpacklo_epi64(m3, m1);
+    tt = blend_epi16(t2, m2, 0xC0);
+    t2 = shuffle_epi32::<1, 3, 2, 0>(tt);
+    g1(row0, row1, row2, row3, t2);
+    t3 = unpackhi_epi32(m1, m3);
+    tt = unpacklo_epi32(m2, t3);
+    t3 = shuffle_epi32::<0, 1, 3, 2>(tt);
+    g2(row0, row1, row2, row3, t3);
+    undiagonalize(row0, row2, row3);
+    m0 = t0;
+    m1 = t1;
+    m2 = t2;
+    m3 = t3;
+
+    // Round 3
+    t0 = shuffle!(m0, m1, 3, 1, 1, 2);
+    t0 = shuffle_epi32::<0, 3, 2, 1>(t0);
+    g1(row0, row1, row2, row3, t0);
+    t1 = shuffle!(m2, m3, 3, 3, 2, 2);
+    tt = shuffle_epi32::<0, 0, 3, 3>(m0);
+    t1 = blend_epi16(tt, t1, 0xCC);
+    g2(row0, row1, row2, row3, t1);
+    diagonalize(row0, row2, row3);
+    t2 = unpacklo_epi64(m3, m1);
+    tt = blend_epi16(t2, m2, 0xC0);
+    t2 = shuffle_epi32::<1, 3, 2, 0>(tt);
+    g1(row0, row1, row2, row3, t2);
+    t3 = unpackhi_epi32(m1, m3);
+    tt = unpacklo_epi32(m2, t3);
+    t3 = shuffle_epi32::<0, 1, 3, 2>(tt);
+    g2(row0, row1, row2, row3, t3);
+    undiagonalize(row0, row2, row3);
+    m0 = t0;
+    m1 = t1;
+    m2 = t2;
+    m3 = t3;
+
+    // Round 4
+    t0 = shuffle!(m0, m1, 3, 1, 1, 2);
+    t0 = shuffle_epi32::<0, 3, 2, 1>(t0);
+    g1(row0, row1, row2, row3, t0);
+    t1 = shuffle!(m2, m3, 3, 3, 2, 2);
+    tt = shuffle_epi32::<0, 0, 3, 3>(m0);
+    t1 = blend_epi16(tt, t1, 0xCC);
+    g2(row0, row1, row2, row3, t1);
+    diagonalize(row0, row2, row3);
+    t2 = unpacklo_epi64(m3, m1);
+    tt = blend_epi16(t2, m2, 0xC0);
+    t2 = shuffle_epi32::<1, 3, 2, 0>(tt);
+    g1(row0, row1, row2, row3, t2);
+    t3 = unpackhi_epi32(m1, m3);
+    tt = unpacklo_epi32(m2, t3);
+    t3 = shuffle_epi32::<0, 1, 3, 2>(tt);
+    g2(row0, row1, row2, row3, t3);
+    undiagonalize(row0, row2, row3);
+    m0 = t0;
+    m1 = t1;
+    m2 = t2;
+    m3 = t3;
+
+    // Round 5
+    t0 = shuffle!(m0, m1, 3, 1, 1, 2);
+    t0 = shuffle_epi32::<0, 3, 2, 1>(t0);
+    g1(row0, row1, row2, row3, t0);
+    t1 = shuffle!(m2, m3, 3, 3, 2, 2);
+    tt = shuffle_epi32::<0, 0, 3, 3>(m0);
+    t1 = blend_epi16(tt, t1, 0xCC);
+    g2(row0, row1, row2, row3, t1);
+    diagonalize(row0, row2, row3);
+    t2 = unpacklo_epi64(m3, m1);
+    tt = blend_epi16(t2, m2, 0xC0);
+    t2 = shuffle_epi32::<1, 3, 2, 0>(tt);
+    g1(row0, row1, row2, row3, t2);
+    t3 = unpackhi_epi32(m1, m3);
+    tt = unpacklo_epi32(m2, t3);
+    t3 = shuffle_epi32::<0, 1, 3, 2>(tt);
+    g2(row0, row1, row2, row3, t3);
+    undiagonalize(row0, row2, row3);
+    m0 = t0;
+    m1 = t1;
+    m2 = t2;
+    m3 = t3;
+
+    // Round 6
+    t0 = shuffle!(m0, m1, 3, 1, 1, 2);
+    t0 = shuffle_epi32::<0, 3, 2, 1>(t0);
+    g1(row0, row1, row2, row3, t0);
+    t1 = shuffle!(m2, m3, 3, 3, 2, 2);
+    tt = shuffle_epi32::<0, 0, 3, 3>(m0);
+    t1 = blend_epi16(tt, t1, 0xCC);
+    g2(row0, row1, row2, row3, t1);
+    diagonalize(row0, row2, row3);
+    t2 = unpacklo_epi64(m3, m1);
+    tt = blend_epi16(t2, m2, 0xC0);
+    t2 = shuffle_epi32::<1, 3, 2, 0>(tt);
+    g1(row0, row1, row2, row3, t2);
+    t3 = unpackhi_epi32(m1, m3);
+    tt = unpacklo_epi32(m2, t3);
+    t3 = shuffle_epi32::<0, 1, 3, 2>(tt);
+    g2(row0, row1, row2, row3, t3);
+    undiagonalize(row0, row2, row3);
+    m0 = t0;
+    m1 = t1;
+    m2 = t2;
+    m3 = t3;
+
+    // Round 7
+    t0 = shuffle!(m0, m1, 3, 1, 1, 2);
+    t0 = shuffle_epi32::<0, 3, 2, 1>(t0);
+    g1(row0, row1, row2, row3, t0);
+    t1 = shuffle!(m2, m3, 3, 3, 2, 2);
+    tt = shuffle_epi32::<0, 0, 3, 3>(m0);
+    t1 = blend_epi16(tt, t1, 0xCC);
+    g2(row0, row1, row2, row3, t1);
+    diagonalize(row0, row2, row3);
+    t2 = unpacklo_epi64(m3, m1);
+    tt = blend_epi16(t2, m2, 0xC0);
+    t2 = shuffle_epi32::<1, 3, 2, 0>(tt);
+    g1(row0, row1, row2, row3, t2);
+    t3 = unpackhi_epi32(m1, m3);
+    tt = unpacklo_epi32(m2, t3);
+    t3 = shuffle_epi32::<0, 1, 3, 2>(tt);
+    g2(row0, row1, row2, row3, t3);
+    undiagonalize(row0, row2, row3);
+
+    [*row0, *row1, *row2, *row3]
+}
+
+#[target_feature(enable = "simd128")]
+pub fn compress_in_place(
+    cv: &mut CVWords,
+    block: &[u8; BLOCK_LEN],
+    block_len: u8,
+    counter: u64,
+    flags: u8,
+) {
+    let [row0, row1, row2, row3] = compress_pre(cv, block, block_len, counter, flags);
+    // it stores in reversed order...
+    // safe because CVWords is [u32; 8]
+    unsafe {
+        storeu(xor(row0, row2), cv.as_mut_ptr().add(0) as *mut u8);
+        storeu(xor(row1, row3), cv.as_mut_ptr().add(4) as *mut u8);
+    }
+}
+
+#[target_feature(enable = "simd128")]
+pub fn compress_xof(
+    cv: &CVWords,
+    block: &[u8; BLOCK_LEN],
+    block_len: u8,
+    counter: u64,
+    flags: u8,
+) -> [u8; 64] {
+    let [mut row0, mut row1, mut row2, mut row3] =
+        compress_pre(cv, block, block_len, counter, flags);
+    row0 = xor(row0, row2);
+    row1 = xor(row1, row3);
+    // safe because CVWords is [u32; 8]
+    row2 = xor(row2, unsafe { loadu(cv.as_ptr().add(0) as *const u8) });
+    row3 = xor(row3, unsafe { loadu(cv.as_ptr().add(4) as *const u8) });
+    // It seems to be architecture dependent, but works.
+    // safe because sizes match, and every state of u8 is valid.
+    unsafe { core::mem::transmute([row0, row1, row2, row3]) }
+}
+
+#[inline(always)]
+fn round(v: &mut [v128; 16], m: &[v128; 16], r: usize) {
+    v[0] = add(v[0], m[MSG_SCHEDULE[r][0] as usize]);
+    v[1] = add(v[1], m[MSG_SCHEDULE[r][2] as usize]);
+    v[2] = add(v[2], m[MSG_SCHEDULE[r][4] as usize]);
+    v[3] = add(v[3], m[MSG_SCHEDULE[r][6] as usize]);
+    v[0] = add(v[0], v[4]);
+    v[1] = add(v[1], v[5]);
+    v[2] = add(v[2], v[6]);
+    v[3] = add(v[3], v[7]);
+    v[12] = xor(v[12], v[0]);
+    v[13] = xor(v[13], v[1]);
+    v[14] = xor(v[14], v[2]);
+    v[15] = xor(v[15], v[3]);
+    v[12] = rot16(v[12]);
+    v[13] = rot16(v[13]);
+    v[14] = rot16(v[14]);
+    v[15] = rot16(v[15]);
+    v[8] = add(v[8], v[12]);
+    v[9] = add(v[9], v[13]);
+    v[10] = add(v[10], v[14]);
+    v[11] = add(v[11], v[15]);
+    v[4] = xor(v[4], v[8]);
+    v[5] = xor(v[5], v[9]);
+    v[6] = xor(v[6], v[10]);
+    v[7] = xor(v[7], v[11]);
+    v[4] = rot12(v[4]);
+    v[5] = rot12(v[5]);
+    v[6] = rot12(v[6]);
+    v[7] = rot12(v[7]);
+    v[0] = add(v[0], m[MSG_SCHEDULE[r][1] as usize]);
+    v[1] = add(v[1], m[MSG_SCHEDULE[r][3] as usize]);
+    v[2] = add(v[2], m[MSG_SCHEDULE[r][5] as usize]);
+    v[3] = add(v[3], m[MSG_SCHEDULE[r][7] as usize]);
+    v[0] = add(v[0], v[4]);
+    v[1] = add(v[1], v[5]);
+    v[2] = add(v[2], v[6]);
+    v[3] = add(v[3], v[7]);
+    v[12] = xor(v[12], v[0]);
+    v[13] = xor(v[13], v[1]);
+    v[14] = xor(v[14], v[2]);
+    v[15] = xor(v[15], v[3]);
+    v[12] = rot8(v[12]);
+    v[13] = rot8(v[13]);
+    v[14] = rot8(v[14]);
+    v[15] = rot8(v[15]);
+    v[8] = add(v[8], v[12]);
+    v[9] = add(v[9], v[13]);
+    v[10] = add(v[10], v[14]);
+    v[11] = add(v[11], v[15]);
+    v[4] = xor(v[4], v[8]);
+    v[5] = xor(v[5], v[9]);
+    v[6] = xor(v[6], v[10]);
+    v[7] = xor(v[7], v[11]);
+    v[4] = rot7(v[4]);
+    v[5] = rot7(v[5]);
+    v[6] = rot7(v[6]);
+    v[7] = rot7(v[7]);
+
+    v[0] = add(v[0], m[MSG_SCHEDULE[r][8] as usize]);
+    v[1] = add(v[1], m[MSG_SCHEDULE[r][10] as usize]);
+    v[2] = add(v[2], m[MSG_SCHEDULE[r][12] as usize]);
+    v[3] = add(v[3], m[MSG_SCHEDULE[r][14] as usize]);
+    v[0] = add(v[0], v[5]);
+    v[1] = add(v[1], v[6]);
+    v[2] = add(v[2], v[7]);
+    v[3] = add(v[3], v[4]);
+    v[15] = xor(v[15], v[0]);
+    v[12] = xor(v[12], v[1]);
+    v[13] = xor(v[13], v[2]);
+    v[14] = xor(v[14], v[3]);
+    v[15] = rot16(v[15]);
+    v[12] = rot16(v[12]);
+    v[13] = rot16(v[13]);
+    v[14] = rot16(v[14]);
+    v[10] = add(v[10], v[15]);
+    v[11] = add(v[11], v[12]);
+    v[8] = add(v[8], v[13]);
+    v[9] = add(v[9], v[14]);
+    v[5] = xor(v[5], v[10]);
+    v[6] = xor(v[6], v[11]);
+    v[7] = xor(v[7], v[8]);
+    v[4] = xor(v[4], v[9]);
+    v[5] = rot12(v[5]);
+    v[6] = rot12(v[6]);
+    v[7] = rot12(v[7]);
+    v[4] = rot12(v[4]);
+    v[0] = add(v[0], m[MSG_SCHEDULE[r][9] as usize]);
+    v[1] = add(v[1], m[MSG_SCHEDULE[r][11] as usize]);
+    v[2] = add(v[2], m[MSG_SCHEDULE[r][13] as usize]);
+    v[3] = add(v[3], m[MSG_SCHEDULE[r][15] as usize]);
+    v[0] = add(v[0], v[5]);
+    v[1] = add(v[1], v[6]);
+    v[2] = add(v[2], v[7]);
+    v[3] = add(v[3], v[4]);
+    v[15] = xor(v[15], v[0]);
+    v[12] = xor(v[12], v[1]);
+    v[13] = xor(v[13], v[2]);
+    v[14] = xor(v[14], v[3]);
+    v[15] = rot8(v[15]);
+    v[12] = rot8(v[12]);
+    v[13] = rot8(v[13]);
+    v[14] = rot8(v[14]);
+    v[10] = add(v[10], v[15]);
+    v[11] = add(v[11], v[12]);
+    v[8] = add(v[8], v[13]);
+    v[9] = add(v[9], v[14]);
+    v[5] = xor(v[5], v[10]);
+    v[6] = xor(v[6], v[11]);
+    v[7] = xor(v[7], v[8]);
+    v[4] = xor(v[4], v[9]);
+    v[5] = rot7(v[5]);
+    v[6] = rot7(v[6]);
+    v[7] = rot7(v[7]);
+    v[4] = rot7(v[4]);
+}
+
+#[inline(always)]
+fn transpose_vecs(vecs: &mut [v128; DEGREE]) {
+    // Interleave 32-bit lanes. The low unpack is lanes 00/11 and the high is
+    // 22/33. Note that this doesn't split the vector into two lanes, as the
+    // AVX2 counterparts do.
+    let ab_01 = unpacklo_epi32(vecs[0], vecs[1]);
+    let ab_23 = unpackhi_epi32(vecs[0], vecs[1]);
+    let cd_01 = unpacklo_epi32(vecs[2], vecs[3]);
+    let cd_23 = unpackhi_epi32(vecs[2], vecs[3]);
+
+    // Interleave 64-bit lanes.
+    let abcd_0 = unpacklo_epi64(ab_01, cd_01);
+    let abcd_1 = unpackhi_epi64(ab_01, cd_01);
+    let abcd_2 = unpacklo_epi64(ab_23, cd_23);
+    let abcd_3 = unpackhi_epi64(ab_23, cd_23);
+
+    vecs[0] = abcd_0;
+    vecs[1] = abcd_1;
+    vecs[2] = abcd_2;
+    vecs[3] = abcd_3;
+}
+
+#[inline(always)]
+unsafe fn transpose_msg_vecs(inputs: &[*const u8; DEGREE], block_offset: usize) -> [v128; 16] {
+    let mut vecs = [
+        loadu(inputs[0].add(block_offset + 0 * 4 * DEGREE)),
+        loadu(inputs[1].add(block_offset + 0 * 4 * DEGREE)),
+        loadu(inputs[2].add(block_offset + 0 * 4 * DEGREE)),
+        loadu(inputs[3].add(block_offset + 0 * 4 * DEGREE)),
+        loadu(inputs[0].add(block_offset + 1 * 4 * DEGREE)),
+        loadu(inputs[1].add(block_offset + 1 * 4 * DEGREE)),
+        loadu(inputs[2].add(block_offset + 1 * 4 * DEGREE)),
+        loadu(inputs[3].add(block_offset + 1 * 4 * DEGREE)),
+        loadu(inputs[0].add(block_offset + 2 * 4 * DEGREE)),
+        loadu(inputs[1].add(block_offset + 2 * 4 * DEGREE)),
+        loadu(inputs[2].add(block_offset + 2 * 4 * DEGREE)),
+        loadu(inputs[3].add(block_offset + 2 * 4 * DEGREE)),
+        loadu(inputs[0].add(block_offset + 3 * 4 * DEGREE)),
+        loadu(inputs[1].add(block_offset + 3 * 4 * DEGREE)),
+        loadu(inputs[2].add(block_offset + 3 * 4 * DEGREE)),
+        loadu(inputs[3].add(block_offset + 3 * 4 * DEGREE)),
+    ];
+    let squares = mut_array_refs!(&mut vecs, DEGREE, DEGREE, DEGREE, DEGREE);
+    transpose_vecs(squares.0);
+    transpose_vecs(squares.1);
+    transpose_vecs(squares.2);
+    transpose_vecs(squares.3);
+    vecs
+}
+
+#[inline(always)]
+fn load_counters(counter: u64, increment_counter: IncrementCounter) -> (v128, v128) {
+    let mask = if increment_counter.yes() { !0 } else { 0 };
+    (
+        set4(
+            counter_low(counter + (mask & 0)),
+            counter_low(counter + (mask & 1)),
+            counter_low(counter + (mask & 2)),
+            counter_low(counter + (mask & 3)),
+        ),
+        set4(
+            counter_high(counter + (mask & 0)),
+            counter_high(counter + (mask & 1)),
+            counter_high(counter + (mask & 2)),
+            counter_high(counter + (mask & 3)),
+        ),
+    )
+}
+
+#[target_feature(enable = "simd128")]
+pub unsafe fn hash4(
+    inputs: &[*const u8; DEGREE],
+    blocks: usize,
+    key: &CVWords,
+    counter: u64,
+    increment_counter: IncrementCounter,
+    flags: u8,
+    flags_start: u8,
+    flags_end: u8,
+    out: &mut [u8; DEGREE * OUT_LEN],
+) {
+    let mut h_vecs = [
+        set1(key[0]),
+        set1(key[1]),
+        set1(key[2]),
+        set1(key[3]),
+        set1(key[4]),
+        set1(key[5]),
+        set1(key[6]),
+        set1(key[7]),
+    ];
+    let (counter_low_vec, counter_high_vec) = load_counters(counter, increment_counter);
+    let mut block_flags = flags | flags_start;
+
+    for block in 0..blocks {
+        if block + 1 == blocks {
+            block_flags |= flags_end;
+        }
+        let block_len_vec = set1(BLOCK_LEN as u32); // full blocks only
+        let block_flags_vec = set1(block_flags as u32);
+        let msg_vecs = transpose_msg_vecs(inputs, block * BLOCK_LEN);
+
+        // The transposed compression function. Note that inlining this
+        // manually here improves compile times by a lot, compared to factoring
+        // it out into its own function and making it #[inline(always)]. Just
+        // guessing, it might have something to do with loop unrolling.
+        let mut v = [
+            h_vecs[0],
+            h_vecs[1],
+            h_vecs[2],
+            h_vecs[3],
+            h_vecs[4],
+            h_vecs[5],
+            h_vecs[6],
+            h_vecs[7],
+            set1(IV[0]),
+            set1(IV[1]),
+            set1(IV[2]),
+            set1(IV[3]),
+            counter_low_vec,
+            counter_high_vec,
+            block_len_vec,
+            block_flags_vec,
+        ];
+        round(&mut v, &msg_vecs, 0);
+        round(&mut v, &msg_vecs, 1);
+        round(&mut v, &msg_vecs, 2);
+        round(&mut v, &msg_vecs, 3);
+        round(&mut v, &msg_vecs, 4);
+        round(&mut v, &msg_vecs, 5);
+        round(&mut v, &msg_vecs, 6);
+        h_vecs[0] = xor(v[0], v[8]);
+        h_vecs[1] = xor(v[1], v[9]);
+        h_vecs[2] = xor(v[2], v[10]);
+        h_vecs[3] = xor(v[3], v[11]);
+        h_vecs[4] = xor(v[4], v[12]);
+        h_vecs[5] = xor(v[5], v[13]);
+        h_vecs[6] = xor(v[6], v[14]);
+        h_vecs[7] = xor(v[7], v[15]);
+
+        block_flags = flags;
+    }
+
+    let squares = mut_array_refs!(&mut h_vecs, DEGREE, DEGREE);
+    transpose_vecs(squares.0);
+    transpose_vecs(squares.1);
+    // The first four vecs now contain the first half of each output, and the
+    // second four vecs contain the second half of each output.
+    storeu(h_vecs[0], out.as_mut_ptr().add(0 * 4 * DEGREE));
+    storeu(h_vecs[4], out.as_mut_ptr().add(1 * 4 * DEGREE));
+    storeu(h_vecs[1], out.as_mut_ptr().add(2 * 4 * DEGREE));
+    storeu(h_vecs[5], out.as_mut_ptr().add(3 * 4 * DEGREE));
+    storeu(h_vecs[2], out.as_mut_ptr().add(4 * 4 * DEGREE));
+    storeu(h_vecs[6], out.as_mut_ptr().add(5 * 4 * DEGREE));
+    storeu(h_vecs[3], out.as_mut_ptr().add(6 * 4 * DEGREE));
+    storeu(h_vecs[7], out.as_mut_ptr().add(7 * 4 * DEGREE));
+}
+
+#[target_feature(enable = "simd128")]
+unsafe fn hash1<const N: usize>(
+    input: &[u8; N],
+    key: &CVWords,
+    counter: u64,
+    flags: u8,
+    flags_start: u8,
+    flags_end: u8,
+    out: &mut CVBytes,
+) {
+    debug_assert_eq!(N % BLOCK_LEN, 0, "uneven blocks");
+    let mut cv = *key;
+    let mut block_flags = flags | flags_start;
+    let mut slice = &input[..];
+    while slice.len() >= BLOCK_LEN {
+        if slice.len() == BLOCK_LEN {
+            block_flags |= flags_end;
+        }
+        compress_in_place(
+            &mut cv,
+            array_ref!(slice, 0, BLOCK_LEN),
+            BLOCK_LEN as u8,
+            counter,
+            block_flags,
+        );
+        block_flags = flags;
+        slice = &slice[BLOCK_LEN..];
+    }
+    *out = core::mem::transmute(cv);
+}
+
+#[target_feature(enable = "simd128")]
+pub unsafe fn hash_many<const N: usize>(
+    mut inputs: &[&[u8; N]],
+    key: &CVWords,
+    mut counter: u64,
+    increment_counter: IncrementCounter,
+    flags: u8,
+    flags_start: u8,
+    flags_end: u8,
+    mut out: &mut [u8],
+) {
+    debug_assert!(out.len() >= inputs.len() * OUT_LEN, "out too short");
+    while inputs.len() >= DEGREE && out.len() >= DEGREE * OUT_LEN {
+        // Safe because the layout of arrays is guaranteed, and because the
+        // `blocks` count is determined statically from the argument type.
+        let input_ptrs: &[*const u8; DEGREE] = &*(inputs.as_ptr() as *const [*const u8; DEGREE]);
+        let blocks = N / BLOCK_LEN;
+        hash4(
+            input_ptrs,
+            blocks,
+            key,
+            counter,
+            increment_counter,
+            flags,
+            flags_start,
+            flags_end,
+            array_mut_ref!(out, 0, DEGREE * OUT_LEN),
+        );
+        if increment_counter.yes() {
+            counter += DEGREE as u64;
+        }
+        inputs = &inputs[DEGREE..];
+        out = &mut out[DEGREE * OUT_LEN..];
+    }
+    for (&input, output) in inputs.iter().zip(out.chunks_exact_mut(OUT_LEN)) {
+        hash1(
+            input,
+            key,
+            counter,
+            flags,
+            flags_start,
+            flags_end,
+            array_mut_ref!(output, 0, OUT_LEN),
+        );
+        if increment_counter.yes() {
+            counter += 1;
+        }
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn test_transpose() {
+        #[target_feature(enable = "simd128")]
+        fn transpose_wrapper(vecs: &mut [v128; DEGREE]) {
+            transpose_vecs(vecs);
+        }
+
+        let mut matrix = [[0 as u32; DEGREE]; DEGREE];
+        for i in 0..DEGREE {
+            for j in 0..DEGREE {
+                matrix[i][j] = (i * DEGREE + j) as u32;
+            }
+        }
+
+        unsafe {
+            let mut vecs: [v128; DEGREE] = core::mem::transmute(matrix);
+            transpose_wrapper(&mut vecs);
+            matrix = core::mem::transmute(vecs);
+        }
+
+        for i in 0..DEGREE {
+            for j in 0..DEGREE {
+                // Reversed indexes from above.
+                assert_eq!(matrix[j][i], (i * DEGREE + j) as u32);
+            }
+        }
+    }
+
+    #[test]
+    fn test_compress() {
+        crate::test::test_compress_fn(compress_in_place, compress_xof);
+    }
+
+    #[test]
+    fn test_hash_many() {
+        crate::test::test_hash_many_fn(hash_many, hash_many);
+    }
+}