path: root/src/kernel2.rs

use crate::{BLOCK_LEN, CHUNK_LEN, IV};
use core::arch::x86_64::*;
use core::arch::{asm, global_asm};
use core::mem;

global_asm!(
    // --------------------------------------------------------------------------------------------
    // blake3_avx512_kernel2_16
    //
    // Inputs:
    //     zmm0-zmm7: transposed input CV
    //    zmm8-zmm11: broadcasted IV0-IV3 words
    //         zmm12: transposed lower order counter words
    //         zmm13: transposed higher order counter words
    //         zmm14: broadcasted block sizes (always 64)
    //         zmm15: broadcasted flag words
    //   zmm16-zmm31: transposed message vectors
    //
    // Outputs:
    //   zmm0-zmm16: transposed output state minus the feed-forward
    //
    // "Minus the feed-forward" means that the final xors have been done in the lower half of the
    // state but not the upper half. XOF callers need to do the upper half xors / CV-feed-forward
    // themselves.
    // --------------------------------------------------------------------------------------------
    "blake3_avx512_kernel2_16:",
    // round 1
    "vpaddd  zmm0, zmm0, zmm16",
    "vpaddd  zmm1, zmm1, zmm18",
    "vpaddd  zmm2, zmm2, zmm20",
    "vpaddd  zmm3, zmm3, zmm22",
    "vpaddd  zmm0, zmm0, zmm4",
    "vpaddd  zmm1, zmm1, zmm5",
    "vpaddd  zmm2, zmm2, zmm6",
    "vpaddd  zmm3, zmm3, zmm7",
    "vpxord  zmm12, zmm12, zmm0",
    "vpxord  zmm13, zmm13, zmm1",
    "vpxord  zmm14, zmm14, zmm2",
    "vpxord  zmm15, zmm15, zmm3",
    "vprord  zmm12, zmm12, 16",
    "vprord  zmm13, zmm13, 16",
    "vprord  zmm14, zmm14, 16",
    "vprord  zmm15, zmm15, 16",
    "vpaddd  zmm8, zmm8, zmm12",
    "vpaddd  zmm9, zmm9, zmm13",
    "vpaddd  zmm10, zmm10, zmm14",
    "vpaddd  zmm11, zmm11, zmm15",
    "vpxord  zmm4, zmm4, zmm8",
    "vpxord  zmm5, zmm5, zmm9",
    "vpxord  zmm6, zmm6, zmm10",
    "vpxord  zmm7, zmm7, zmm11",
    "vprord  zmm4, zmm4, 12",
    "vprord  zmm5, zmm5, 12",
    "vprord  zmm6, zmm6, 12",
    "vprord  zmm7, zmm7, 12",
    "vpaddd  zmm0, zmm0, zmm17",
    "vpaddd  zmm1, zmm1, zmm19",
    "vpaddd  zmm2, zmm2, zmm21",
    "vpaddd  zmm3, zmm3, zmm23",
    "vpaddd  zmm0, zmm0, zmm4",
    "vpaddd  zmm1, zmm1, zmm5",
    "vpaddd  zmm2, zmm2, zmm6",
    "vpaddd  zmm3, zmm3, zmm7",
    "vpxord  zmm12, zmm12, zmm0",
    "vpxord  zmm13, zmm13, zmm1",
    "vpxord  zmm14, zmm14, zmm2",
    "vpxord  zmm15, zmm15, zmm3",
    "vprord  zmm12, zmm12, 8",
    "vprord  zmm13, zmm13, 8",
    "vprord  zmm14, zmm14, 8",
    "vprord  zmm15, zmm15, 8",
    "vpaddd  zmm8, zmm8, zmm12",
    "vpaddd  zmm9, zmm9, zmm13",
    "vpaddd  zmm10, zmm10, zmm14",
    "vpaddd  zmm11, zmm11, zmm15",
    "vpxord  zmm4, zmm4, zmm8",
    "vpxord  zmm5, zmm5, zmm9",
    "vpxord  zmm6, zmm6, zmm10",
    "vpxord  zmm7, zmm7, zmm11",
    "vprord  zmm4, zmm4, 7",
    "vprord  zmm5, zmm5, 7",
    "vprord  zmm6, zmm6, 7",
    "vprord  zmm7, zmm7, 7",
    "vpaddd  zmm0, zmm0, zmm24",
    "vpaddd  zmm1, zmm1, zmm26",
    "vpaddd  zmm2, zmm2, zmm28",
    "vpaddd  zmm3, zmm3, zmm30",
    "vpaddd  zmm0, zmm0, zmm5",
    "vpaddd  zmm1, zmm1, zmm6",
    "vpaddd  zmm2, zmm2, zmm7",
    "vpaddd  zmm3, zmm3, zmm4",
    "vpxord  zmm15, zmm15, zmm0",
    "vpxord  zmm12, zmm12, zmm1",
    "vpxord  zmm13, zmm13, zmm2",
    "vpxord  zmm14, zmm14, zmm3",
    "vprord  zmm15, zmm15, 16",
    "vprord  zmm12, zmm12, 16",
    "vprord  zmm13, zmm13, 16",
    "vprord  zmm14, zmm14, 16",
    "vpaddd  zmm10, zmm10, zmm15",
    "vpaddd  zmm11, zmm11, zmm12",
    "vpaddd  zmm8, zmm8, zmm13",
    "vpaddd  zmm9, zmm9, zmm14",
    "vpxord  zmm5, zmm5, zmm10",
    "vpxord  zmm6, zmm6, zmm11",
    "vpxord  zmm7, zmm7, zmm8",
    "vpxord  zmm4, zmm4, zmm9",
    "vprord  zmm5, zmm5, 12",
    "vprord  zmm6, zmm6, 12",
    "vprord  zmm7, zmm7, 12",
    "vprord  zmm4, zmm4, 12",
    "vpaddd  zmm0, zmm0, zmm25",
    "vpaddd  zmm1, zmm1, zmm27",
    "vpaddd  zmm2, zmm2, zmm29",
    "vpaddd  zmm3, zmm3, zmm31",
    "vpaddd  zmm0, zmm0, zmm5",
    "vpaddd  zmm1, zmm1, zmm6",
    "vpaddd  zmm2, zmm2, zmm7",
    "vpaddd  zmm3, zmm3, zmm4",
    "vpxord  zmm15, zmm15, zmm0",
    "vpxord  zmm12, zmm12, zmm1",
    "vpxord  zmm13, zmm13, zmm2",
    "vpxord  zmm14, zmm14, zmm3",
    "vprord  zmm15, zmm15, 8",
    "vprord  zmm12, zmm12, 8",
    "vprord  zmm13, zmm13, 8",
    "vprord  zmm14, zmm14, 8",
    "vpaddd  zmm10, zmm10, zmm15",
    "vpaddd  zmm11, zmm11, zmm12",
    "vpaddd  zmm8, zmm8, zmm13",
    "vpaddd  zmm9, zmm9, zmm14",
    "vpxord  zmm5, zmm5, zmm10",
    "vpxord  zmm6, zmm6, zmm11",
    "vpxord  zmm7, zmm7, zmm8",
    "vpxord  zmm4, zmm4, zmm9",
    "vprord  zmm5, zmm5, 7",
    "vprord  zmm6, zmm6, 7",
    "vprord  zmm7, zmm7, 7",
    "vprord  zmm4, zmm4, 7",
    // round 2
    "vpaddd  zmm0, zmm0, zmm18",
    "vpaddd  zmm1, zmm1, zmm19",
    "vpaddd  zmm2, zmm2, zmm23",
    "vpaddd  zmm3, zmm3, zmm20",
    "vpaddd  zmm0, zmm0, zmm4",
    "vpaddd  zmm1, zmm1, zmm5",
    "vpaddd  zmm2, zmm2, zmm6",
    "vpaddd  zmm3, zmm3, zmm7",
    "vpxord  zmm12, zmm12, zmm0",
    "vpxord  zmm13, zmm13, zmm1",
    "vpxord  zmm14, zmm14, zmm2",
    "vpxord  zmm15, zmm15, zmm3",
    "vprord  zmm12, zmm12, 16",
    "vprord  zmm13, zmm13, 16",
    "vprord  zmm14, zmm14, 16",
    "vprord  zmm15, zmm15, 16",
    "vpaddd  zmm8, zmm8, zmm12",
    "vpaddd  zmm9, zmm9, zmm13",
    "vpaddd  zmm10, zmm10, zmm14",
    "vpaddd  zmm11, zmm11, zmm15",
    "vpxord  zmm4, zmm4, zmm8",
    "vpxord  zmm5, zmm5, zmm9",
    "vpxord  zmm6, zmm6, zmm10",
    "vpxord  zmm7, zmm7, zmm11",
    "vprord  zmm4, zmm4, 12",
    "vprord  zmm5, zmm5, 12",
    "vprord  zmm6, zmm6, 12",
    "vprord  zmm7, zmm7, 12",
    "vpaddd  zmm0, zmm0, zmm22",
    "vpaddd  zmm1, zmm1, zmm26",
    "vpaddd  zmm2, zmm2, zmm16",
    "vpaddd  zmm3, zmm3, zmm29",
    "vpaddd  zmm0, zmm0, zmm4",
    "vpaddd  zmm1, zmm1, zmm5",
    "vpaddd  zmm2, zmm2, zmm6",
    "vpaddd  zmm3, zmm3, zmm7",
    "vpxord  zmm12, zmm12, zmm0",
    "vpxord  zmm13, zmm13, zmm1",
    "vpxord  zmm14, zmm14, zmm2",
    "vpxord  zmm15, zmm15, zmm3",
    "vprord  zmm12, zmm12, 8",
    "vprord  zmm13, zmm13, 8",
    "vprord  zmm14, zmm14, 8",
    "vprord  zmm15, zmm15, 8",
    "vpaddd  zmm8, zmm8, zmm12",
    "vpaddd  zmm9, zmm9, zmm13",
    "vpaddd  zmm10, zmm10, zmm14",
    "vpaddd  zmm11, zmm11, zmm15",
    "vpxord  zmm4, zmm4, zmm8",
    "vpxord  zmm5, zmm5, zmm9",
    "vpxord  zmm6, zmm6, zmm10",
    "vpxord  zmm7, zmm7, zmm11",
    "vprord  zmm4, zmm4, 7",
    "vprord  zmm5, zmm5, 7",
    "vprord  zmm6, zmm6, 7",
    "vprord  zmm7, zmm7, 7",
    "vpaddd  zmm0, zmm0, zmm17",
    "vpaddd  zmm1, zmm1, zmm28",
    "vpaddd  zmm2, zmm2, zmm25",
    "vpaddd  zmm3, zmm3, zmm31",
    "vpaddd  zmm0, zmm0, zmm5",
    "vpaddd  zmm1, zmm1, zmm6",
    "vpaddd  zmm2, zmm2, zmm7",
    "vpaddd  zmm3, zmm3, zmm4",
    "vpxord  zmm15, zmm15, zmm0",
    "vpxord  zmm12, zmm12, zmm1",
    "vpxord  zmm13, zmm13, zmm2",
    "vpxord  zmm14, zmm14, zmm3",
    "vprord  zmm15, zmm15, 16",
    "vprord  zmm12, zmm12, 16",
    "vprord  zmm13, zmm13, 16",
    "vprord  zmm14, zmm14, 16",
    "vpaddd  zmm10, zmm10, zmm15",
    "vpaddd  zmm11, zmm11, zmm12",
    "vpaddd  zmm8, zmm8, zmm13",
    "vpaddd  zmm9, zmm9, zmm14",
    "vpxord  zmm5, zmm5, zmm10",
    "vpxord  zmm6, zmm6, zmm11",
    "vpxord  zmm7, zmm7, zmm8",
    "vpxord  zmm4, zmm4, zmm9",
    "vprord  zmm5, zmm5, 12",
    "vprord  zmm6, zmm6, 12",
    "vprord  zmm7, zmm7, 12",
    "vprord  zmm4, zmm4, 12",
    "vpaddd  zmm0, zmm0, zmm27",
    "vpaddd  zmm1, zmm1, zmm21",
    "vpaddd  zmm2, zmm2, zmm30",
    "vpaddd  zmm3, zmm3, zmm24",
    "vpaddd  zmm0, zmm0, zmm5",
    "vpaddd  zmm1, zmm1, zmm6",
    "vpaddd  zmm2, zmm2, zmm7",
    "vpaddd  zmm3, zmm3, zmm4",
    "vpxord  zmm15, zmm15, zmm0",
    "vpxord  zmm12, zmm12, zmm1",
    "vpxord  zmm13, zmm13, zmm2",
    "vpxord  zmm14, zmm14, zmm3",
    "vprord  zmm15, zmm15, 8",
    "vprord  zmm12, zmm12, 8",
    "vprord  zmm13, zmm13, 8",
    "vprord  zmm14, zmm14, 8",
    "vpaddd  zmm10, zmm10, zmm15",
    "vpaddd  zmm11, zmm11, zmm12",
    "vpaddd  zmm8, zmm8, zmm13",
    "vpaddd  zmm9, zmm9, zmm14",
    "vpxord  zmm5, zmm5, zmm10",
    "vpxord  zmm6, zmm6, zmm11",
    "vpxord  zmm7, zmm7, zmm8",
    "vpxord  zmm4, zmm4, zmm9",
    "vprord  zmm5, zmm5, 7",
    "vprord  zmm6, zmm6, 7",
    "vprord  zmm7, zmm7, 7",
    "vprord  zmm4, zmm4, 7",
    // round 3
    "vpaddd  zmm0, zmm0, zmm19",
    "vpaddd  zmm1, zmm1, zmm26",
    "vpaddd  zmm2, zmm2, zmm29",
    "vpaddd  zmm3, zmm3, zmm23",
    "vpaddd  zmm0, zmm0, zmm4",
    "vpaddd  zmm1, zmm1, zmm5",
    "vpaddd  zmm2, zmm2, zmm6",
    "vpaddd  zmm3, zmm3, zmm7",
    "vpxord  zmm12, zmm12, zmm0",
    "vpxord  zmm13, zmm13, zmm1",
    "vpxord  zmm14, zmm14, zmm2",
    "vpxord  zmm15, zmm15, zmm3",
    "vprord  zmm12, zmm12, 16",
    "vprord  zmm13, zmm13, 16",
    "vprord  zmm14, zmm14, 16",
    "vprord  zmm15, zmm15, 16",
    "vpaddd  zmm8, zmm8, zmm12",
    "vpaddd  zmm9, zmm9, zmm13",
    "vpaddd  zmm10, zmm10, zmm14",
    "vpaddd  zmm11, zmm11, zmm15",
    "vpxord  zmm4, zmm4, zmm8",
    "vpxord  zmm5, zmm5, zmm9",
    "vpxord  zmm6, zmm6, zmm10",
    "vpxord  zmm7, zmm7, zmm11",
    "vprord  zmm4, zmm4, 12",
    "vprord  zmm5, zmm5, 12",
    "vprord  zmm6, zmm6, 12",
    "vprord  zmm7, zmm7, 12",
    "vpaddd  zmm0, zmm0, zmm20",
    "vpaddd  zmm1, zmm1, zmm28",
    "vpaddd  zmm2, zmm2, zmm18",
    "vpaddd  zmm3, zmm3, zmm30",
    "vpaddd  zmm0, zmm0, zmm4",
    "vpaddd  zmm1, zmm1, zmm5",
    "vpaddd  zmm2, zmm2, zmm6",
    "vpaddd  zmm3, zmm3, zmm7",
    "vpxord  zmm12, zmm12, zmm0",
    "vpxord  zmm13, zmm13, zmm1",
    "vpxord  zmm14, zmm14, zmm2",
    "vpxord  zmm15, zmm15, zmm3",
    "vprord  zmm12, zmm12, 8",
    "vprord  zmm13, zmm13, 8",
    "vprord  zmm14, zmm14, 8",
    "vprord  zmm15, zmm15, 8",
    "vpaddd  zmm8, zmm8, zmm12",
    "vpaddd  zmm9, zmm9, zmm13",
    "vpaddd  zmm10, zmm10, zmm14",
    "vpaddd  zmm11, zmm11, zmm15",
    "vpxord  zmm4, zmm4, zmm8",
    "vpxord  zmm5, zmm5, zmm9",
    "vpxord  zmm6, zmm6, zmm10",
    "vpxord  zmm7, zmm7, zmm11",
    "vprord  zmm4, zmm4, 7",
    "vprord  zmm5, zmm5, 7",
    "vprord  zmm6, zmm6, 7",
    "vprord  zmm7, zmm7, 7",
    "vpaddd  zmm0, zmm0, zmm22",
    "vpaddd  zmm1, zmm1, zmm25",
    "vpaddd  zmm2, zmm2, zmm27",
    "vpaddd  zmm3, zmm3, zmm24",
    "vpaddd  zmm0, zmm0, zmm5",
    "vpaddd  zmm1, zmm1, zmm6",
    "vpaddd  zmm2, zmm2, zmm7",
    "vpaddd  zmm3, zmm3, zmm4",
    "vpxord  zmm15, zmm15, zmm0",
    "vpxord  zmm12, zmm12, zmm1",
    "vpxord  zmm13, zmm13, zmm2",
    "vpxord  zmm14, zmm14, zmm3",
    "vprord  zmm15, zmm15, 16",
    "vprord  zmm12, zmm12, 16",
    "vprord  zmm13, zmm13, 16",
    "vprord  zmm14, zmm14, 16",
    "vpaddd  zmm10, zmm10, zmm15",
    "vpaddd  zmm11, zmm11, zmm12",
    "vpaddd  zmm8, zmm8, zmm13",
    "vpaddd  zmm9, zmm9, zmm14",
    "vpxord  zmm5, zmm5, zmm10",
    "vpxord  zmm6, zmm6, zmm11",
    "vpxord  zmm7, zmm7, zmm8",
    "vpxord  zmm4, zmm4, zmm9",
    "vprord  zmm5, zmm5, 12",
    "vprord  zmm6, zmm6, 12",
    "vprord  zmm7, zmm7, 12",
    "vprord  zmm4, zmm4, 12",
    "vpaddd  zmm0, zmm0, zmm21",
    "vpaddd  zmm1, zmm1, zmm16",
    "vpaddd  zmm2, zmm2, zmm31",
    "vpaddd  zmm3, zmm3, zmm17",
    "vpaddd  zmm0, zmm0, zmm5",
    "vpaddd  zmm1, zmm1, zmm6",
    "vpaddd  zmm2, zmm2, zmm7",
    "vpaddd  zmm3, zmm3, zmm4",
    "vpxord  zmm15, zmm15, zmm0",
    "vpxord  zmm12, zmm12, zmm1",
    "vpxord  zmm13, zmm13, zmm2",
    "vpxord  zmm14, zmm14, zmm3",
    "vprord  zmm15, zmm15, 8",
    "vprord  zmm12, zmm12, 8",
    "vprord  zmm13, zmm13, 8",
    "vprord  zmm14, zmm14, 8",
    "vpaddd  zmm10, zmm10, zmm15",
    "vpaddd  zmm11, zmm11, zmm12",
    "vpaddd  zmm8, zmm8, zmm13",
    "vpaddd  zmm9, zmm9, zmm14",
    "vpxord  zmm5, zmm5, zmm10",
    "vpxord  zmm6, zmm6, zmm11",
    "vpxord  zmm7, zmm7, zmm8",
    "vpxord  zmm4, zmm4, zmm9",
    "vprord  zmm5, zmm5, 7",
    "vprord  zmm6, zmm6, 7",
    "vprord  zmm7, zmm7, 7",
    "vprord  zmm4, zmm4, 7",
    // round 4
    "vpaddd  zmm0, zmm0, zmm26",
    "vpaddd  zmm1, zmm1, zmm28",
    "vpaddd  zmm2, zmm2, zmm30",
    "vpaddd  zmm3, zmm3, zmm29",
    "vpaddd  zmm0, zmm0, zmm4",
    "vpaddd  zmm1, zmm1, zmm5",
    "vpaddd  zmm2, zmm2, zmm6",
    "vpaddd  zmm3, zmm3, zmm7",
    "vpxord  zmm12, zmm12, zmm0",
    "vpxord  zmm13, zmm13, zmm1",
    "vpxord  zmm14, zmm14, zmm2",
    "vpxord  zmm15, zmm15, zmm3",
    "vprord  zmm12, zmm12, 16",
    "vprord  zmm13, zmm13, 16",
    "vprord  zmm14, zmm14, 16",
    "vprord  zmm15, zmm15, 16",
    "vpaddd  zmm8, zmm8, zmm12",
    "vpaddd  zmm9, zmm9, zmm13",
    "vpaddd  zmm10, zmm10, zmm14",
    "vpaddd  zmm11, zmm11, zmm15",
    "vpxord  zmm4, zmm4, zmm8",
    "vpxord  zmm5, zmm5, zmm9",
    "vpxord  zmm6, zmm6, zmm10",
    "vpxord  zmm7, zmm7, zmm11",
    "vprord  zmm4, zmm4, 12",
    "vprord  zmm5, zmm5, 12",
    "vprord  zmm6, zmm6, 12",
    "vprord  zmm7, zmm7, 12",
    "vpaddd  zmm0, zmm0, zmm23",
    "vpaddd  zmm1, zmm1, zmm25",
    "vpaddd  zmm2, zmm2, zmm19",
    "vpaddd  zmm3, zmm3, zmm31",
    "vpaddd  zmm0, zmm0, zmm4",
    "vpaddd  zmm1, zmm1, zmm5",
    "vpaddd  zmm2, zmm2, zmm6",
    "vpaddd  zmm3, zmm3, zmm7",
    "vpxord  zmm12, zmm12, zmm0",
    "vpxord  zmm13, zmm13, zmm1",
    "vpxord  zmm14, zmm14, zmm2",
    "vpxord  zmm15, zmm15, zmm3",
    "vprord  zmm12, zmm12, 8",
    "vprord  zmm13, zmm13, 8",
    "vprord  zmm14, zmm14, 8",
    "vprord  zmm15, zmm15, 8",
    "vpaddd  zmm8, zmm8, zmm12",
    "vpaddd  zmm9, zmm9, zmm13",
    "vpaddd  zmm10, zmm10, zmm14",
    "vpaddd  zmm11, zmm11, zmm15",
    "vpxord  zmm4, zmm4, zmm8",
    "vpxord  zmm5, zmm5, zmm9",
    "vpxord  zmm6, zmm6, zmm10",
    "vpxord  zmm7, zmm7, zmm11",
    "vprord  zmm4, zmm4, 7",
    "vprord  zmm5, zmm5, 7",
    "vprord  zmm6, zmm6, 7",
    "vprord  zmm7, zmm7, 7",
    "vpaddd  zmm0, zmm0, zmm20",
    "vpaddd  zmm1, zmm1, zmm27",
    "vpaddd  zmm2, zmm2, zmm21",
    "vpaddd  zmm3, zmm3, zmm17",
    "vpaddd  zmm0, zmm0, zmm5",
    "vpaddd  zmm1, zmm1, zmm6",
    "vpaddd  zmm2, zmm2, zmm7",
    "vpaddd  zmm3, zmm3, zmm4",
    "vpxord  zmm15, zmm15, zmm0",
    "vpxord  zmm12, zmm12, zmm1",
    "vpxord  zmm13, zmm13, zmm2",
    "vpxord  zmm14, zmm14, zmm3",
    "vprord  zmm15, zmm15, 16",
    "vprord  zmm12, zmm12, 16",
    "vprord  zmm13, zmm13, 16",
    "vprord  zmm14, zmm14, 16",
    "vpaddd  zmm10, zmm10, zmm15",
    "vpaddd  zmm11, zmm11, zmm12",
    "vpaddd  zmm8, zmm8, zmm13",
    "vpaddd  zmm9, zmm9, zmm14",
    "vpxord  zmm5, zmm5, zmm10",
    "vpxord  zmm6, zmm6, zmm11",
    "vpxord  zmm7, zmm7, zmm8",
    "vpxord  zmm4, zmm4, zmm9",
    "vprord  zmm5, zmm5, 12",
    "vprord  zmm6, zmm6, 12",
    "vprord  zmm7, zmm7, 12",
    "vprord  zmm4, zmm4, 12",
    "vpaddd  zmm0, zmm0, zmm16",
    "vpaddd  zmm1, zmm1, zmm18",
    "vpaddd  zmm2, zmm2, zmm24",
    "vpaddd  zmm3, zmm3, zmm22",
    "vpaddd  zmm0, zmm0, zmm5",
    "vpaddd  zmm1, zmm1, zmm6",
    "vpaddd  zmm2, zmm2, zmm7",
    "vpaddd  zmm3, zmm3, zmm4",
    "vpxord  zmm15, zmm15, zmm0",
    "vpxord  zmm12, zmm12, zmm1",
    "vpxord  zmm13, zmm13, zmm2",
    "vpxord  zmm14, zmm14, zmm3",
    "vprord  zmm15, zmm15, 8",
    "vprord  zmm12, zmm12, 8",
    "vprord  zmm13, zmm13, 8",
    "vprord  zmm14, zmm14, 8",
    "vpaddd  zmm10, zmm10, zmm15",
    "vpaddd  zmm11, zmm11, zmm12",
    "vpaddd  zmm8, zmm8, zmm13",
    "vpaddd  zmm9, zmm9, zmm14",
    "vpxord  zmm5, zmm5, zmm10",
    "vpxord  zmm6, zmm6, zmm11",
    "vpxord  zmm7, zmm7, zmm8",
    "vpxord  zmm4, zmm4, zmm9",
    "vprord  zmm5, zmm5, 7",
    "vprord  zmm6, zmm6, 7",
    "vprord  zmm7, zmm7, 7",
    "vprord  zmm4, zmm4, 7",
    // round 5
    "vpaddd  zmm0, zmm0, zmm28",
    "vpaddd  zmm1, zmm1, zmm25",
    "vpaddd  zmm2, zmm2, zmm31",
    "vpaddd  zmm3, zmm3, zmm30",
    "vpaddd  zmm0, zmm0, zmm4",
    "vpaddd  zmm1, zmm1, zmm5",
    "vpaddd  zmm2, zmm2, zmm6",
    "vpaddd  zmm3, zmm3, zmm7",
    "vpxord  zmm12, zmm12, zmm0",
    "vpxord  zmm13, zmm13, zmm1",
    "vpxord  zmm14, zmm14, zmm2",
    "vpxord  zmm15, zmm15, zmm3",
    "vprord  zmm12, zmm12, 16",
    "vprord  zmm13, zmm13, 16",
    "vprord  zmm14, zmm14, 16",
    "vprord  zmm15, zmm15, 16",
    "vpaddd  zmm8, zmm8, zmm12",
    "vpaddd  zmm9, zmm9, zmm13",
    "vpaddd  zmm10, zmm10, zmm14",
    "vpaddd  zmm11, zmm11, zmm15",
    "vpxord  zmm4, zmm4, zmm8",
    "vpxord  zmm5, zmm5, zmm9",
    "vpxord  zmm6, zmm6, zmm10",
    "vpxord  zmm7, zmm7, zmm11",
    "vprord  zmm4, zmm4, 12",
    "vprord  zmm5, zmm5, 12",
    "vprord  zmm6, zmm6, 12",
    "vprord  zmm7, zmm7, 12",
    "vpaddd  zmm0, zmm0, zmm29",
    "vpaddd  zmm1, zmm1, zmm27",
    "vpaddd  zmm2, zmm2, zmm26",
    "vpaddd  zmm3, zmm3, zmm24",
    "vpaddd  zmm0, zmm0, zmm4",
    "vpaddd  zmm1, zmm1, zmm5",
    "vpaddd  zmm2, zmm2, zmm6",
    "vpaddd  zmm3, zmm3, zmm7",
    "vpxord  zmm12, zmm12, zmm0",
    "vpxord  zmm13, zmm13, zmm1",
    "vpxord  zmm14, zmm14, zmm2",
    "vpxord  zmm15, zmm15, zmm3",
    "vprord  zmm12, zmm12, 8",
    "vprord  zmm13, zmm13, 8",
    "vprord  zmm14, zmm14, 8",
    "vprord  zmm15, zmm15, 8",
    "vpaddd  zmm8, zmm8, zmm12",
    "vpaddd  zmm9, zmm9, zmm13",
    "vpaddd  zmm10, zmm10, zmm14",
    "vpaddd  zmm11, zmm11, zmm15",
    "vpxord  zmm4, zmm4, zmm8",
    "vpxord  zmm5, zmm5, zmm9",
    "vpxord  zmm6, zmm6, zmm10",
    "vpxord  zmm7, zmm7, zmm11",
    "vprord  zmm4, zmm4, 7",
    "vprord  zmm5, zmm5, 7",
    "vprord  zmm6, zmm6, 7",
    "vprord  zmm7, zmm7, 7",
    "vpaddd  zmm0, zmm0, zmm23",
    "vpaddd  zmm1, zmm1, zmm21",
    "vpaddd  zmm2, zmm2, zmm16",
    "vpaddd  zmm3, zmm3, zmm22",
    "vpaddd  zmm0, zmm0, zmm5",
    "vpaddd  zmm1, zmm1, zmm6",
    "vpaddd  zmm2, zmm2, zmm7",
    "vpaddd  zmm3, zmm3, zmm4",
    "vpxord  zmm15, zmm15, zmm0",
    "vpxord  zmm12, zmm12, zmm1",
    "vpxord  zmm13, zmm13, zmm2",
    "vpxord  zmm14, zmm14, zmm3",
    "vprord  zmm15, zmm15, 16",
    "vprord  zmm12, zmm12, 16",
    "vprord  zmm13, zmm13, 16",
    "vprord  zmm14, zmm14, 16",
    "vpaddd  zmm10, zmm10, zmm15",
    "vpaddd  zmm11, zmm11, zmm12",
    "vpaddd  zmm8, zmm8, zmm13",
    "vpaddd  zmm9, zmm9, zmm14",
    "vpxord  zmm5, zmm5, zmm10",
    "vpxord  zmm6, zmm6, zmm11",
    "vpxord  zmm7, zmm7, zmm8",
    "vpxord  zmm4, zmm4, zmm9",
    "vprord  zmm5, zmm5, 12",
    "vprord  zmm6, zmm6, 12",
    "vprord  zmm7, zmm7, 12",
    "vprord  zmm4, zmm4, 12",
    "vpaddd  zmm0, zmm0, zmm18",
    "vpaddd  zmm1, zmm1, zmm19",
    "vpaddd  zmm2, zmm2, zmm17",
    "vpaddd  zmm3, zmm3, zmm20",
    "vpaddd  zmm0, zmm0, zmm5",
    "vpaddd  zmm1, zmm1, zmm6",
    "vpaddd  zmm2, zmm2, zmm7",
    "vpaddd  zmm3, zmm3, zmm4",
    "vpxord  zmm15, zmm15, zmm0",
    "vpxord  zmm12, zmm12, zmm1",
    "vpxord  zmm13, zmm13, zmm2",
    "vpxord  zmm14, zmm14, zmm3",
    "vprord  zmm15, zmm15, 8",
    "vprord  zmm12, zmm12, 8",
    "vprord  zmm13, zmm13, 8",
    "vprord  zmm14, zmm14, 8",
    "vpaddd  zmm10, zmm10, zmm15",
    "vpaddd  zmm11, zmm11, zmm12",
    "vpaddd  zmm8, zmm8, zmm13",
    "vpaddd  zmm9, zmm9, zmm14",
    "vpxord  zmm5, zmm5, zmm10",
    "vpxord  zmm6, zmm6, zmm11",
    "vpxord  zmm7, zmm7, zmm8",
    "vpxord  zmm4, zmm4, zmm9",
    "vprord  zmm5, zmm5, 7",
    "vprord  zmm6, zmm6, 7",
    "vprord  zmm7, zmm7, 7",
    "vprord  zmm4, zmm4, 7",
    // round 6
    "vpaddd  zmm0, zmm0, zmm25",
    "vpaddd  zmm1, zmm1, zmm27",
    "vpaddd  zmm2, zmm2, zmm24",
    "vpaddd  zmm3, zmm3, zmm31",
    "vpaddd  zmm0, zmm0, zmm4",
    "vpaddd  zmm1, zmm1, zmm5",
    "vpaddd  zmm2, zmm2, zmm6",
    "vpaddd  zmm3, zmm3, zmm7",
    "vpxord  zmm12, zmm12, zmm0",
    "vpxord  zmm13, zmm13, zmm1",
    "vpxord  zmm14, zmm14, zmm2",
    "vpxord  zmm15, zmm15, zmm3",
    "vprord  zmm12, zmm12, 16",
    "vprord  zmm13, zmm13, 16",
    "vprord  zmm14, zmm14, 16",
    "vprord  zmm15, zmm15, 16",
    "vpaddd  zmm8, zmm8, zmm12",
    "vpaddd  zmm9, zmm9, zmm13",
    "vpaddd  zmm10, zmm10, zmm14",
    "vpaddd  zmm11, zmm11, zmm15",
    "vpxord  zmm4, zmm4, zmm8",
    "vpxord  zmm5, zmm5, zmm9",
    "vpxord  zmm6, zmm6, zmm10",
    "vpxord  zmm7, zmm7, zmm11",
    "vprord  zmm4, zmm4, 12",
    "vprord  zmm5, zmm5, 12",
    "vprord  zmm6, zmm6, 12",
    "vprord  zmm7, zmm7, 12",
    "vpaddd  zmm0, zmm0, zmm30",
    "vpaddd  zmm1, zmm1, zmm21",
    "vpaddd  zmm2, zmm2, zmm28",
    "vpaddd  zmm3, zmm3, zmm17",
    "vpaddd  zmm0, zmm0, zmm4",
    "vpaddd  zmm1, zmm1, zmm5",
    "vpaddd  zmm2, zmm2, zmm6",
    "vpaddd  zmm3, zmm3, zmm7",
    "vpxord  zmm12, zmm12, zmm0",
    "vpxord  zmm13, zmm13, zmm1",
    "vpxord  zmm14, zmm14, zmm2",
    "vpxord  zmm15, zmm15, zmm3",
    "vprord  zmm12, zmm12, 8",
    "vprord  zmm13, zmm13, 8",
    "vprord  zmm14, zmm14, 8",
    "vprord  zmm15, zmm15, 8",
    "vpaddd  zmm8, zmm8, zmm12",
    "vpaddd  zmm9, zmm9, zmm13",
    "vpaddd  zmm10, zmm10, zmm14",
    "vpaddd  zmm11, zmm11, zmm15",
    "vpxord  zmm4, zmm4, zmm8",
    "vpxord  zmm5, zmm5, zmm9",
    "vpxord  zmm6, zmm6, zmm10",
    "vpxord  zmm7, zmm7, zmm11",
    "vprord  zmm4, zmm4, 7",
    "vprord  zmm5, zmm5, 7",
    "vprord  zmm6, zmm6, 7",
    "vprord  zmm7, zmm7, 7",
    "vpaddd  zmm0, zmm0, zmm29",
    "vpaddd  zmm1, zmm1, zmm16",
    "vpaddd  zmm2, zmm2, zmm18",
    "vpaddd  zmm3, zmm3, zmm20",
    "vpaddd  zmm0, zmm0, zmm5",
    "vpaddd  zmm1, zmm1, zmm6",
    "vpaddd  zmm2, zmm2, zmm7",
    "vpaddd  zmm3, zmm3, zmm4",
    "vpxord  zmm15, zmm15, zmm0",
    "vpxord  zmm12, zmm12, zmm1",
    "vpxord  zmm13, zmm13, zmm2",
    "vpxord  zmm14, zmm14, zmm3",
    "vprord  zmm15, zmm15, 16",
    "vprord  zmm12, zmm12, 16",
    "vprord  zmm13, zmm13, 16",
    "vprord  zmm14, zmm14, 16",
    "vpaddd  zmm10, zmm10, zmm15",
    "vpaddd  zmm11, zmm11, zmm12",
    "vpaddd  zmm8, zmm8, zmm13",
    "vpaddd  zmm9, zmm9, zmm14",
    "vpxord  zmm5, zmm5, zmm10",
    "vpxord  zmm6, zmm6, zmm11",
    "vpxord  zmm7, zmm7, zmm8",
    "vpxord  zmm4, zmm4, zmm9",
    "vprord  zmm5, zmm5, 12",
    "vprord  zmm6, zmm6, 12",
    "vprord  zmm7, zmm7, 12",
    "vprord  zmm4, zmm4, 12",
    "vpaddd  zmm0, zmm0, zmm19",
    "vpaddd  zmm1, zmm1, zmm26",
    "vpaddd  zmm2, zmm2, zmm22",
    "vpaddd  zmm3, zmm3, zmm23",
    "vpaddd  zmm0, zmm0, zmm5",
    "vpaddd  zmm1, zmm1, zmm6",
    "vpaddd  zmm2, zmm2, zmm7",
    "vpaddd  zmm3, zmm3, zmm4",
    "vpxord  zmm15, zmm15, zmm0",
    "vpxord  zmm12, zmm12, zmm1",
    "vpxord  zmm13, zmm13, zmm2",
    "vpxord  zmm14, zmm14, zmm3",
    "vprord  zmm15, zmm15, 8",
    "vprord  zmm12, zmm12, 8",
    "vprord  zmm13, zmm13, 8",
    "vprord  zmm14, zmm14, 8",
    "vpaddd  zmm10, zmm10, zmm15",
    "vpaddd  zmm11, zmm11, zmm12",
    "vpaddd  zmm8, zmm8, zmm13",
    "vpaddd  zmm9, zmm9, zmm14",
    "vpxord  zmm5, zmm5, zmm10",
    "vpxord  zmm6, zmm6, zmm11",
    "vpxord  zmm7, zmm7, zmm8",
    "vpxord  zmm4, zmm4, zmm9",
    "vprord  zmm5, zmm5, 7",
    "vprord  zmm6, zmm6, 7",
    "vprord  zmm7, zmm7, 7",
    "vprord  zmm4, zmm4, 7",
    // round 7
    "vpaddd  zmm0, zmm0, zmm27",
    "vpaddd  zmm1, zmm1, zmm21",
    "vpaddd  zmm2, zmm2, zmm17",
    "vpaddd  zmm3, zmm3, zmm24",
    "vpaddd  zmm0, zmm0, zmm4",
    "vpaddd  zmm1, zmm1, zmm5",
    "vpaddd  zmm2, zmm2, zmm6",
    "vpaddd  zmm3, zmm3, zmm7",
    "vpxord  zmm12, zmm12, zmm0",
    "vpxord  zmm13, zmm13, zmm1",
    "vpxord  zmm14, zmm14, zmm2",
    "vpxord  zmm15, zmm15, zmm3",
    "vprord  zmm12, zmm12, 16",
    "vprord  zmm13, zmm13, 16",
    "vprord  zmm14, zmm14, 16",
    "vprord  zmm15, zmm15, 16",
    "vpaddd  zmm8, zmm8, zmm12",
    "vpaddd  zmm9, zmm9, zmm13",
    "vpaddd  zmm10, zmm10, zmm14",
    "vpaddd  zmm11, zmm11, zmm15",
    "vpxord  zmm4, zmm4, zmm8",
    "vpxord  zmm5, zmm5, zmm9",
    "vpxord  zmm6, zmm6, zmm10",
    "vpxord  zmm7, zmm7, zmm11",
    "vprord  zmm4, zmm4, 12",
    "vprord  zmm5, zmm5, 12",
    "vprord  zmm6, zmm6, 12",
    "vprord  zmm7, zmm7, 12",
    "vpaddd  zmm0, zmm0, zmm31",
    "vpaddd  zmm1, zmm1, zmm16",
    "vpaddd  zmm2, zmm2, zmm25",
    "vpaddd  zmm3, zmm3, zmm22",
    "vpaddd  zmm0, zmm0, zmm4",
    "vpaddd  zmm1, zmm1, zmm5",
    "vpaddd  zmm2, zmm2, zmm6",
    "vpaddd  zmm3, zmm3, zmm7",
    "vpxord  zmm12, zmm12, zmm0",
    "vpxord  zmm13, zmm13, zmm1",
    "vpxord  zmm14, zmm14, zmm2",
    "vpxord  zmm15, zmm15, zmm3",
    "vprord  zmm12, zmm12, 8",
    "vprord  zmm13, zmm13, 8",
    "vprord  zmm14, zmm14, 8",
    "vprord  zmm15, zmm15, 8",
    "vpaddd  zmm8, zmm8, zmm12",
    "vpaddd  zmm9, zmm9, zmm13",
    "vpaddd  zmm10, zmm10, zmm14",
    "vpaddd  zmm11, zmm11, zmm15",
    "vpxord  zmm4, zmm4, zmm8",
    "vpxord  zmm5, zmm5, zmm9",
    "vpxord  zmm6, zmm6, zmm10",
    "vpxord  zmm7, zmm7, zmm11",
    "vprord  zmm4, zmm4, 7",
    "vprord  zmm5, zmm5, 7",
    "vprord  zmm6, zmm6, 7",
    "vprord  zmm7, zmm7, 7",
    "vpaddd  zmm0, zmm0, zmm30",
    "vpaddd  zmm1, zmm1, zmm18",
    "vpaddd  zmm2, zmm2, zmm19",
    "vpaddd  zmm3, zmm3, zmm23",
    "vpaddd  zmm0, zmm0, zmm5",
    "vpaddd  zmm1, zmm1, zmm6",
    "vpaddd  zmm2, zmm2, zmm7",
    "vpaddd  zmm3, zmm3, zmm4",
    "vpxord  zmm15, zmm15, zmm0",
    "vpxord  zmm12, zmm12, zmm1",
    "vpxord  zmm13, zmm13, zmm2",
    "vpxord  zmm14, zmm14, zmm3",
    "vprord  zmm15, zmm15, 16",
    "vprord  zmm12, zmm12, 16",
    "vprord  zmm13, zmm13, 16",
    "vprord  zmm14, zmm14, 16",
    "vpaddd  zmm10, zmm10, zmm15",
    "vpaddd  zmm11, zmm11, zmm12",
    "vpaddd  zmm8, zmm8, zmm13",
    "vpaddd  zmm9, zmm9, zmm14",
    "vpxord  zmm5, zmm5, zmm10",
    "vpxord  zmm6, zmm6, zmm11",
    "vpxord  zmm7, zmm7, zmm8",
    "vpxord  zmm4, zmm4, zmm9",
    "vprord  zmm5, zmm5, 12",
    "vprord  zmm6, zmm6, 12",
    "vprord  zmm7, zmm7, 12",
    "vprord  zmm4, zmm4, 12",
    "vpaddd  zmm0, zmm0, zmm26",
    "vpaddd  zmm1, zmm1, zmm28",
    "vpaddd  zmm2, zmm2, zmm20",
    "vpaddd  zmm3, zmm3, zmm29",
    "vpaddd  zmm0, zmm0, zmm5",
    "vpaddd  zmm1, zmm1, zmm6",
    "vpaddd  zmm2, zmm2, zmm7",
    "vpaddd  zmm3, zmm3, zmm4",
    "vpxord  zmm15, zmm15, zmm0",
    "vpxord  zmm12, zmm12, zmm1",
    "vpxord  zmm13, zmm13, zmm2",
    "vpxord  zmm14, zmm14, zmm3",
    "vprord  zmm15, zmm15, 8",
    "vprord  zmm12, zmm12, 8",
    "vprord  zmm13, zmm13, 8",
    "vprord  zmm14, zmm14, 8",
    "vpaddd  zmm10, zmm10, zmm15",
    "vpaddd  zmm11, zmm11, zmm12",
    "vpaddd  zmm8, zmm8, zmm13",
    "vpaddd  zmm9, zmm9, zmm14",
    "vpxord  zmm5, zmm5, zmm10",
    "vpxord  zmm6, zmm6, zmm11",
    "vpxord  zmm7, zmm7, zmm8",
    "vpxord  zmm4, zmm4, zmm9",
    "vprord  zmm5, zmm5, 7",
    "vprord  zmm6, zmm6, 7",
    "vprord  zmm7, zmm7, 7",
    "vprord  zmm4, zmm4, 7",
    // final xors
    "vpxord  zmm0, zmm0, zmm8",
    "vpxord  zmm1, zmm1, zmm9",
    "vpxord  zmm2, zmm2, zmm10",
    "vpxord  zmm3, zmm3, zmm11",
    "vpxord  zmm4, zmm4, zmm12",
    "vpxord  zmm5, zmm5, zmm13",
    "vpxord  zmm6, zmm6, zmm14",
    "vpxord  zmm7, zmm7, zmm15",
    "ret",
);

#[inline(always)]
unsafe fn load_transposed_16(input: *const u8) -> [__m512i; 16] {
    // lane selectors for _mm512_permutex2var_epi64
    let lower_256 = _mm512_setr_epi64(0x0, 0x1, 0x2, 0x3, 0x8, 0x9, 0xa, 0xb);
    let upper_256 = _mm512_setr_epi64(0x4, 0x5, 0x6, 0x7, 0xc, 0xd, 0xe, 0xf);
    let lower_128 = _mm512_setr_epi64(0x0, 0x1, 0x8, 0x9, 0x4, 0x5, 0xc, 0xd);
    let upper_128 = _mm512_setr_epi64(0x2, 0x3, 0xa, 0xb, 0x6, 0x7, 0xe, 0xf);

    let a = _mm512_loadu_si512(input.add(0x0 * CHUNK_LEN) as *const i32);
    let i = _mm512_loadu_si512(input.add(0x8 * CHUNK_LEN) as *const i32);
    let ai_01234567 = _mm512_permutex2var_epi64(a, lower_256, i);
    let ai_89abcdef = _mm512_permutex2var_epi64(a, upper_256, i);
    let e = _mm512_loadu_si512(input.add(0x4 * CHUNK_LEN) as *const i32);
    let m = _mm512_loadu_si512(input.add(0xc * CHUNK_LEN) as *const i32);
    let em_01234567 = _mm512_permutex2var_epi64(e, lower_256, m);
    let em_89abcdef = _mm512_permutex2var_epi64(e, upper_256, m);
    let aeim_0123 = _mm512_permutex2var_epi64(ai_01234567, lower_128, em_01234567);
    let aeim_4567 = _mm512_permutex2var_epi64(ai_01234567, upper_128, em_01234567);
    let aeim_89ab = _mm512_permutex2var_epi64(ai_89abcdef, lower_128, em_89abcdef);
    let aeim_cdef = _mm512_permutex2var_epi64(ai_89abcdef, upper_128, em_89abcdef);

    let b = _mm512_loadu_si512(input.add(0x1 * CHUNK_LEN) as *const i32);
    let j = _mm512_loadu_si512(input.add(0x9 * CHUNK_LEN) as *const i32);
    let bj_01234567 = _mm512_permutex2var_epi64(b, lower_256, j);
    let bj_89abcdef = _mm512_permutex2var_epi64(b, upper_256, j);
    let f = _mm512_loadu_si512(input.add(0x5 * CHUNK_LEN) as *const i32);
    let n = _mm512_loadu_si512(input.add(0xd * CHUNK_LEN) as *const i32);
    let fn_01234567 = _mm512_permutex2var_epi64(f, lower_256, n);
    let fn_89abcdef = _mm512_permutex2var_epi64(f, upper_256, n);
    let bfjn_0123 = _mm512_permutex2var_epi64(bj_01234567, lower_128, fn_01234567);
    let bfjn_4567 = _mm512_permutex2var_epi64(bj_01234567, upper_128, fn_01234567);
    let bfjn_89ab = _mm512_permutex2var_epi64(bj_89abcdef, lower_128, fn_89abcdef);
    let bfjn_cdef = _mm512_permutex2var_epi64(bj_89abcdef, upper_128, fn_89abcdef);

    // Interleave 32-bit words. This results in vectors like:
    // a0, b0, a1, b1, e0, f0, e1, f1, i0, j0, i1, j1, m0, n0, m1, n1
    let abefijmn_01 = _mm512_unpacklo_epi32(aeim_0123, bfjn_0123);
    let abefijmn_23 = _mm512_unpackhi_epi32(aeim_0123, bfjn_0123);
    let abefijmn_45 = _mm512_unpacklo_epi32(aeim_4567, bfjn_4567);
    let abefijmn_67 = _mm512_unpackhi_epi32(aeim_4567, bfjn_4567);
    let abefijmn_89 = _mm512_unpacklo_epi32(aeim_89ab, bfjn_89ab);
    let abefijmn_ab = _mm512_unpackhi_epi32(aeim_89ab, bfjn_89ab);
    let abefijmn_cd = _mm512_unpacklo_epi32(aeim_cdef, bfjn_cdef);
    let abefijmn_ef = _mm512_unpackhi_epi32(aeim_cdef, bfjn_cdef);

    let c = _mm512_loadu_si512(input.add(0x2 * CHUNK_LEN) as *const i32);
    let k = _mm512_loadu_si512(input.add(0xa * CHUNK_LEN) as *const i32);
    let ck_01234567 = _mm512_permutex2var_epi64(c, lower_256, k);
    let ck_89abcdef = _mm512_permutex2var_epi64(c, upper_256, k);
    let g = _mm512_loadu_si512(input.add(0x6 * CHUNK_LEN) as *const i32);
    let o = _mm512_loadu_si512(input.add(0xe * CHUNK_LEN) as *const i32);
    let go_01234567 = _mm512_permutex2var_epi64(g, lower_256, o);
    let go_89abcdef = _mm512_permutex2var_epi64(g, upper_256, o);
    let cgko_0123 = _mm512_permutex2var_epi64(ck_01234567, lower_128, go_01234567);
    let cgko_4567 = _mm512_permutex2var_epi64(ck_01234567, upper_128, go_01234567);
    let cgko_89ab = _mm512_permutex2var_epi64(ck_89abcdef, lower_128, go_89abcdef);
    let cgko_cdef = _mm512_permutex2var_epi64(ck_89abcdef, upper_128, go_89abcdef);

    let d = _mm512_loadu_si512(input.add(0x3 * CHUNK_LEN) as *const i32);
    let l = _mm512_loadu_si512(input.add(0xb * CHUNK_LEN) as *const i32);
    let dl_01234567 = _mm512_permutex2var_epi64(d, lower_256, l);
    let dl_89abcdef = _mm512_permutex2var_epi64(d, upper_256, l);
    let h = _mm512_loadu_si512(input.add(0x7 * CHUNK_LEN) as *const i32);
    let p = _mm512_loadu_si512(input.add(0xf * CHUNK_LEN) as *const i32);
    let hp_01234567 = _mm512_permutex2var_epi64(h, lower_256, p);
    let hp_89abcdef = _mm512_permutex2var_epi64(h, upper_256, p);
    let dhlp_0123 = _mm512_permutex2var_epi64(dl_01234567, lower_128, hp_01234567);
    let dhlp_4567 = _mm512_permutex2var_epi64(dl_01234567, upper_128, hp_01234567);
    let dhlp_89ab = _mm512_permutex2var_epi64(dl_89abcdef, lower_128, hp_89abcdef);
    let dhlp_cdef = _mm512_permutex2var_epi64(dl_89abcdef, upper_128, hp_89abcdef);

    // Interleave 32-bit words. This results in vectors like:
    // a0, b0, a1, b1, e0, f0, e1, f1, i0, j0, i1, j1, m0, n0, m1, n1
    let cdghklop_01 = _mm512_unpacklo_epi32(cgko_0123, dhlp_0123);
    let cdghklop_23 = _mm512_unpackhi_epi32(cgko_0123, dhlp_0123);
    let cdghklop_45 = _mm512_unpacklo_epi32(cgko_4567, dhlp_4567);
    let cdghklop_67 = _mm512_unpackhi_epi32(cgko_4567, dhlp_4567);
    let cdghklop_89 = _mm512_unpacklo_epi32(cgko_89ab, dhlp_89ab);
    let cdghklop_ab = _mm512_unpackhi_epi32(cgko_89ab, dhlp_89ab);
    let cdghklop_cd = _mm512_unpacklo_epi32(cgko_cdef, dhlp_cdef);
    let cdghklop_ef = _mm512_unpackhi_epi32(cgko_cdef, dhlp_cdef);

    // Finally, interleave 64-bit words. This gives us our goal, which is vectors like:
    // a0, b0, c0, d0, e0, f0, g0, h0, i0, j0, k0, l0, m0, n0, o0, p0
    [
        _mm512_unpacklo_epi64(abefijmn_01, cdghklop_01),
        _mm512_unpackhi_epi64(abefijmn_01, cdghklop_01),
        _mm512_unpacklo_epi64(abefijmn_23, cdghklop_23),
        _mm512_unpackhi_epi64(abefijmn_23, cdghklop_23),
        _mm512_unpacklo_epi64(abefijmn_45, cdghklop_45),
        _mm512_unpackhi_epi64(abefijmn_45, cdghklop_45),
        _mm512_unpacklo_epi64(abefijmn_67, cdghklop_67),
        _mm512_unpackhi_epi64(abefijmn_67, cdghklop_67),
        _mm512_unpacklo_epi64(abefijmn_89, cdghklop_89),
        _mm512_unpackhi_epi64(abefijmn_89, cdghklop_89),
        _mm512_unpacklo_epi64(abefijmn_ab, cdghklop_ab),
        _mm512_unpackhi_epi64(abefijmn_ab, cdghklop_ab),
        _mm512_unpacklo_epi64(abefijmn_cd, cdghklop_cd),
        _mm512_unpackhi_epi64(abefijmn_cd, cdghklop_cd),
        _mm512_unpacklo_epi64(abefijmn_ef, cdghklop_ef),
        _mm512_unpackhi_epi64(abefijmn_ef, cdghklop_ef),
    ]
}

#[test]
fn test_load_transpose_16() {
    if !crate::platform::avx512_detected() {
        return;
    }
    // 16 chunks, as 32-bit words rather than bytes
    let mut input = [0u32; (CHUNK_LEN / 4) * 16];
    // Populate the first 16 words of each chunk with an incrementing counter.
    for chunk in 0..16 {
        for word in 0..16 {
            let chunk_start = (CHUNK_LEN / 4) * chunk;
            let val = 16 * chunk as u32 + word as u32;
            input[chunk_start + word] = val;
        }
    }
    // Run the load-transpose and cast the vectors back to words.
    let transposed: [u32; 16 * 16] = unsafe {
        let vecs = load_transposed_16(input.as_ptr() as *const u8);
        mem::transmute(vecs)
    };
    // Now check for the same incrementing counter as above, except we've swapped the inner and
    // outer loops to account for the transposition.
    for word in 0..16 {
        for vec in 0..16 {
            let vec_start = 16 * vec;
            let val = 16 * word as u32 + vec as u32;
            assert_eq!(transposed[vec_start + word], val, "word {word} vec {vec}");
        }
    }
}

// returns (low_words, high_words)
#[inline(always)]
unsafe fn incrementing_counter(initial_value: u64) -> (__m512i, __m512i) {
    let increments = _mm512_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15);
    let initial_low_words = _mm512_set1_epi32(initial_value as u32 as i32);
    let low_words = _mm512_add_epi32(initial_low_words, increments);
    let less_than_mask = _mm512_cmplt_epu32_mask(low_words, initial_low_words);
    let initial_high_words = _mm512_set1_epi32((initial_value >> 32) as u32 as i32);
    let high_words = _mm512_mask_add_epi32(
        initial_high_words,
        less_than_mask,
        initial_high_words,
        _mm512_set1_epi32(1),
    );
    (low_words, high_words)
}

#[target_feature(enable = "avx512f,avx512vl")]
pub unsafe fn chunks_16(
    message: &[u8; 16 * CHUNK_LEN],
    key: &[u32; 8],
    counter: u64,
    flags: u32,
) -> [__m512i; 8] {
    let (counters_low, counters_high) = incrementing_counter(counter);
    let mut state_regs = [
        _mm512_set1_epi32(key[0] as i32),
        _mm512_set1_epi32(key[1] as i32),
        _mm512_set1_epi32(key[2] as i32),
        _mm512_set1_epi32(key[3] as i32),
        _mm512_set1_epi32(key[4] as i32),
        _mm512_set1_epi32(key[5] as i32),
        _mm512_set1_epi32(key[6] as i32),
        _mm512_set1_epi32(key[7] as i32),
    ];
    for block in 0..(CHUNK_LEN / BLOCK_LEN) {
        let message_regs = load_transposed_16(message.as_ptr().add(BLOCK_LEN * block));
        let mut block_flags = flags;
        if block == 0 {
            block_flags |= crate::CHUNK_START as u32;
        }
        if block == (CHUNK_LEN / BLOCK_LEN) - 1 {
            block_flags |= crate::CHUNK_END as u32;
        }
        asm!(
            "call blake3_avx512_kernel2_16",
            inout("zmm0") state_regs[0],
            inout("zmm1") state_regs[1],
            inout("zmm2") state_regs[2],
            inout("zmm3") state_regs[3],
            inout("zmm4") state_regs[4],
            inout("zmm5") state_regs[5],
            inout("zmm6") state_regs[6],
            inout("zmm7") state_regs[7],
            in("zmm8") _mm512_set1_epi32(IV[0] as i32),
            in("zmm9") _mm512_set1_epi32(IV[1] as i32),
            in("zmm10") _mm512_set1_epi32(IV[2] as i32),
            in("zmm11") _mm512_set1_epi32(IV[3] as i32),
            in("zmm12") counters_low,
            in("zmm13") counters_high,
            in("zmm14") _mm512_set1_epi32(BLOCK_LEN as i32),
            in("zmm15") _mm512_set1_epi32(block_flags as i32),
            in("zmm16") message_regs[0],
            in("zmm17") message_regs[1],
            in("zmm18") message_regs[2],
            in("zmm19") message_regs[3],
            in("zmm20") message_regs[4],
            in("zmm21") message_regs[5],
            in("zmm22") message_regs[6],
            in("zmm23") message_regs[7],
            in("zmm24") message_regs[8],
            in("zmm25") message_regs[9],
            in("zmm26") message_regs[10],
            in("zmm27") message_regs[11],
            in("zmm28") message_regs[12],
            in("zmm29") message_regs[13],
            in("zmm30") message_regs[14],
            in("zmm31") message_regs[15],
        );
    }
    state_regs
}

pub fn portable_transpose<T, const N: usize>(words: &mut [[T; N]; N]) {
    if N == 0 {
        return;
    }
    for row in 0..(N - 1) {
        for col in (row + 1)..N {
            let (upper_rows, lower_rows) = words.split_at_mut(row + 1);
            mem::swap(
                &mut upper_rows[row][col],
                &mut lower_rows[col - row - 1][row],
            );
        }
    }
}

#[test]
fn test_portable_transpose() {
    let mut empty: [[u32; 0]; 0] = [];
    portable_transpose(&mut empty);

    let mut one = [[42u32]];
    portable_transpose(&mut one);
    assert_eq!(one[0][0], 42);

    let mut two = [[0u32, 1], [2, 3]];
    portable_transpose(&mut two);
    assert_eq!(two, [[0, 2], [1, 3]]);

    let mut three = [[0u32, 1, 2], [3, 4, 5], [6, 7, 8]];
    portable_transpose(&mut three);
    assert_eq!(three, [[0, 3, 6], [1, 4, 7], [2, 5, 8]]);

    let mut painted_bytes = [0; 100];
    crate::test::paint_test_input(&mut painted_bytes);
    let painted_rows: [[u8; 10]; 10] = unsafe { mem::transmute(painted_bytes) };
    let mut transposed_rows = painted_rows;
    portable_transpose(&mut transposed_rows);
    for row in 0..10 {
        for col in 0..10 {
            assert_eq!(painted_rows[row][col], transposed_rows[col][row]);
        }
    }
}

#[test]
fn test_chunks_16() {
    if !crate::platform::avx512_detected() {
        return;
    }
    let mut chunks = [0; 16 * CHUNK_LEN];
    crate::test::paint_test_input(&mut chunks);
    let key = [42, 43, 44, 45, 46, 47, 48, 49];
    let counter = (1 << 32) - 1;
    let outputs: [__m512i; 8] =
        unsafe { chunks_16(&chunks, &key, counter, crate::KEYED_HASH as u32) };

    let separate_chunks: [&[u8; CHUNK_LEN]; 16] =
        core::array::from_fn(|i| chunks[i * CHUNK_LEN..][..CHUNK_LEN].try_into().unwrap());
    let mut expected = [0u8; 16 * 32];
    crate::portable::hash_many(
        &separate_chunks,
        &key,
        counter,
        crate::IncrementCounter::Yes,
        crate::KEYED_HASH,
        crate::CHUNK_START,
        crate::CHUNK_END,
        &mut expected,
    );

    let outputs_u32: [[u32; 16]; 8] = unsafe { mem::transmute(outputs) };
    let mut compare = [0u8; 16 * 32];
    for vec in 0..8 {
        for word in 0..16 {
            compare[word * 32 + vec * 4..][..4]
                .copy_from_slice(&outputs_u32[vec][word].to_le_bytes());
        }
    }
    assert_eq!(expected, compare);
}

#[target_feature(enable = "avx512f,avx512vl")]
pub unsafe fn parents_16(
    left_children: &[__m512i; 8],
    right_children: &[__m512i; 8],
    key: &[u32; 8],
    flags: u32,
) -> [__m512i; 8] {
    let even_indexes = _mm512_setr_epi32(0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30);
    let odd_indexes = _mm512_setr_epi32(1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31);
    let mut state_regs = [
        _mm512_set1_epi32(key[0] as i32),
        _mm512_set1_epi32(key[1] as i32),
        _mm512_set1_epi32(key[2] as i32),
        _mm512_set1_epi32(key[3] as i32),
        _mm512_set1_epi32(key[4] as i32),
        _mm512_set1_epi32(key[5] as i32),
        _mm512_set1_epi32(key[6] as i32),
        _mm512_set1_epi32(key[7] as i32),
    ];
    asm!(
        "call blake3_avx512_kernel2_16",
        inout("zmm0") state_regs[0],
        inout("zmm1") state_regs[1],
        inout("zmm2") state_regs[2],
        inout("zmm3") state_regs[3],
        inout("zmm4") state_regs[4],
        inout("zmm5") state_regs[5],
        inout("zmm6") state_regs[6],
        inout("zmm7") state_regs[7],
        in("zmm8") _mm512_set1_epi32(IV[0] as i32),
        in("zmm9") _mm512_set1_epi32(IV[1] as i32),
        in("zmm10") _mm512_set1_epi32(IV[2] as i32),
        in("zmm11") _mm512_set1_epi32(IV[3] as i32),
        in("zmm12") _mm512_set1_epi32(0),
        in("zmm13") _mm512_set1_epi32(0),
        in("zmm14") _mm512_set1_epi32(BLOCK_LEN as i32),
        in("zmm15") _mm512_set1_epi32((flags | crate::PARENT as u32) as i32),
        in("zmm16") _mm512_permutex2var_epi32(left_children[0], even_indexes, right_children[0]),
        in("zmm17") _mm512_permutex2var_epi32(left_children[1], even_indexes, right_children[1]),
        in("zmm18") _mm512_permutex2var_epi32(left_children[2], even_indexes, right_children[2]),
        in("zmm19") _mm512_permutex2var_epi32(left_children[3], even_indexes, right_children[3]),
        in("zmm20") _mm512_permutex2var_epi32(left_children[4], even_indexes, right_children[4]),
        in("zmm21") _mm512_permutex2var_epi32(left_children[5], even_indexes, right_children[5]),
        in("zmm22") _mm512_permutex2var_epi32(left_children[6], even_indexes, right_children[6]),
        in("zmm23") _mm512_permutex2var_epi32(left_children[7], even_indexes, right_children[7]),
        in("zmm24") _mm512_permutex2var_epi32(left_children[0], odd_indexes, right_children[0]),
        in("zmm25") _mm512_permutex2var_epi32(left_children[1], odd_indexes, right_children[1]),
        in("zmm26") _mm512_permutex2var_epi32(left_children[2], odd_indexes, right_children[2]),
        in("zmm27") _mm512_permutex2var_epi32(left_children[3], odd_indexes, right_children[3]),
        in("zmm28") _mm512_permutex2var_epi32(left_children[4], odd_indexes, right_children[4]),
        in("zmm29") _mm512_permutex2var_epi32(left_children[5], odd_indexes, right_children[5]),
        in("zmm30") _mm512_permutex2var_epi32(left_children[6], odd_indexes, right_children[6]),
        in("zmm31") _mm512_permutex2var_epi32(left_children[7], odd_indexes, right_children[7]),
    );
    state_regs
}

#[test]
fn test_parents_16() {
    if !crate::platform::avx512_detected() {
        return;
    }

    // the (untransposed) bytes of 32 concatenated child CVs
    let mut child_bytes = [0; 32 * 32];
    crate::test::paint_test_input(&mut child_bytes);
    // the same bytes, reinterpreted as words
    let child_words: [u32; 32 * 8] = core::array::from_fn(|word| {
        u32::from_le_bytes(child_bytes[4 * word..][..4].try_into().unwrap())
    });
    // manually transpose the words into vector layout
    let mut left_child_vecs = [[0u32; 16]; 8];
    for cv in 0..16 {
        for word in 0..8 {
            left_child_vecs[word][cv] = child_words[8 * cv + word];
        }
    }
    let mut right_child_vecs = [[0u32; 16]; 8];
    for cv in 0..16 {
        for word in 0..8 {
            right_child_vecs[word][cv] = child_words[8 * (16 + cv) + word];
        }
    }

    let left_children: [__m512i; 8] = unsafe { mem::transmute(left_child_vecs) };
    let right_children: [__m512i; 8] = unsafe { mem::transmute(right_child_vecs) };
    let key = [42, 43, 44, 45, 46, 47, 48, 49];
    let outputs = unsafe {
        parents_16(
            &left_children,
            &right_children,
            &key,
            crate::KEYED_HASH as u32,
        )
    };
    let output_words: [[u32; 16]; 8] = unsafe { mem::transmute(outputs) };
    let mut untransposed_output_words = [[0; 8]; 16];
    for vec in 0..8 {
        for word in 0..16 {
            untransposed_output_words[word][vec] = output_words[vec][word];
        }
    }
    let untransposed_output_bytes: [u8; 16 * 32] =
        unsafe { mem::transmute(untransposed_output_words) };

    let child_blocks: [&[u8; 64]; 16] =
        core::array::from_fn(|block| child_bytes[64 * block..][..64].try_into().unwrap());
    let mut expected = [0u8; 16 * 32];
    crate::portable::hash_many(
        &child_blocks,
        &key,
        0,
        crate::IncrementCounter::No,
        crate::PARENT | crate::KEYED_HASH,
        0,
        0,
        &mut expected,
    );
    assert_eq!(expected, untransposed_output_bytes);
}

#[target_feature(enable = "avx512f,avx512vl")]
pub unsafe fn just_kernel2() {
    asm!(
        "call blake3_avx512_kernel2_16",
        in("zmm0") _mm512_set1_epi32(0),
        in("zmm1") _mm512_set1_epi32(0),
        in("zmm2") _mm512_set1_epi32(0),
        in("zmm3") _mm512_set1_epi32(0),
        in("zmm4") _mm512_set1_epi32(0),
        in("zmm5") _mm512_set1_epi32(0),
        in("zmm6") _mm512_set1_epi32(0),
        in("zmm7") _mm512_set1_epi32(0),
        in("zmm8") _mm512_set1_epi32(0),
        in("zmm9") _mm512_set1_epi32(0),
        in("zmm10") _mm512_set1_epi32(0),
        in("zmm11") _mm512_set1_epi32(0),
        in("zmm12") _mm512_set1_epi32(0),
        in("zmm13") _mm512_set1_epi32(0),
        in("zmm14") _mm512_set1_epi32(0),
        in("zmm15") _mm512_set1_epi32(0),
        in("zmm16") _mm512_set1_epi32(0),
        in("zmm17") _mm512_set1_epi32(0),
        in("zmm18") _mm512_set1_epi32(0),
        in("zmm19") _mm512_set1_epi32(0),
        in("zmm20") _mm512_set1_epi32(0),
        in("zmm21") _mm512_set1_epi32(0),
        in("zmm22") _mm512_set1_epi32(0),
        in("zmm23") _mm512_set1_epi32(0),
        in("zmm24") _mm512_set1_epi32(0),
        in("zmm25") _mm512_set1_epi32(0),
        in("zmm26") _mm512_set1_epi32(0),
        in("zmm27") _mm512_set1_epi32(0),
        in("zmm28") _mm512_set1_epi32(0),
        in("zmm29") _mm512_set1_epi32(0),
        in("zmm30") _mm512_set1_epi32(0),
        in("zmm31") _mm512_set1_epi32(0),
    );
}

#[inline]
#[target_feature(enable = "avx512f,avx512vl")]
unsafe fn xof_inner_16(
    block: &[u8; BLOCK_LEN],
    cv: &[u32; 8],
    counter: u64,
    block_len: u32,
    flags: u32,
) -> [__m512i; 16] {
    let (counters_low, counters_high) = incrementing_counter(counter);
    let mut state = [
        _mm512_set1_epi32(cv[0] as i32),
        _mm512_set1_epi32(cv[1] as i32),
        _mm512_set1_epi32(cv[2] as i32),
        _mm512_set1_epi32(cv[3] as i32),
        _mm512_set1_epi32(cv[4] as i32),
        _mm512_set1_epi32(cv[5] as i32),
        _mm512_set1_epi32(cv[6] as i32),
        _mm512_set1_epi32(cv[7] as i32),
        _mm512_set1_epi32(IV[0] as i32),
        _mm512_set1_epi32(IV[1] as i32),
        _mm512_set1_epi32(IV[2] as i32),
        _mm512_set1_epi32(IV[3] as i32),
        counters_low,
        counters_high,
        _mm512_set1_epi32(block_len as i32),
        _mm512_set1_epi32(flags as i32),
        _mm512_set1_epi32(i32::from_le_bytes(block[0..4].try_into().unwrap())),
        _mm512_set1_epi32(i32::from_le_bytes(block[4..8].try_into().unwrap())),
        _mm512_set1_epi32(i32::from_le_bytes(block[8..12].try_into().unwrap())),
        _mm512_set1_epi32(i32::from_le_bytes(block[12..16].try_into().unwrap())),
        _mm512_set1_epi32(i32::from_le_bytes(block[16..20].try_into().unwrap())),
        _mm512_set1_epi32(i32::from_le_bytes(block[20..24].try_into().unwrap())),
        _mm512_set1_epi32(i32::from_le_bytes(block[24..28].try_into().unwrap())),
        _mm512_set1_epi32(i32::from_le_bytes(block[28..32].try_into().unwrap())),
        _mm512_set1_epi32(i32::from_le_bytes(block[32..36].try_into().unwrap())),
        _mm512_set1_epi32(i32::from_le_bytes(block[36..40].try_into().unwrap())),
        _mm512_set1_epi32(i32::from_le_bytes(block[40..44].try_into().unwrap())),
        _mm512_set1_epi32(i32::from_le_bytes(block[44..48].try_into().unwrap())),
        _mm512_set1_epi32(i32::from_le_bytes(block[48..52].try_into().unwrap())),
        _mm512_set1_epi32(i32::from_le_bytes(block[52..56].try_into().unwrap())),
        _mm512_set1_epi32(i32::from_le_bytes(block[56..60].try_into().unwrap())),
        _mm512_set1_epi32(i32::from_le_bytes(block[60..64].try_into().unwrap())),
    ];
    asm!(
        "call blake3_avx512_kernel2_16",
        inout("zmm0") state[0],
        inout("zmm1") state[1],
        inout("zmm2") state[2],
        inout("zmm3") state[3],
        inout("zmm4") state[4],
        inout("zmm5") state[5],
        inout("zmm6") state[6],
        inout("zmm7") state[7],
        inout("zmm8") state[8],
        inout("zmm9") state[9],
        inout("zmm10") state[10],
        inout("zmm11") state[11],
        inout("zmm12") state[12],
        inout("zmm13") state[13],
        inout("zmm14") state[14],
        inout("zmm15") state[15],
        in("zmm16") state[16],
        in("zmm17") state[17],
        in("zmm18") state[18],
        in("zmm19") state[19],
        in("zmm20") state[20],
        in("zmm21") state[21],
        in("zmm22") state[22],
        in("zmm23") state[23],
        in("zmm24") state[24],
        in("zmm25") state[25],
        in("zmm26") state[26],
        in("zmm27") state[27],
        in("zmm28") state[28],
        in("zmm29") state[29],
        in("zmm30") state[30],
        in("zmm31") state[31],
    );

    // Perform the feed-forward xors into the upper half of the state. (The xors in the lower half
    // of the state are done in the kernel function.)
    for i in 0..8 {
        state[i + 8] = _mm512_xor_si512(state[i + 8], _mm512_set1_epi32(cv[i] as i32));
    }

    // Partially untranspose the state vectors. We'll use the same trick here as with message
    // loading, where we avoid doing any relatively expensive cross-128-bit-lane operations, and
    // instead we delay reordering 128-bit lanes until the store step.

    // Interleave 32-bit words, producing vectors like:
    // a0, a1, b0, b1, e0, e1, f0, f1, i0, i1, j0, j1, m0, m1, n0, n1
    let abefijmn_01 = _mm512_unpacklo_epi32(state[0], state[1]);
    let cdghklop_01 = _mm512_unpackhi_epi32(state[0], state[1]);
    let abefijmn_23 = _mm512_unpacklo_epi32(state[2], state[3]);
    let cdghklop_23 = _mm512_unpackhi_epi32(state[2], state[3]);
    let abefijmn_45 = _mm512_unpacklo_epi32(state[4], state[5]);
    let cdghklop_45 = _mm512_unpackhi_epi32(state[4], state[5]);
    let abefijmn_67 = _mm512_unpacklo_epi32(state[6], state[7]);
    let cdghklop_67 = _mm512_unpackhi_epi32(state[6], state[7]);
    let abefijmn_89 = _mm512_unpacklo_epi32(state[8], state[9]);
    let cdghklop_89 = _mm512_unpackhi_epi32(state[8], state[9]);
    let abefijmn_ab = _mm512_unpacklo_epi32(state[10], state[11]);
    let cdghklop_ab = _mm512_unpackhi_epi32(state[10], state[11]);
    let abefijmn_cd = _mm512_unpacklo_epi32(state[12], state[13]);
    let cdghklop_cd = _mm512_unpackhi_epi32(state[12], state[13]);
    let abefijmn_ef = _mm512_unpacklo_epi32(state[14], state[15]);
    let cdghklop_ef = _mm512_unpackhi_epi32(state[14], state[15]);

    // Interleave 64-bit words, producing vectors like:
    // a0, a1, a2, a3, e0, e1, e2, e3, i0, i1, i2, i3, m0, m1, m2, m3
    let aeim_0123 = _mm512_unpacklo_epi64(abefijmn_01, abefijmn_23);
    let bfjn_0123 = _mm512_unpackhi_epi64(abefijmn_01, abefijmn_23);
    let cgko_0123 = _mm512_unpacklo_epi64(cdghklop_01, cdghklop_23);
    let dhlp_0123 = _mm512_unpackhi_epi64(cdghklop_01, cdghklop_23);
    let aeim_4567 = _mm512_unpacklo_epi64(abefijmn_45, abefijmn_67);
    let bfjn_4567 = _mm512_unpackhi_epi64(abefijmn_45, abefijmn_67);
    let cgko_4567 = _mm512_unpacklo_epi64(cdghklop_45, cdghklop_67);
    let dhlp_4567 = _mm512_unpackhi_epi64(cdghklop_45, cdghklop_67);
    let aeim_89ab = _mm512_unpacklo_epi64(abefijmn_89, abefijmn_ab);
    let bfjn_89ab = _mm512_unpackhi_epi64(abefijmn_89, abefijmn_ab);
    let cgko_89ab = _mm512_unpacklo_epi64(cdghklop_89, cdghklop_ab);
    let dhlp_89ab = _mm512_unpackhi_epi64(cdghklop_89, cdghklop_ab);
    let aeim_cdef = _mm512_unpacklo_epi64(abefijmn_cd, abefijmn_ef);
    let bfjn_cdef = _mm512_unpackhi_epi64(abefijmn_cd, abefijmn_ef);
    let cgko_cdef = _mm512_unpacklo_epi64(cdghklop_cd, cdghklop_ef);
    let dhlp_cdef = _mm512_unpackhi_epi64(cdghklop_cd, cdghklop_ef);

    // Then interleave 128-bit lanes, producing vectors like:
    // a0, a1, a2, a3, i0, i1, i2, i3, a4, a5, a6, a7, i4, i5, i6, i7
    const LO_LANES: i32 = 0x88; // 0b10001000 = (0, 2, 0, 2)
    const HI_LANES: i32 = 0xdd; // 0b11011101 = (1, 3, 1, 3)
    let ai_01234567 = _mm512_shuffle_i32x4(aeim_0123, aeim_4567, LO_LANES);
    let bj_01234567 = _mm512_shuffle_i32x4(bfjn_0123, bfjn_4567, LO_LANES);
    let ck_01234567 = _mm512_shuffle_i32x4(cgko_0123, cgko_4567, LO_LANES);
    let dl_01234567 = _mm512_shuffle_i32x4(dhlp_0123, dhlp_4567, LO_LANES);
    let em_01234567 = _mm512_shuffle_i32x4(aeim_0123, aeim_4567, HI_LANES);
    let fn_01234567 = _mm512_shuffle_i32x4(bfjn_0123, bfjn_4567, HI_LANES);
    let go_01234567 = _mm512_shuffle_i32x4(cgko_0123, cgko_4567, HI_LANES);
    let hp_01234567 = _mm512_shuffle_i32x4(dhlp_0123, dhlp_4567, HI_LANES);
    let ai_89abcdef = _mm512_shuffle_i32x4(aeim_89ab, aeim_cdef, LO_LANES);
    let bj_89abcdef = _mm512_shuffle_i32x4(bfjn_89ab, bfjn_cdef, LO_LANES);
    let ck_89abcdef = _mm512_shuffle_i32x4(cgko_89ab, cgko_cdef, LO_LANES);
    let dl_89abcdef = _mm512_shuffle_i32x4(dhlp_89ab, dhlp_cdef, LO_LANES);
    let em_89abcdef = _mm512_shuffle_i32x4(aeim_89ab, aeim_cdef, HI_LANES);
    let fn_89abcdef = _mm512_shuffle_i32x4(bfjn_89ab, bfjn_cdef, HI_LANES);
    let go_89abcdef = _mm512_shuffle_i32x4(cgko_89ab, cgko_cdef, HI_LANES);
    let hp_89abcdef = _mm512_shuffle_i32x4(dhlp_89ab, dhlp_cdef, HI_LANES);

    // Finally interleave 128-bit lanes again (the same permutation as the previous pass, but
    // different inputs), producing vectors like:
    //
    // a0, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14, a15
    [
        _mm512_shuffle_i32x4(ai_01234567, ai_89abcdef, LO_LANES), // a_0123456789abcdef
        _mm512_shuffle_i32x4(bj_01234567, bj_89abcdef, LO_LANES), // b_0123456789abcdef
        _mm512_shuffle_i32x4(ck_01234567, ck_89abcdef, LO_LANES), // c_0123456789abcdef
        _mm512_shuffle_i32x4(dl_01234567, dl_89abcdef, LO_LANES), // d_0123456789abcdef
        _mm512_shuffle_i32x4(em_01234567, em_89abcdef, LO_LANES), // e_0123456789abcdef
        _mm512_shuffle_i32x4(fn_01234567, fn_89abcdef, LO_LANES), // f_0123456789abcdef
        _mm512_shuffle_i32x4(go_01234567, go_89abcdef, LO_LANES), // g_0123456789abcdef
        _mm512_shuffle_i32x4(hp_01234567, hp_89abcdef, LO_LANES), // h_0123456789abcdef
        _mm512_shuffle_i32x4(ai_01234567, ai_89abcdef, HI_LANES), // i_0123456789abcdef
        _mm512_shuffle_i32x4(bj_01234567, bj_89abcdef, HI_LANES), // j_0123456789abcdef
        _mm512_shuffle_i32x4(ck_01234567, ck_89abcdef, HI_LANES), // k_0123456789abcdef
        _mm512_shuffle_i32x4(dl_01234567, dl_89abcdef, HI_LANES), // l_0123456789abcdef
        _mm512_shuffle_i32x4(em_01234567, em_89abcdef, HI_LANES), // m_0123456789abcdef
        _mm512_shuffle_i32x4(fn_01234567, fn_89abcdef, HI_LANES), // n_0123456789abcdef
        _mm512_shuffle_i32x4(go_01234567, go_89abcdef, HI_LANES), // o_0123456789abcdef
        _mm512_shuffle_i32x4(hp_01234567, hp_89abcdef, HI_LANES), // p_0123456789abcdef
    ]
}

#[target_feature(enable = "avx512f,avx512vl")]
pub unsafe fn xof_16(
    block: &[u8; BLOCK_LEN],
    cv: &[u32; 8],
    counter: u64,
    block_len: u32,
    flags: u32,
    output: &mut [u8; BLOCK_LEN * 16],
) {
    let vecs = xof_inner_16(block, cv, counter, block_len, flags);
    #[inline(always)]
    unsafe fn write_vec(vecs: &[__m512i; 16], out: *mut u8, i: usize) {
        let addr = out.add(64 * i) as *mut i32;
        _mm512_storeu_si512(addr, vecs[i]);
    }
    write_vec(&vecs, output.as_mut_ptr(), 0x0);
    write_vec(&vecs, output.as_mut_ptr(), 0x1);
    write_vec(&vecs, output.as_mut_ptr(), 0x2);
    write_vec(&vecs, output.as_mut_ptr(), 0x3);
    write_vec(&vecs, output.as_mut_ptr(), 0x4);
    write_vec(&vecs, output.as_mut_ptr(), 0x5);
    write_vec(&vecs, output.as_mut_ptr(), 0x6);
    write_vec(&vecs, output.as_mut_ptr(), 0x7);
    write_vec(&vecs, output.as_mut_ptr(), 0x8);
    write_vec(&vecs, output.as_mut_ptr(), 0x9);
    write_vec(&vecs, output.as_mut_ptr(), 0xa);
    write_vec(&vecs, output.as_mut_ptr(), 0xb);
    write_vec(&vecs, output.as_mut_ptr(), 0xc);
    write_vec(&vecs, output.as_mut_ptr(), 0xd);
    write_vec(&vecs, output.as_mut_ptr(), 0xe);
    write_vec(&vecs, output.as_mut_ptr(), 0xf);
}

#[test]
fn test_xof_16() {
    if !crate::platform::avx512_detected() {
        return;
    }
    let block_len = 63;
    let mut block = [0; 64];
    crate::test::paint_test_input(&mut block[..block_len as usize]); // all but last byte
    let flags = (crate::CHUNK_START | crate::CHUNK_END | crate::ROOT) as u32;

    // Test a few different initial counter values.
    // - 0: The base case.
    // - u32::MAX: The low word of the counter overflows for all inputs except the first.
    // - i32::MAX: *No* overflow. But carry bugs in tricky SIMD code can screw this up, if you XOR
    //   when you're supposed to ANDNOT...
    let initial_counters = [0, u32::MAX as u64, i32::MAX as u64];
    for counter in initial_counters {
        dbg!(counter);
        let mut output = [0xff; BLOCK_LEN * 16];
        unsafe {
            xof_16(&block, IV, counter, block_len, flags, &mut output);
        }
        for i in 0..16 {
            dbg!(i);
            let expected = crate::portable::compress_xof(
                IV,
                &block,
                block_len as u8,
                counter + i as u64,
                flags as u8,
            );
            assert_eq!(expected, output[BLOCK_LEN * i..][..BLOCK_LEN]);
        }
    }
}

#[target_feature(enable = "avx512f,avx512vl")]
pub unsafe fn xof_xor_16(
    block: &[u8; BLOCK_LEN],
    cv: &[u32; 8],
    counter: u64,
    block_len: u32,
    flags: u32,
    output: &mut [u8; BLOCK_LEN * 16],
) {
    let vecs = xof_inner_16(block, cv, counter, block_len, flags);
    #[inline(always)]
    unsafe fn write_vec(vecs: &[__m512i; 16], out: *mut u8, i: usize) {
        let addr = out.add(64 * i) as *mut i32;
        _mm512_storeu_si512(addr, _mm512_xor_si512(vecs[i], _mm512_loadu_si512(addr)));
    }
    write_vec(&vecs, output.as_mut_ptr(), 0x0);
    write_vec(&vecs, output.as_mut_ptr(), 0x1);
    write_vec(&vecs, output.as_mut_ptr(), 0x2);
    write_vec(&vecs, output.as_mut_ptr(), 0x3);
    write_vec(&vecs, output.as_mut_ptr(), 0x4);
    write_vec(&vecs, output.as_mut_ptr(), 0x5);
    write_vec(&vecs, output.as_mut_ptr(), 0x6);
    write_vec(&vecs, output.as_mut_ptr(), 0x7);
    write_vec(&vecs, output.as_mut_ptr(), 0x8);
    write_vec(&vecs, output.as_mut_ptr(), 0x9);
    write_vec(&vecs, output.as_mut_ptr(), 0xa);
    write_vec(&vecs, output.as_mut_ptr(), 0xb);
    write_vec(&vecs, output.as_mut_ptr(), 0xc);
    write_vec(&vecs, output.as_mut_ptr(), 0xd);
    write_vec(&vecs, output.as_mut_ptr(), 0xe);
    write_vec(&vecs, output.as_mut_ptr(), 0xf);
}

#[test]
fn test_xof_xor_16() {
    if !crate::platform::avx512_detected() {
        return;
    }
    let block_len = 63;
    let mut block = [0; 64];
    crate::test::paint_test_input(&mut block[..block_len as usize]); // all but last byte
    let flags = (crate::CHUNK_START | crate::CHUNK_END | crate::ROOT) as u32;

    // Test a few different initial counter values.
    // - 0: The base case.
    // - u32::MAX: The low word of the counter overflows for all inputs except the first.
    // - i32::MAX: *No* overflow. But carry bugs in tricky SIMD code can screw this up, if you XOR
    //   when you're supposed to ANDNOT...
    let initial_counters = [0, u32::MAX as u64, i32::MAX as u64];
    for counter in initial_counters {
        dbg!(counter);
        let mut initial_output_buffer = [0; BLOCK_LEN * 16];
        crate::test::paint_test_input(&mut initial_output_buffer);
        let mut output = initial_output_buffer;
        unsafe {
            xof_xor_16(&block, IV, counter, block_len, flags, &mut output);
        }
        for i in 0..16 {
            dbg!(i);
            let mut expected_block = crate::portable::compress_xof(
                IV,
                &block,
                block_len as u8,
                counter + i as u64,
                flags as u8,
            );
            for j in 0..expected_block.len() {
                expected_block[j] ^= initial_output_buffer[i * BLOCK_LEN + j];
            }
            assert_eq!(expected_block, output[BLOCK_LEN * i..][..BLOCK_LEN]);
        }
    }
}