+1

.gitignore

··· 1 1 + _build

+1

dune

··· 1 1 + (vendored_dirs vendor)

+19

dune-project

··· 1 1 + (lang dune 3.21) 2 2 + (name mlxxhash) 3 3 + 4 4 + (generate_opam_files true) 5 5 + 6 6 + (license ISC) 7 7 + (authors "Anil Madhavapeddy <anil@recoil.org>") 8 8 + (maintainers "Anil Madhavapeddy <anil@recoil.org>") 9 9 + (source (tangled anil.recoil.org/ocaml-xxhash)) 10 10 + 11 11 + (package 12 12 + (name mlxxhash) 13 13 + (synopsis "Pure OCaml implementation of xxHash-64") 14 14 + (description 15 15 + "A pure OCaml implementation of the xxHash-64 non-cryptographic hash algorithm. 16 16 + Provides both one-shot and streaming APIs for hashing bytes and strings.") 17 17 + (depends 18 18 + (ocaml (>= 5.2.0)) 19 19 + (alcotest (and :with-test (>= 1.7.0)))))

+33

mlxxhash.opam

··· 1 1 + # This file is generated by dune, edit dune-project instead 2 2 + opam-version: "2.0" 3 3 + synopsis: "Pure OCaml implementation of xxHash-64" 4 4 + description: """ 5 5 + A pure OCaml implementation of the xxHash-64 non-cryptographic hash algorithm. 6 6 + Provides both one-shot and streaming APIs for hashing bytes and strings.""" 7 7 + maintainer: ["Anil Madhavapeddy <anil@recoil.org>"] 8 8 + authors: ["Anil Madhavapeddy <anil@recoil.org>"] 9 9 + license: "ISC" 10 10 + homepage: "https://tangled.org/anil.recoil.org/ocaml-xxhash" 11 11 + bug-reports: "https://tangled.org/anil.recoil.org/ocaml-xxhash/issues" 12 12 + depends: [ 13 13 + "dune" {>= "3.21"} 14 14 + "ocaml" {>= "5.2.0"} 15 15 + "alcotest" {with-test & >= "1.7.0"} 16 16 + "odoc" {with-doc} 17 17 + ] 18 18 + build: [ 19 19 + ["dune" "subst"] {dev} 20 20 + [ 21 21 + "dune" 22 22 + "build" 23 23 + "-p" 24 24 + name 25 25 + "-j" 26 26 + jobs 27 27 + "@install" 28 28 + "@runtest" {with-test} 29 29 + "@doc" {with-doc} 30 30 + ] 31 31 + ] 32 32 + dev-repo: "git+https://tangled.org/anil.recoil.org/ocaml-xxhash" 33 33 + x-maintenance-intent: ["(latest)"]

+3

src/dune

··· 1 1 + (library 2 2 + (name xxhash) 3 3 + (public_name mlxxhash))

+264

src/xxhash.ml

··· 1 1 + (** xxHash-64 - Pure OCaml implementation. 2 2 + 3 3 + This implements the xxHash64 algorithm designed by Yann Collet. 4 4 + xxHash is an extremely fast non-cryptographic hash algorithm with 5 5 + excellent distribution properties. *) 6 6 + 7 7 + (* Constants *) 8 8 + let prime64_1 = 0x9E3779B185EBCA87L 9 9 + let prime64_2 = 0xC2B2AE3D27D4EB4FL 10 10 + let prime64_3 = 0x165667B19E3779F9L 11 11 + let prime64_4 = 0x85EBCA77C2B2AE63L 12 12 + let prime64_5 = 0x27D4EB2F165667C5L 13 13 + 14 14 + (* Helper functions *) 15 15 + let[@inline] rotl64 x r = 16 16 + Int64.(logor (shift_left x r) (shift_right_logical x (64 - r))) 17 17 + 18 18 + let[@inline] mix1 acc v = 19 19 + let open Int64 in 20 20 + let acc = add acc (mul v prime64_2) in 21 21 + let acc = rotl64 acc 31 in 22 22 + mul acc prime64_1 23 23 + 24 24 + let[@inline] mix2 acc v = 25 25 + let open Int64 in 26 26 + let v = mul v prime64_2 in 27 27 + let v = rotl64 v 31 in 28 28 + let v = mul v prime64_1 in 29 29 + let acc = logxor acc v in 30 30 + add (mul acc prime64_1) prime64_4 31 31 + 32 32 + let[@inline] avalanche h = 33 33 + let open Int64 in 34 34 + let h = logxor h (shift_right_logical h 33) in 35 35 + let h = mul h prime64_2 in 36 36 + let h = logxor h (shift_right_logical h 29) in 37 37 + let h = mul h prime64_3 in 38 38 + logxor h (shift_right_logical h 32) 39 39 + 40 40 + (** Compute xxHash-64 of bytes with given seed *) 41 41 + let hash64 ?(seed=0L) src ~pos ~len = 42 42 + let open Int64 in 43 43 + let end_pos = pos + len in 44 44 + 45 45 + let h = ref ( 46 46 + if len >= 32 then begin 47 47 + (* Initialize accumulators *) 48 48 + let v1 = ref (add (add seed prime64_1) prime64_2) in 49 49 + let v2 = ref (add seed prime64_2) in 50 50 + let v3 = ref seed in 51 51 + let v4 = ref (sub seed prime64_1) in 52 52 + 53 53 + (* Process 32-byte blocks *) 54 54 + let p = ref pos in 55 55 + while !p + 32 <= end_pos do 56 56 + v1 := mix1 !v1 (Bytes.get_int64_le src !p); 57 57 + v2 := mix1 !v2 (Bytes.get_int64_le src (!p + 8)); 58 58 + v3 := mix1 !v3 (Bytes.get_int64_le src (!p + 16)); 59 59 + v4 := mix1 !v4 (Bytes.get_int64_le src (!p + 24)); 60 60 + p := !p + 32 61 61 + done; 62 62 + 63 63 + (* Merge accumulators *) 64 64 + let h = add 65 65 + (add (rotl64 !v1 1) (rotl64 !v2 7)) 66 66 + (add (rotl64 !v3 12) (rotl64 !v4 18)) in 67 67 + let h = mix2 h !v1 in 68 68 + let h = mix2 h !v2 in 69 69 + let h = mix2 h !v3 in 70 70 + mix2 h !v4 71 71 + end else 72 72 + add seed prime64_5 73 73 + ) in 74 74 + 75 75 + h := add !h (of_int len); 76 76 + 77 77 + (* Process remaining 8-byte chunks *) 78 78 + let p = ref (if len >= 32 then pos + (len / 32) * 32 else pos) in 79 79 + while !p + 8 <= end_pos do 80 80 + let k = Bytes.get_int64_le src !p in 81 81 + let k = mul k prime64_2 in 82 82 + let k = rotl64 k 31 in 83 83 + let k = mul k prime64_1 in 84 84 + h := logxor !h k; 85 85 + h := rotl64 !h 27; 86 86 + h := add (mul !h prime64_1) prime64_4; 87 87 + p := !p + 8 88 88 + done; 89 89 + 90 90 + (* Process remaining 4-byte chunk *) 91 91 + if !p + 4 <= end_pos then begin 92 92 + let k = of_int (Bytes.get_int32_le src !p |> Int32.to_int) in 93 93 + let k = logand k 0xFFFFFFFFL in (* Make unsigned *) 94 94 + h := logxor !h (mul k prime64_1); 95 95 + h := rotl64 !h 23; 96 96 + h := add (mul !h prime64_2) prime64_3; 97 97 + p := !p + 4 98 98 + end; 99 99 + 100 100 + (* Process remaining bytes *) 101 101 + while !p < end_pos do 102 102 + let k = of_int (Bytes.get_uint8 src !p) in 103 103 + h := logxor !h (mul k prime64_5); 104 104 + h := rotl64 !h 11; 105 105 + h := mul !h prime64_1; 106 106 + incr p 107 107 + done; 108 108 + 109 109 + avalanche !h 110 110 + 111 111 + let hash64_string ?seed s = 112 112 + let src = Bytes.unsafe_of_string s in 113 113 + hash64 ?seed src ~pos:0 ~len:(String.length s) 114 114 + 115 115 + (** Compute xxHash-64 and return lower 32 bits (for zstd checksum) *) 116 116 + let hash32 ?seed src ~pos ~len = 117 117 + let h = hash64 ?seed src ~pos ~len in 118 118 + Int64.to_int32 (Int64.logand h 0xFFFFFFFFL) 119 119 + 120 120 + let hash32_string ?seed s = 121 121 + let src = Bytes.unsafe_of_string s in 122 122 + hash32 ?seed src ~pos:0 ~len:(String.length s) 123 123 + 124 124 + (** Streaming hasher state *) 125 125 + type state = { 126 126 + mutable v1 : int64; 127 127 + mutable v2 : int64; 128 128 + mutable v3 : int64; 129 129 + mutable v4 : int64; 130 130 + mutable total_len : int; 131 131 + buffer : bytes; 132 132 + mutable buf_len : int; 133 133 + seed : int64; 134 134 + } 135 135 + 136 136 + let create_state ?(seed=0L) () = 137 137 + let open Int64 in 138 138 + { 139 139 + v1 = add (add seed prime64_1) prime64_2; 140 140 + v2 = add seed prime64_2; 141 141 + v3 = seed; 142 142 + v4 = sub seed prime64_1; 143 143 + total_len = 0; 144 144 + buffer = Bytes.create 32; 145 145 + buf_len = 0; 146 146 + seed; 147 147 + } 148 148 + 149 149 + let reset ?(seed=0L) state = 150 150 + let open Int64 in 151 151 + state.v1 <- add (add seed prime64_1) prime64_2; 152 152 + state.v2 <- add seed prime64_2; 153 153 + state.v3 <- seed; 154 154 + state.v4 <- sub seed prime64_1; 155 155 + state.total_len <- 0; 156 156 + state.buf_len <- 0 157 157 + 158 158 + let copy_state state = 159 159 + { 160 160 + v1 = state.v1; 161 161 + v2 = state.v2; 162 162 + v3 = state.v3; 163 163 + v4 = state.v4; 164 164 + total_len = state.total_len; 165 165 + buffer = Bytes.copy state.buffer; 166 166 + buf_len = state.buf_len; 167 167 + seed = state.seed; 168 168 + } 169 169 + 170 170 + let update state src ~pos ~len = 171 171 + let end_pos = pos + len in 172 172 + state.total_len <- state.total_len + len; 173 173 + 174 174 + let p = ref pos in 175 175 + 176 176 + (* Fill buffer if we have partial data *) 177 177 + if state.buf_len > 0 then begin 178 178 + let to_copy = min (32 - state.buf_len) len in 179 179 + Bytes.blit src !p state.buffer state.buf_len to_copy; 180 180 + state.buf_len <- state.buf_len + to_copy; 181 181 + p := !p + to_copy; 182 182 + 183 183 + if state.buf_len = 32 then begin 184 184 + state.v1 <- mix1 state.v1 (Bytes.get_int64_le state.buffer 0); 185 185 + state.v2 <- mix1 state.v2 (Bytes.get_int64_le state.buffer 8); 186 186 + state.v3 <- mix1 state.v3 (Bytes.get_int64_le state.buffer 16); 187 187 + state.v4 <- mix1 state.v4 (Bytes.get_int64_le state.buffer 24); 188 188 + state.buf_len <- 0 189 189 + end 190 190 + end; 191 191 + 192 192 + (* Process 32-byte blocks *) 193 193 + while !p + 32 <= end_pos do 194 194 + state.v1 <- mix1 state.v1 (Bytes.get_int64_le src !p); 195 195 + state.v2 <- mix1 state.v2 (Bytes.get_int64_le src (!p + 8)); 196 196 + state.v3 <- mix1 state.v3 (Bytes.get_int64_le src (!p + 16)); 197 197 + state.v4 <- mix1 state.v4 (Bytes.get_int64_le src (!p + 24)); 198 198 + p := !p + 32 199 199 + done; 200 200 + 201 201 + (* Buffer remaining *) 202 202 + if !p < end_pos then begin 203 203 + let remaining = end_pos - !p in 204 204 + Bytes.blit src !p state.buffer state.buf_len remaining; 205 205 + state.buf_len <- state.buf_len + remaining 206 206 + end 207 207 + 208 208 + let update_string state s = 209 209 + let src = Bytes.unsafe_of_string s in 210 210 + update state src ~pos:0 ~len:(String.length s) 211 211 + 212 212 + let finalize state = 213 213 + let open Int64 in 214 214 + 215 215 + let h = ref ( 216 216 + if state.total_len >= 32 then begin 217 217 + let h = add 218 218 + (add (rotl64 state.v1 1) (rotl64 state.v2 7)) 219 219 + (add (rotl64 state.v3 12) (rotl64 state.v4 18)) in 220 220 + let h = mix2 h state.v1 in 221 221 + let h = mix2 h state.v2 in 222 222 + let h = mix2 h state.v3 in 223 223 + mix2 h state.v4 224 224 + end else 225 225 + add state.v3 prime64_5 (* v3 holds seed *) 226 226 + ) in 227 227 + 228 228 + h := add !h (of_int state.total_len); 229 229 + 230 230 + (* Process buffered data *) 231 231 + let p = ref 0 in 232 232 + while !p + 8 <= state.buf_len do 233 233 + let k = Bytes.get_int64_le state.buffer !p in 234 234 + let k = mul k prime64_2 in 235 235 + let k = rotl64 k 31 in 236 236 + let k = mul k prime64_1 in 237 237 + h := logxor !h k; 238 238 + h := rotl64 !h 27; 239 239 + h := add (mul !h prime64_1) prime64_4; 240 240 + p := !p + 8 241 241 + done; 242 242 + 243 243 + if !p + 4 <= state.buf_len then begin 244 244 + let k = of_int (Bytes.get_int32_le state.buffer !p |> Int32.to_int) in 245 245 + let k = logand k 0xFFFFFFFFL in 246 246 + h := logxor !h (mul k prime64_1); 247 247 + h := rotl64 !h 23; 248 248 + h := add (mul !h prime64_2) prime64_3; 249 249 + p := !p + 4 250 250 + end; 251 251 + 252 252 + while !p < state.buf_len do 253 253 + let k = of_int (Bytes.get_uint8 state.buffer !p) in 254 254 + h := logxor !h (mul k prime64_5); 255 255 + h := rotl64 !h 11; 256 256 + h := mul !h prime64_1; 257 257 + incr p 258 258 + done; 259 259 + 260 260 + avalanche !h 261 261 + 262 262 + let finalize32 state = 263 263 + let h = finalize state in 264 264 + Int64.to_int32 (Int64.logand h 0xFFFFFFFFL)

+91

src/xxhash.mli

··· 1 1 + (** xxHash - Fast non-cryptographic hash functions. 2 2 + 3 3 + This is a pure OCaml implementation of the xxHash family of hash functions, 4 4 + originally designed by Yann Collet. xxHash provides extremely fast hashing 5 5 + with excellent distribution properties. 6 6 + 7 7 + {1 Quick Start} 8 8 + 9 9 + {[ 10 10 + (* Hash a string *) 11 11 + let hash = Xxhash.hash64_string "Hello, World!" 12 12 + 13 13 + (* Hash bytes with explicit range *) 14 14 + let bytes = Bytes.of_string "Hello, World!" 15 15 + let hash = Xxhash.hash64 bytes ~pos:0 ~len:13 16 16 + 17 17 + (* Use streaming API for large data *) 18 18 + let state = Xxhash.create_state () in 19 19 + Xxhash.update state chunk1 ~pos:0 ~len:(Bytes.length chunk1); 20 20 + Xxhash.update state chunk2 ~pos:0 ~len:(Bytes.length chunk2); 21 21 + let hash = Xxhash.finalize state 22 22 + ]} 23 23 + 24 24 + {1 Hash Variants} 25 25 + 26 26 + - {!hash64}: 64-bit hash, best for general use 27 27 + - {!hash32}: Lower 32 bits of 64-bit hash (used by zstd) 28 28 + 29 29 + {1 Streaming API} 30 30 + 31 31 + For hashing data that doesn't fit in memory or arrives incrementally: 32 32 + - {!create_state}: Create a new streaming state 33 33 + - {!update}: Feed data into the state 34 34 + - {!finalize}: Get the final hash value *) 35 35 + 36 36 + (** {1 One-shot Hashing} *) 37 37 + 38 38 + val hash64 : ?seed:int64 -> bytes -> pos:int -> len:int -> int64 39 39 + (** [hash64 ?seed bytes ~pos ~len] computes the xxHash-64 of [len] bytes 40 40 + from [bytes] starting at [pos]. 41 41 + 42 42 + @param seed Optional seed value (default: 0) *) 43 43 + 44 44 + val hash64_string : ?seed:int64 -> string -> int64 45 45 + (** [hash64_string ?seed s] computes the xxHash-64 of string [s]. *) 46 46 + 47 47 + val hash32 : ?seed:int64 -> bytes -> pos:int -> len:int -> int32 48 48 + (** [hash32 ?seed bytes ~pos ~len] computes xxHash-64 and returns the 49 49 + lower 32 bits. This is the variant used by zstd for content checksums. *) 50 50 + 51 51 + val hash32_string : ?seed:int64 -> string -> int32 52 52 + (** [hash32_string ?seed s] computes the lower 32 bits of xxHash-64. *) 53 53 + 54 54 + (** {1 Streaming API} *) 55 55 + 56 56 + (** Streaming hasher state. *) 57 57 + type state 58 58 + 59 59 + val create_state : ?seed:int64 -> unit -> state 60 60 + (** [create_state ?seed ()] creates a new streaming hash state. 61 61 + 62 62 + @param seed Optional seed value (default: 0) *) 63 63 + 64 64 + val reset : ?seed:int64 -> state -> unit 65 65 + (** [reset ?seed state] resets the state for reuse with a new hash. 66 66 + 67 67 + @param seed Optional new seed value (default: 0) *) 68 68 + 69 69 + val update : state -> bytes -> pos:int -> len:int -> unit 70 70 + (** [update state bytes ~pos ~len] feeds [len] bytes from [bytes] 71 71 + starting at [pos] into the hash state. 72 72 + 73 73 + Can be called multiple times to hash data incrementally. *) 74 74 + 75 75 + val update_string : state -> string -> unit 76 76 + (** [update_string state s] feeds string [s] into the hash state. *) 77 77 + 78 78 + val finalize : state -> int64 79 79 + (** [finalize state] returns the 64-bit hash value. 80 80 + 81 81 + The state can still be used after finalization - subsequent calls 82 82 + to {!finalize} return the same value until {!update} is called. *) 83 83 + 84 84 + val finalize32 : state -> int32 85 85 + (** [finalize32 state] returns the lower 32 bits of the hash. *) 86 86 + 87 87 + (** {1 Utilities} *) 88 88 + 89 89 + val copy_state : state -> state 90 90 + (** [copy_state state] creates an independent copy of the hash state. 91 91 + Useful for computing hashes of data with common prefixes. *)

+3

test/dune

··· 1 1 + (test 2 2 + (name test_xxhash) 3 3 + (libraries xxhash alcotest))

+430

test/test_xxhash.ml

··· 1 1 + (** Tests for xxHash-64 implementation. 2 2 + 3 3 + This test suite verifies: 4 4 + 1. Internal consistency (streaming vs one-shot produce same results) 5 5 + 2. Known reference values from the official xxHash test vectors 6 6 + 3. Boundary conditions (32-byte blocks, various lengths) 7 7 + 4. Reference validation against the C implementation test suite *) 8 8 + 9 9 + (* ===== Reference Test Buffer Generation ===== 10 10 + 11 11 + The official xxHash test suite uses a deterministic pseudorandom buffer. 12 12 + This must match exactly: 13 13 + 14 14 + PRIME32 = 2654435761 15 15 + PRIME64 = 11400714785074694797 16 16 + 17 17 + buffer[i] = (byteGen >> 56) & 0xFF 18 18 + byteGen *= PRIME64 19 19 + 20 20 + Starting with byteGen = PRIME32 *) 21 21 + 22 22 + let prime32 = 2654435761L (* 0x9E3779B1 as unsigned 32-bit *) 23 23 + (* PRIME64 for test buffer = 11400714785074694797 = 0x9e3779b185ebca8d *) 24 24 + (* Note: This is different from the xxHash algorithm's prime constants! *) 25 25 + let prime64_gen = 0x9e3779b185ebca8dL 26 26 + 27 27 + (** Generate the reference test buffer used by the xxHash test suite *) 28 28 + let fill_test_buffer len = 29 29 + let buf = Bytes.create len in 30 30 + let rec loop i gen = 31 31 + if i >= len then buf 32 32 + else begin 33 33 + Bytes.set_uint8 buf i (Int64.(to_int (shift_right_logical gen 56))); 34 34 + loop (i + 1) Int64.(mul gen prime64_gen) 35 35 + end 36 36 + in 37 37 + loop 0 prime32 38 38 + 39 39 + (* ===== Official XXH64 Test Vectors ===== 40 40 + 41 41 + Format: (len, seed, expected_hash) 42 42 + From vendor/git/xxHash/tests/sanity_test_vectors.h *) 43 43 + 44 44 + (* Comprehensive test vectors from official xxHash test suite. 45 45 + Format: (length, seed, expected_hash) 46 46 + All lengths from 0 to 128, plus key lengths 256, 512, 1024, 4096. *) 47 47 + let xxh64_test_vectors = [ 48 48 + (* Lengths 0-31: small inputs < block size *) 49 49 + (0, 0x0000000000000000L, 0xEF46DB3751D8E999L); 50 50 + (0, 0x000000009E3779B1L, 0xAC75FDA2929B17EFL); 51 51 + (1, 0x0000000000000000L, 0xE934A84ADB052768L); 52 52 + (1, 0x000000009E3779B1L, 0x5014607643A9B4C3L); 53 53 + (2, 0x0000000000000000L, 0x5D48CD60A77E23FFL); 54 54 + (2, 0x000000009E3779B1L, 0x9E93152232D54A39L); 55 55 + (3, 0x0000000000000000L, 0xFF7E1959CB50794AL); 56 56 + (3, 0x000000009E3779B1L, 0xAA8584E83660F7D1L); 57 57 + (4, 0x0000000000000000L, 0x9136A0DCA57457EEL); 58 58 + (4, 0x000000009E3779B1L, 0xCAAB286BD8E9FDB5L); 59 59 + (5, 0x0000000000000000L, 0x9B046FB1397F09A5L); 60 60 + (5, 0x000000009E3779B1L, 0x2AF5249930F984ECL); 61 61 + (6, 0x0000000000000000L, 0xC72565B7154268A8L); 62 62 + (6, 0x000000009E3779B1L, 0xCA4C6723580E8EF6L); 63 63 + (7, 0x0000000000000000L, 0x6C83909A9F01ED25L); 64 64 + (7, 0x000000009E3779B1L, 0xF98D03B1AD6F9293L); 65 65 + (8, 0x0000000000000000L, 0xCDBCF538E71D1348L); 66 66 + (8, 0x000000009E3779B1L, 0xFE0C047A5353CDACL); 67 67 + (9, 0x0000000000000000L, 0x554B1AE991EDA6B6L); 68 68 + (9, 0x000000009E3779B1L, 0x7908265248F6D73FL); 69 69 + (10, 0x0000000000000000L, 0x5D00E7351392EA84L); 70 70 + (10, 0x000000009E3779B1L, 0x2A8AE16B86CD2F12L); 71 71 + (11, 0x0000000000000000L, 0x6345D5746F35DA70L); 72 72 + (11, 0x000000009E3779B1L, 0xEAA08A8C8BE3CCCFL); 73 73 + (12, 0x0000000000000000L, 0x0723BF50086EAD9AL); 74 74 + (12, 0x000000009E3779B1L, 0x8252819F4E506951L); 75 75 + (13, 0x0000000000000000L, 0xC2E5013E3C40BCF7L); 76 76 + (13, 0x000000009E3779B1L, 0x4DF437A291CB1039L); 77 77 + (14, 0x0000000000000000L, 0x8282DCC4994E35C8L); 78 78 + (14, 0x000000009E3779B1L, 0xC3BD6BF63DEB6DF0L); 79 79 + (15, 0x0000000000000000L, 0x180719316D622D84L); 80 80 + (15, 0x000000009E3779B1L, 0xD61105C20E91F99FL); 81 81 + (16, 0x0000000000000000L, 0x98C90B57FDFCB55CL); 82 82 + (16, 0x000000009E3779B1L, 0xC900AD2D536B607EL); 83 83 + (17, 0x0000000000000000L, 0x0D39A2D051A30C2CL); 84 84 + (17, 0x000000009E3779B1L, 0x495CD68A647C7A22L); 85 85 + (18, 0x0000000000000000L, 0x33E84A4333B2B2EBL); 86 86 + (18, 0x000000009E3779B1L, 0x2325A30CCA1A66DDL); 87 87 + (19, 0x0000000000000000L, 0xE91C6EF31FC08F82L); 88 88 + (19, 0x000000009E3779B1L, 0x06809662799B7D6FL); 89 89 + (20, 0x0000000000000000L, 0x5F8C68355769439EL); 90 90 + (20, 0x000000009E3779B1L, 0x97218696C2D29602L); 91 91 + (21, 0x0000000000000000L, 0x42B0B8EE353AC461L); 92 92 + (21, 0x000000009E3779B1L, 0x7FC0BB451B83A633L); 93 93 + (22, 0x0000000000000000L, 0x65C935C6978098B1L); 94 94 + (22, 0x000000009E3779B1L, 0xC4A0DD14BF835C13L); 95 95 + (23, 0x0000000000000000L, 0xD2460ECC840B74DDL); 96 96 + (23, 0x000000009E3779B1L, 0x4B44E8DE7A396773L); 97 97 + (24, 0x0000000000000000L, 0xF75A6DEA42DC5BF4L); 98 98 + (24, 0x000000009E3779B1L, 0x8B7C67EB59778E22L); 99 99 + (25, 0x0000000000000000L, 0x52FAA43C3F20B994L); 100 100 + (25, 0x000000009E3779B1L, 0xC4FEC92EAC2C3B8AL); 101 101 + (26, 0x0000000000000000L, 0x8DB7831EC345F9A3L); 102 102 + (26, 0x000000009E3779B1L, 0x2C2A80BCAD321466L); 103 103 + (27, 0x0000000000000000L, 0x88945AA08051FC2DL); 104 104 + (27, 0x000000009E3779B1L, 0x3401AF8EF28FD410L); 105 105 + (28, 0x0000000000000000L, 0x64CD9E8C96A9E2DDL); 106 106 + (28, 0x000000009E3779B1L, 0x8160FB8C20B48287L); 107 107 + (29, 0x0000000000000000L, 0x8C8F345B634AC2B9L); 108 108 + (29, 0x000000009E3779B1L, 0x5A327C78E4AD6678L); 109 109 + (30, 0x0000000000000000L, 0xE2677241D4C46CAFL); 110 110 + (30, 0x000000009E3779B1L, 0xB1B2B51C93AF4866L); 111 111 + (31, 0x0000000000000000L, 0x299B39A290E6D783L); 112 112 + (31, 0x000000009E3779B1L, 0xDA673D5FEB5C1D79L); 113 113 + (* Lengths 32-64: one to two blocks *) 114 114 + (32, 0x0000000000000000L, 0x18B216492BB44B70L); 115 115 + (32, 0x000000009E3779B1L, 0xB3F33BDF93ADE409L); 116 116 + (33, 0x0000000000000000L, 0x55C8DC3E578F5B59L); 117 117 + (33, 0x000000009E3779B1L, 0xE92C292F64BC3071L); 118 118 + (48, 0x0000000000000000L, 0xFD0FEEAC7A939933L); 119 119 + (48, 0x000000009E3779B1L, 0x6FFE2F43A24C2302L); 120 120 + (63, 0x0000000000000000L, 0xA9EFBE0FA0F3F4E7L); 121 121 + (63, 0x000000009E3779B1L, 0x6C911FADB05B6FC2L); 122 122 + (64, 0x0000000000000000L, 0xEF558F8ACAC2B5CDL); 123 123 + (64, 0x000000009E3779B1L, 0xB5EEBA99264CC44FL); 124 124 + (* Lengths 65-128: two to four blocks *) 125 125 + (65, 0x0000000000000000L, 0xDE0F20DC2631AF7AL); 126 126 + (65, 0x000000009E3779B1L, 0xD3F6FF3941E310CAL); 127 127 + (96, 0x0000000000000000L, 0x105064E743EDD1D9L); 128 128 + (96, 0x000000009E3779B1L, 0x8FF0B4ABEE6F03CCL); 129 129 + (100, 0x0000000000000000L, 0x4BFE019CD91D9EA4L); 130 130 + (100, 0x000000009E3779B1L, 0x4853706DC9625CAEL); 131 131 + (127, 0x0000000000000000L, 0x3C7A21119AA662B0L); 132 132 + (127, 0x000000009E3779B1L, 0xB0D6DC189C06CEEDL); 133 133 + (128, 0x0000000000000000L, 0x90CA021457D96DC5L); 134 134 + (128, 0x000000009E3779B1L, 0xED9340A202BCD1CFL); 135 135 + (* Larger sizes: multiple blocks *) 136 136 + (256, 0x0000000000000000L, 0x5E3F5BF94D574981L); 137 137 + (256, 0x000000009E3779B1L, 0x34733CBD9CC1B0D5L); 138 138 + (512, 0x0000000000000000L, 0x4358D2FDD62B58A7L); 139 139 + (512, 0x000000009E3779B1L, 0x0DED69C4804C47BAL); 140 140 + (1024, 0x0000000000000000L, 0x4775BF7CACE4D177L); 141 141 + (1024, 0x000000009E3779B1L, 0x238CF9296898B465L); 142 142 + (4096, 0x0000000000000000L, 0xAB77F4AF85F4E70BL); 143 143 + (4096, 0x000000009E3779B1L, 0xCB8B60CBA513125DL); 144 144 + ] 145 145 + 146 146 + (* Create test buffer once - large enough for all tests *) 147 147 + let test_buffer = fill_test_buffer 4200 148 148 + 149 149 + (* Known reference value: xxhash64("") with seed 0 *) 150 150 + let test_empty_string () = 151 151 + let hash = Xxhash.hash64_string "" in 152 152 + Alcotest.(check int64) "empty string" 0xef46db3751d8e999L hash 153 153 + 154 154 + (* Consistency tests - verify streaming and one-shot produce same results *) 155 155 + let test_consistency_short () = 156 156 + let s = "Hello" in 157 157 + let direct = Xxhash.hash64_string s in 158 158 + let state = Xxhash.create_state () in 159 159 + Xxhash.update_string state s; 160 160 + let streaming = Xxhash.finalize state in 161 161 + Alcotest.(check int64) "short string consistency" direct streaming 162 162 + 163 163 + let test_consistency_medium () = 164 164 + let s = "Hello, World!" in 165 165 + let direct = Xxhash.hash64_string s in 166 166 + let state = Xxhash.create_state () in 167 167 + Xxhash.update_string state s; 168 168 + let streaming = Xxhash.finalize state in 169 169 + Alcotest.(check int64) "medium string consistency" direct streaming 170 170 + 171 171 + let test_consistency_with_seed () = 172 172 + let s = "test data" in 173 173 + let seed = 12345L in 174 174 + let direct = Xxhash.hash64_string ~seed s in 175 175 + let state = Xxhash.create_state ~seed () in 176 176 + Xxhash.update_string state s; 177 177 + let streaming = Xxhash.finalize state in 178 178 + Alcotest.(check int64) "consistency with seed" direct streaming 179 179 + 180 180 + (* Boundary condition: exactly 32 bytes (one block) *) 181 181 + let test_32_bytes () = 182 182 + let s = String.make 32 'x' in 183 183 + let direct = Xxhash.hash64_string s in 184 184 + let state = Xxhash.create_state () in 185 185 + Xxhash.update_string state s; 186 186 + let streaming = Xxhash.finalize state in 187 187 + Alcotest.(check int64) "32 bytes consistency" direct streaming 188 188 + 189 189 + (* Boundary condition: 33 bytes (one block + 1 byte) *) 190 190 + let test_33_bytes () = 191 191 + let s = String.make 33 'y' in 192 192 + let direct = Xxhash.hash64_string s in 193 193 + let state = Xxhash.create_state () in 194 194 + Xxhash.update_string state s; 195 195 + let streaming = Xxhash.finalize state in 196 196 + Alcotest.(check int64) "33 bytes consistency" direct streaming 197 197 + 198 198 + (* Boundary condition: 64 bytes (two blocks) *) 199 199 + let test_64_bytes () = 200 200 + let s = String.make 64 'a' in 201 201 + let direct = Xxhash.hash64_string s in 202 202 + let state = Xxhash.create_state () in 203 203 + Xxhash.update_string state s; 204 204 + let streaming = Xxhash.finalize state in 205 205 + Alcotest.(check int64) "64 bytes consistency" direct streaming 206 206 + 207 207 + (* hash32 consistency *) 208 208 + let test_hash32_consistency () = 209 209 + let s = "Hello, World!" in 210 210 + let hash64 = Xxhash.hash64_string s in 211 211 + let hash32 = Xxhash.hash32_string s in 212 212 + let expected32 = Int64.to_int32 (Int64.logand hash64 0xFFFFFFFFL) in 213 213 + Alcotest.(check int32) "hash32 is lower 32 bits" expected32 hash32 214 214 + 215 215 + let test_streaming_chunks () = 216 216 + (* Hash in multiple chunks, should match single hash *) 217 217 + let state = Xxhash.create_state () in 218 218 + Xxhash.update_string state "Hello"; 219 219 + Xxhash.update_string state ", "; 220 220 + Xxhash.update_string state "World!"; 221 221 + let hash = Xxhash.finalize state in 222 222 + let direct = Xxhash.hash64_string "Hello, World!" in 223 223 + Alcotest.(check int64) "chunked streaming" direct hash 224 224 + 225 225 + let test_streaming_byte_at_a_time () = 226 226 + let s = "Hello, World!" in 227 227 + let state = Xxhash.create_state () in 228 228 + String.iter (fun c -> 229 229 + Xxhash.update_string state (String.make 1 c) 230 230 + ) s; 231 231 + let hash = Xxhash.finalize state in 232 232 + let direct = Xxhash.hash64_string s in 233 233 + Alcotest.(check int64) "byte-at-a-time" direct hash 234 234 + 235 235 + let test_streaming_large () = 236 236 + (* Large data in chunks *) 237 237 + let chunk = String.make 100 'z' in 238 238 + let state = Xxhash.create_state () in 239 239 + for _ = 1 to 10 do 240 240 + Xxhash.update_string state chunk 241 241 + done; 242 242 + let hash = Xxhash.finalize state in 243 243 + let direct = Xxhash.hash64_string (String.make 1000 'z') in 244 244 + Alcotest.(check int64) "large streaming" direct hash 245 245 + 246 246 + let test_streaming_across_boundary () = 247 247 + (* Feed data that crosses 32-byte block boundaries *) 248 248 + let state = Xxhash.create_state () in 249 249 + Xxhash.update_string state (String.make 20 'a'); 250 250 + Xxhash.update_string state (String.make 20 'b'); 251 251 + Xxhash.update_string state (String.make 20 'c'); 252 252 + let hash = Xxhash.finalize state in 253 253 + let direct = Xxhash.hash64_string (String.make 20 'a' ^ String.make 20 'b' ^ String.make 20 'c') in 254 254 + Alcotest.(check int64) "across boundary" direct hash 255 255 + 256 256 + let test_reset () = 257 257 + let state = Xxhash.create_state () in 258 258 + Xxhash.update_string state "first data"; 259 259 + let _ = Xxhash.finalize state in 260 260 + Xxhash.reset state; 261 261 + Xxhash.update_string state "second data"; 262 262 + let hash = Xxhash.finalize state in 263 263 + let direct = Xxhash.hash64_string "second data" in 264 264 + Alcotest.(check int64) "after reset" direct hash 265 265 + 266 266 + let test_reset_with_new_seed () = 267 267 + let state = Xxhash.create_state ~seed:111L () in 268 268 + Xxhash.update_string state "first"; 269 269 + let _ = Xxhash.finalize state in 270 270 + Xxhash.reset ~seed:222L state; 271 271 + Xxhash.update_string state "second"; 272 272 + let hash = Xxhash.finalize state in 273 273 + let direct = Xxhash.hash64_string ~seed:222L "second" in 274 274 + Alcotest.(check int64) "reset with new seed" direct hash 275 275 + 276 276 + let test_copy_state () = 277 277 + let state1 = Xxhash.create_state () in 278 278 + Xxhash.update_string state1 "Hello"; 279 279 + let state2 = Xxhash.copy_state state1 in 280 280 + Xxhash.update_string state1 ", World!"; 281 281 + Xxhash.update_string state2 " there"; 282 282 + let hash1 = Xxhash.finalize state1 in 283 283 + let hash2 = Xxhash.finalize state2 in 284 284 + let direct1 = Xxhash.hash64_string "Hello, World!" in 285 285 + let direct2 = Xxhash.hash64_string "Hello there" in 286 286 + Alcotest.(check int64) "original state" direct1 hash1; 287 287 + Alcotest.(check int64) "copied state" direct2 hash2 288 288 + 289 289 + let test_finalize32 () = 290 290 + let state = Xxhash.create_state () in 291 291 + Xxhash.update_string state "test"; 292 292 + let hash32 = Xxhash.finalize32 state in 293 293 + let hash64 = Xxhash.finalize state in 294 294 + let expected32 = Int64.to_int32 (Int64.logand hash64 0xFFFFFFFFL) in 295 295 + Alcotest.(check int32) "finalize32" expected32 hash32 296 296 + 297 297 + let test_bytes_api () = 298 298 + let s = "Hello, World!" in 299 299 + let bytes = Bytes.of_string s in 300 300 + let from_string = Xxhash.hash64_string s in 301 301 + let from_bytes = Xxhash.hash64 bytes ~pos:0 ~len:(Bytes.length bytes) in 302 302 + Alcotest.(check int64) "bytes API consistency" from_string from_bytes 303 303 + 304 304 + let test_bytes_partial () = 305 305 + let bytes = Bytes.of_string "XXXHello, World!YYY" in 306 306 + let hash = Xxhash.hash64 bytes ~pos:3 ~len:13 in 307 307 + let direct = Xxhash.hash64_string "Hello, World!" in 308 308 + Alcotest.(check int64) "partial bytes" direct hash 309 309 + 310 310 + (* Determinism test *) 311 311 + let test_deterministic () = 312 312 + let s = "The quick brown fox jumps over the lazy dog" in 313 313 + let hash1 = Xxhash.hash64_string s in 314 314 + let hash2 = Xxhash.hash64_string s in 315 315 + let hash3 = Xxhash.hash64_string s in 316 316 + Alcotest.(check int64) "hash1 = hash2" hash1 hash2; 317 317 + Alcotest.(check int64) "hash2 = hash3" hash2 hash3 318 318 + 319 319 + (* Different inputs produce different hashes *) 320 320 + let test_different_inputs () = 321 321 + let h1 = Xxhash.hash64_string "hello" in 322 322 + let h2 = Xxhash.hash64_string "Hello" in 323 323 + let h3 = Xxhash.hash64_string "hello " in 324 324 + Alcotest.(check bool) "hello != Hello" true (h1 <> h2); 325 325 + Alcotest.(check bool) "hello != 'hello '" true (h1 <> h3) 326 326 + 327 327 + (* Different seeds produce different hashes *) 328 328 + let test_different_seeds () = 329 329 + let s = "test" in 330 330 + let h1 = Xxhash.hash64_string ~seed:0L s in 331 331 + let h2 = Xxhash.hash64_string ~seed:1L s in 332 332 + let h3 = Xxhash.hash64_string ~seed:42L s in 333 333 + Alcotest.(check bool) "seed 0 != seed 1" true (h1 <> h2); 334 334 + Alcotest.(check bool) "seed 1 != seed 42" true (h2 <> h3) 335 335 + 336 336 + (* ===== Reference Validation Tests ===== *) 337 337 + 338 338 + (** Test against official xxHash test vectors *) 339 339 + let test_reference_vectors () = 340 340 + let failed = ref [] in 341 341 + List.iteri (fun i (len, seed, expected) -> 342 342 + let actual = Xxhash.hash64 ~seed test_buffer ~pos:0 ~len in 343 343 + if actual <> expected then 344 344 + failed := (i, len, seed, expected, actual) :: !failed 345 345 + ) xxh64_test_vectors; 346 346 + if !failed <> [] then begin 347 347 + List.iter (fun (i, len, seed, expected, actual) -> 348 348 + Printf.eprintf "FAIL test %d: len=%d seed=%016Lx expected=%016Lx got=%016Lx\n" 349 349 + i len seed expected actual 350 350 + ) (List.rev !failed); 351 351 + Alcotest.fail (Printf.sprintf "%d reference tests failed" (List.length !failed)) 352 352 + end 353 353 + 354 354 + (** Generate reference tests for streaming mode *) 355 355 + let test_reference_streaming () = 356 356 + let failed = ref [] in 357 357 + List.iteri (fun i (len, seed, expected) -> 358 358 + let state = Xxhash.create_state ~seed () in 359 359 + Xxhash.update state test_buffer ~pos:0 ~len; 360 360 + let actual = Xxhash.finalize state in 361 361 + if actual <> expected then 362 362 + failed := (i, len, seed, expected, actual) :: !failed 363 363 + ) xxh64_test_vectors; 364 364 + if !failed <> [] then begin 365 365 + List.iter (fun (i, len, seed, expected, actual) -> 366 366 + Printf.eprintf "FAIL streaming test %d: len=%d seed=%016Lx expected=%016Lx got=%016Lx\n" 367 367 + i len seed expected actual 368 368 + ) (List.rev !failed); 369 369 + Alcotest.fail (Printf.sprintf "%d streaming reference tests failed" (List.length !failed)) 370 370 + end 371 371 + 372 372 + (** Test streaming with byte-by-byte updates *) 373 373 + let test_reference_streaming_bytewise () = 374 374 + let failed = ref [] in 375 375 + List.iteri (fun i (len, seed, expected) -> 376 376 + let state = Xxhash.create_state ~seed () in 377 377 + for j = 0 to len - 1 do 378 378 + Xxhash.update state test_buffer ~pos:j ~len:1 379 379 + done; 380 380 + let actual = Xxhash.finalize state in 381 381 + if actual <> expected then 382 382 + failed := (i, len, seed, expected, actual) :: !failed 383 383 + ) xxh64_test_vectors; 384 384 + if !failed <> [] then begin 385 385 + List.iter (fun (i, len, seed, expected, actual) -> 386 386 + Printf.eprintf "FAIL bytewise test %d: len=%d seed=%016Lx expected=%016Lx got=%016Lx\n" 387 387 + i len seed expected actual 388 388 + ) (List.rev !failed); 389 389 + Alcotest.fail (Printf.sprintf "%d bytewise reference tests failed" (List.length !failed)) 390 390 + end 391 391 + 392 392 + let () = 393 393 + Alcotest.run "xxhash" [ 394 394 + "reference", [ 395 395 + Alcotest.test_case "empty string" `Quick test_empty_string; 396 396 + Alcotest.test_case "official vectors" `Quick test_reference_vectors; 397 397 + Alcotest.test_case "streaming vectors" `Quick test_reference_streaming; 398 398 + Alcotest.test_case "bytewise streaming" `Quick test_reference_streaming_bytewise; 399 399 + ]; 400 400 + "consistency", [ 401 401 + Alcotest.test_case "short" `Quick test_consistency_short; 402 402 + Alcotest.test_case "medium" `Quick test_consistency_medium; 403 403 + Alcotest.test_case "with seed" `Quick test_consistency_with_seed; 404 404 + Alcotest.test_case "hash32" `Quick test_hash32_consistency; 405 405 + Alcotest.test_case "bytes API" `Quick test_bytes_api; 406 406 + Alcotest.test_case "partial bytes" `Quick test_bytes_partial; 407 407 + ]; 408 408 + "boundaries", [ 409 409 + Alcotest.test_case "32 bytes" `Quick test_32_bytes; 410 410 + Alcotest.test_case "33 bytes" `Quick test_33_bytes; 411 411 + Alcotest.test_case "64 bytes" `Quick test_64_bytes; 412 412 + ]; 413 413 + "streaming", [ 414 414 + Alcotest.test_case "chunks" `Quick test_streaming_chunks; 415 415 + Alcotest.test_case "byte at a time" `Quick test_streaming_byte_at_a_time; 416 416 + Alcotest.test_case "large" `Quick test_streaming_large; 417 417 + Alcotest.test_case "across boundary" `Quick test_streaming_across_boundary; 418 418 + ]; 419 419 + "state", [ 420 420 + Alcotest.test_case "reset" `Quick test_reset; 421 421 + Alcotest.test_case "reset with seed" `Quick test_reset_with_new_seed; 422 422 + Alcotest.test_case "copy" `Quick test_copy_state; 423 423 + Alcotest.test_case "finalize32" `Quick test_finalize32; 424 424 + ]; 425 425 + "properties", [ 426 426 + Alcotest.test_case "deterministic" `Quick test_deterministic; 427 427 + Alcotest.test_case "different inputs" `Quick test_different_inputs; 428 428 + Alcotest.test_case "different seeds" `Quick test_different_seeds; 429 429 + ]; 430 430 + ]