+1

.gitignore

··· 1 1 + _build

+34

bitstream.opam

··· 1 1 + # This file is generated by dune, edit dune-project instead 2 2 + opam-version: "2.0" 3 3 + synopsis: "Bit-level I/O for binary format parsing and generation" 4 4 + description: """ 5 5 + Forward and backward bitstream reading/writing for binary formats. 6 6 + Supports bit-level operations required by compression algorithms like 7 7 + FSE, ANS, and Huffman coding.""" 8 8 + maintainer: ["Anil Madhavapeddy <anil@recoil.org>"] 9 9 + authors: ["Anil Madhavapeddy <anil@recoil.org>"] 10 10 + license: "ISC" 11 11 + homepage: "https://tangled.org/anil.recoil.org/ocaml-bitstream" 12 12 + bug-reports: "https://tangled.org/anil.recoil.org/ocaml-bitstream/issues" 13 13 + depends: [ 14 14 + "dune" {>= "3.21"} 15 15 + "ocaml" {>= "5.2.0"} 16 16 + "alcotest" {with-test & >= "1.7.0"} 17 17 + "odoc" {with-doc} 18 18 + ] 19 19 + build: [ 20 20 + ["dune" "subst"] {dev} 21 21 + [ 22 22 + "dune" 23 23 + "build" 24 24 + "-p" 25 25 + name 26 26 + "-j" 27 27 + jobs 28 28 + "@install" 29 29 + "@runtest" {with-test} 30 30 + "@doc" {with-doc} 31 31 + ] 32 32 + ] 33 33 + dev-repo: "git+https://tangled.org/anil.recoil.org/ocaml-bitstream" 34 34 + x-maintenance-intent: ["(latest)"]

+1

dune

··· 1 1 + (vendored_dirs vendor)

+20

dune-project

··· 1 1 + (lang dune 3.21) 2 2 + (name bitstream) 3 3 + 4 4 + (generate_opam_files true) 5 5 + 6 6 + (license ISC) 7 7 + (authors "Anil Madhavapeddy <anil@recoil.org>") 8 8 + (maintainers "Anil Madhavapeddy <anil@recoil.org>") 9 9 + (source (tangled anil.recoil.org/ocaml-bitstream)) 10 10 + 11 11 + (package 12 12 + (name bitstream) 13 13 + (synopsis "Bit-level I/O for binary format parsing and generation") 14 14 + (description 15 15 + "Forward and backward bitstream reading/writing for binary formats. 16 16 + Supports bit-level operations required by compression algorithms like 17 17 + FSE, ANS, and Huffman coding.") 18 18 + (depends 19 19 + (ocaml (>= 5.2.0)) 20 20 + (alcotest (and :with-test (>= 1.7.0)))))

+385

src/bitstream.ml

··· 1 1 + (** Bitstream - Bit-level I/O for binary formats. 2 2 + 3 3 + Provides forward and backward bitstream reading and writing for parsing 4 4 + and generating binary formats that operate at the bit level. 5 5 + 6 6 + Forward streams read/write from the start of a buffer towards the end. 7 7 + Backward streams read/write from the end of a buffer towards the start, 8 8 + which is required by some compression algorithms (FSE, ANS). *) 9 9 + 10 10 + (** {1 Slice Type} *) 11 11 + 12 12 + module Slice = struct 13 13 + type t = { 14 14 + bytes : bytes; 15 15 + first : int; 16 16 + length : int; 17 17 + } 18 18 + 19 19 + let make bytes ~first ~length = 20 20 + { bytes; first; length } 21 21 + 22 22 + let of_bytes ?first ?length bytes = 23 23 + let first = Option.value first ~default:0 in 24 24 + let length = Option.value length ~default:(Bytes.length bytes - first) in 25 25 + { bytes; first; length } 26 26 + 27 27 + let to_bytes t = 28 28 + Bytes.sub t.bytes t.first t.length 29 29 + 30 30 + let is_empty t = 31 31 + t.length = 0 32 32 + 33 33 + let sub t ~first ~length = 34 34 + { bytes = t.bytes; first = t.first + first; length } 35 35 + end 36 36 + 37 37 + (** {1 Exceptions} *) 38 38 + 39 39 + exception End_of_stream 40 40 + (** Raised when attempting to read past the end of the stream. *) 41 41 + 42 42 + exception Invalid_state of string 43 43 + (** Raised when an operation requires a specific state (e.g., byte alignment). *) 44 44 + 45 45 + exception Corrupted_stream of string 46 46 + (** Raised when stream data is malformed (e.g., invalid padding marker). *) 47 47 + 48 48 + (** {1 Forward Bitstream Reader} *) 49 49 + 50 50 + module Forward_reader = struct 51 51 + type t = { 52 52 + src : bytes; 53 53 + start_pos : int; 54 54 + limit : int; 55 55 + mutable byte_pos : int; 56 56 + mutable bit_pos : int; (* 0-7, bits consumed in current byte *) 57 57 + } 58 58 + 59 59 + let of_slice (slice : Slice.t) = 60 60 + { src = slice.bytes; 61 61 + start_pos = slice.first; 62 62 + limit = slice.first + slice.length; 63 63 + byte_pos = slice.first; 64 64 + bit_pos = 0 } 65 65 + 66 66 + let of_bytes src = 67 67 + of_slice (Slice.of_bytes src) 68 68 + 69 69 + let create src ~pos ~len = 70 70 + of_slice (Slice.make src ~first:pos ~length:len) 71 71 + 72 72 + let[@inline] remaining t = 73 73 + (t.limit - t.byte_pos) * 8 - t.bit_pos 74 74 + 75 75 + let[@inline] is_byte_aligned t = 76 76 + t.bit_pos = 0 77 77 + 78 78 + let[@inline] read_bits t n = 79 79 + if n <= 0 then 0 80 80 + else if n > 57 then invalid_arg "read_bits: n > 57" 81 81 + else begin 82 82 + let result = ref 0 in 83 83 + let bits_read = ref 0 in 84 84 + while !bits_read < n do 85 85 + if t.byte_pos >= t.limit then 86 86 + raise End_of_stream; 87 87 + let byte = Bytes.get_uint8 t.src t.byte_pos in 88 88 + let available = 8 - t.bit_pos in 89 89 + let to_read = min available (n - !bits_read) in 90 90 + let mask = (1 lsl to_read) - 1 in 91 91 + let bits = (byte lsr t.bit_pos) land mask in 92 92 + result := !result lor (bits lsl !bits_read); 93 93 + bits_read := !bits_read + to_read; 94 94 + t.bit_pos <- t.bit_pos + to_read; 95 95 + if t.bit_pos >= 8 then begin 96 96 + t.bit_pos <- 0; 97 97 + t.byte_pos <- t.byte_pos + 1 98 98 + end 99 99 + done; 100 100 + !result 101 101 + end 102 102 + 103 103 + let[@inline] read_byte t = 104 104 + if t.bit_pos <> 0 then 105 105 + raise (Invalid_state "read_byte: not byte aligned"); 106 106 + if t.byte_pos >= t.limit then 107 107 + raise End_of_stream; 108 108 + let b = Bytes.get_uint8 t.src t.byte_pos in 109 109 + t.byte_pos <- t.byte_pos + 1; 110 110 + b 111 111 + 112 112 + let rewind_bits t n = 113 113 + let total_bits = (t.byte_pos - t.start_pos) * 8 + t.bit_pos in 114 114 + let new_total = total_bits - n in 115 115 + if new_total < 0 then 116 116 + raise End_of_stream; 117 117 + t.byte_pos <- t.start_pos + new_total / 8; 118 118 + t.bit_pos <- new_total mod 8 119 119 + 120 120 + let align t = 121 121 + if t.bit_pos <> 0 then begin 122 122 + t.bit_pos <- 0; 123 123 + t.byte_pos <- t.byte_pos + 1 124 124 + end 125 125 + 126 126 + let byte_position t = 127 127 + if t.bit_pos <> 0 then 128 128 + raise (Invalid_state "byte_position: not byte aligned"); 129 129 + t.byte_pos 130 130 + 131 131 + let get_slice t n : Slice.t = 132 132 + if t.bit_pos <> 0 then 133 133 + raise (Invalid_state "get_slice: not byte aligned"); 134 134 + if t.byte_pos + n > t.limit then 135 135 + raise End_of_stream; 136 136 + let result = Slice.make t.src ~first:t.byte_pos ~length:n in 137 137 + t.byte_pos <- t.byte_pos + n; 138 138 + result 139 139 + 140 140 + let get_bytes t n = 141 141 + Slice.to_bytes (get_slice t n) 142 142 + 143 143 + let to_slice t : Slice.t = 144 144 + if t.bit_pos <> 0 then 145 145 + raise (Invalid_state "to_slice: not byte aligned"); 146 146 + Slice.make t.src ~first:t.byte_pos ~length:(t.limit - t.byte_pos) 147 147 + 148 148 + let advance t n = 149 149 + if t.bit_pos <> 0 then 150 150 + raise (Invalid_state "advance: not byte aligned"); 151 151 + if t.byte_pos + n > t.limit then 152 152 + raise End_of_stream; 153 153 + t.byte_pos <- t.byte_pos + n 154 154 + 155 155 + let sub t n = 156 156 + if t.bit_pos <> 0 then 157 157 + raise (Invalid_state "sub: not byte aligned"); 158 158 + if t.byte_pos + n > t.limit then 159 159 + raise End_of_stream; 160 160 + let result = of_slice (Slice.make t.src ~first:t.byte_pos ~length:n) in 161 161 + t.byte_pos <- t.byte_pos + n; 162 162 + result 163 163 + 164 164 + let remaining_bytes t = 165 165 + if t.bit_pos <> 0 then 166 166 + raise (Invalid_state "remaining_bytes: not byte aligned"); 167 167 + t.limit - t.byte_pos 168 168 + 169 169 + let skip_bits t n = 170 170 + ignore (read_bits t n) 171 171 + end 172 172 + 173 173 + (** {1 Backward Bitstream Reader} 174 174 + 175 175 + Reads bits from the end of a buffer towards the start. The stream 176 176 + starts with a padding marker (highest 1-bit indicates start of data). *) 177 177 + 178 178 + module Backward_reader = struct 179 179 + type t = { 180 180 + src : bytes; 181 181 + start_pos : int; 182 182 + mutable bit_offset : int; (* Bits remaining from end, decreasing *) 183 183 + } 184 184 + 185 185 + let of_slice (slice : Slice.t) = 186 186 + if slice.length = 0 then 187 187 + raise End_of_stream; 188 188 + let last_byte_pos = slice.first + slice.length - 1 in 189 189 + let last_byte = Bytes.get_uint8 slice.bytes last_byte_pos in 190 190 + if last_byte = 0 then 191 191 + raise (Corrupted_stream "invalid padding marker"); 192 192 + (* Find the highest set bit - this is the padding marker *) 193 193 + let rec find_marker byte bit = 194 194 + if bit < 0 then 0 195 195 + else if (byte land (1 lsl bit)) <> 0 then bit 196 196 + else find_marker byte (bit - 1) 197 197 + in 198 198 + let padding = 8 - find_marker last_byte 7 in 199 199 + let bit_offset = slice.length * 8 - padding in 200 200 + { src = slice.bytes; start_pos = slice.first; bit_offset } 201 201 + 202 202 + let of_bytes src ~pos ~len = 203 203 + of_slice (Slice.make src ~first:pos ~length:len) 204 204 + 205 205 + let[@inline] remaining t = t.bit_offset 206 206 + 207 207 + let[@inline] is_empty t = t.bit_offset <= 0 208 208 + 209 209 + let[@inline] read_bits t n = 210 210 + if n <= 0 then 0 211 211 + else if n > 57 then invalid_arg "read_bits: n > 57" 212 212 + else begin 213 213 + t.bit_offset <- t.bit_offset - n; 214 214 + let actual_offset = max 0 t.bit_offset in 215 215 + let actual_bits = if t.bit_offset < 0 then n + t.bit_offset else n in 216 216 + if actual_bits <= 0 then 0 217 217 + else begin 218 218 + let byte_offset = t.start_pos + (actual_offset / 8) in 219 219 + let bit_offset = actual_offset mod 8 in 220 220 + let result = ref 0 in 221 221 + let bits_read = ref 0 in 222 222 + let current_byte = ref byte_offset in 223 223 + let current_bit = ref bit_offset in 224 224 + while !bits_read < actual_bits do 225 225 + let byte = Bytes.get_uint8 t.src !current_byte in 226 226 + let available = 8 - !current_bit in 227 227 + let to_read = min available (actual_bits - !bits_read) in 228 228 + let mask = (1 lsl to_read) - 1 in 229 229 + let bits = (byte lsr !current_bit) land mask in 230 230 + result := !result lor (bits lsl !bits_read); 231 231 + bits_read := !bits_read + to_read; 232 232 + current_bit := !current_bit + to_read; 233 233 + if !current_bit >= 8 then begin 234 234 + current_bit := 0; 235 235 + incr current_byte 236 236 + end 237 237 + done; 238 238 + (* If we read past the beginning, shift the result *) 239 239 + if t.bit_offset < 0 then 240 240 + !result lsl (-t.bit_offset) 241 241 + else 242 242 + !result 243 243 + end 244 244 + end 245 245 + 246 246 + let peek_bits t n = 247 247 + let saved_offset = t.bit_offset in 248 248 + let result = read_bits t n in 249 249 + t.bit_offset <- saved_offset; 250 250 + result 251 251 + end 252 252 + 253 253 + (** {1 Forward Bitstream Writer} *) 254 254 + 255 255 + module Forward_writer = struct 256 256 + type t = { 257 257 + dst : bytes; 258 258 + start_pos : int; 259 259 + mutable byte_pos : int; 260 260 + mutable bit_pos : int; (* 0-7, bits written in current byte *) 261 261 + mutable current_byte : int; 262 262 + } 263 263 + 264 264 + let of_slice (slice : Slice.t) = 265 265 + { dst = slice.bytes; 266 266 + start_pos = slice.first; 267 267 + byte_pos = slice.first; 268 268 + bit_pos = 0; 269 269 + current_byte = 0 } 270 270 + 271 271 + let of_bytes dst = 272 272 + of_slice (Slice.of_bytes dst) 273 273 + 274 274 + let create dst ~pos = 275 275 + of_slice (Slice.make dst ~first:pos ~length:(Bytes.length dst - pos)) 276 276 + 277 277 + let flush t = 278 278 + if t.bit_pos > 0 then begin 279 279 + Bytes.set_uint8 t.dst t.byte_pos t.current_byte; 280 280 + t.byte_pos <- t.byte_pos + 1; 281 281 + t.bit_pos <- 0; 282 282 + t.current_byte <- 0 283 283 + end 284 284 + 285 285 + let[@inline] write_bits t value n = 286 286 + if n <= 0 then () 287 287 + else if n > 57 then invalid_arg "write_bits: n > 57" 288 288 + else begin 289 289 + let value = ref value in 290 290 + let remaining = ref n in 291 291 + 292 292 + while !remaining > 0 do 293 293 + let available = 8 - t.bit_pos in 294 294 + let to_write = min available !remaining in 295 295 + let mask = (1 lsl to_write) - 1 in 296 296 + t.current_byte <- t.current_byte lor ((!value land mask) lsl t.bit_pos); 297 297 + value := !value lsr to_write; 298 298 + remaining := !remaining - to_write; 299 299 + t.bit_pos <- t.bit_pos + to_write; 300 300 + 301 301 + if t.bit_pos = 8 then begin 302 302 + Bytes.set_uint8 t.dst t.byte_pos t.current_byte; 303 303 + t.byte_pos <- t.byte_pos + 1; 304 304 + t.bit_pos <- 0; 305 305 + t.current_byte <- 0 306 306 + end 307 307 + done 308 308 + end 309 309 + 310 310 + let write_byte t value = 311 311 + if t.bit_pos <> 0 then flush t; 312 312 + Bytes.set_uint8 t.dst t.byte_pos value; 313 313 + t.byte_pos <- t.byte_pos + 1 314 314 + 315 315 + let write_slice t (slice : Slice.t) = 316 316 + if t.bit_pos <> 0 then flush t; 317 317 + Bytes.blit slice.bytes slice.first t.dst t.byte_pos slice.length; 318 318 + t.byte_pos <- t.byte_pos + slice.length 319 319 + 320 320 + let write_bytes t src = 321 321 + write_slice t (Slice.of_bytes src) 322 322 + 323 323 + let byte_position t = 324 324 + if t.bit_pos > 0 then t.byte_pos + 1 else t.byte_pos 325 325 + 326 326 + let finalize t = 327 327 + flush t; 328 328 + t.byte_pos - t.start_pos 329 329 + 330 330 + let to_slice t : Slice.t = 331 331 + flush t; 332 332 + Slice.make t.dst ~first:t.start_pos ~length:(t.byte_pos - t.start_pos) 333 333 + end 334 334 + 335 335 + (** {1 Backward Bitstream Writer} 336 336 + 337 337 + Accumulates bits to be read backwards. Used for FSE and Huffman encoding. *) 338 338 + 339 339 + module Backward_writer = struct 340 340 + type t = { 341 341 + mutable bits : int64; 342 342 + mutable num_bits : int; 343 343 + buffer : bytes; 344 344 + mutable buf_pos : int; 345 345 + } 346 346 + 347 347 + let create size = 348 348 + { bits = 0L; num_bits = 0; buffer = Bytes.create size; buf_pos = size } 349 349 + 350 350 + let[@inline] write_bits t value n = 351 351 + if n > 0 then begin 352 352 + t.bits <- Int64.logor t.bits (Int64.shift_left (Int64.of_int value) t.num_bits); 353 353 + t.num_bits <- t.num_bits + n 354 354 + end 355 355 + 356 356 + let flush_bytes t = 357 357 + while t.num_bits >= 8 do 358 358 + t.buf_pos <- t.buf_pos - 1; 359 359 + Bytes.set_uint8 t.buffer t.buf_pos (Int64.to_int (Int64.logand t.bits 0xFFL)); 360 360 + t.bits <- Int64.shift_right_logical t.bits 8; 361 361 + t.num_bits <- t.num_bits - 8 362 362 + done 363 363 + 364 364 + let finalize_to_slice t : Slice.t = 365 365 + write_bits t 1 1; 366 366 + if t.num_bits mod 8 <> 0 then 367 367 + t.num_bits <- ((t.num_bits + 7) / 8) * 8; 368 368 + flush_bytes t; 369 369 + let len = Bytes.length t.buffer - t.buf_pos in 370 370 + (* Reverse bytes in place so marker ends up at the end *) 371 371 + for i = 0 to len / 2 - 1 do 372 372 + let j = t.buf_pos + i in 373 373 + let k = t.buf_pos + len - 1 - i in 374 374 + let tmp = Bytes.get t.buffer j in 375 375 + Bytes.set t.buffer j (Bytes.get t.buffer k); 376 376 + Bytes.set t.buffer k tmp 377 377 + done; 378 378 + Slice.make t.buffer ~first:t.buf_pos ~length:len 379 379 + 380 380 + let finalize t = 381 381 + Slice.to_bytes (finalize_to_slice t) 382 382 + 383 383 + let current_size t = 384 384 + Bytes.length t.buffer - t.buf_pos + (t.num_bits + 7) / 8 385 385 + end

+267

src/bitstream.mli

··· 1 1 + (** Bitstream - Bit-level I/O for binary formats. 2 2 + 3 3 + This library provides efficient bit-level reading and writing for parsing 4 4 + and generating binary formats. It supports both forward (start-to-end) and 5 5 + backward (end-to-start) operations, as required by various compression 6 6 + algorithms. 7 7 + 8 8 + {1 Overview} 9 9 + 10 10 + {[ 11 11 + (* Forward reading from a slice (zero-copy) *) 12 12 + let slice = { Bitstream.Slice.bytes = data; first = 0; length = n } in 13 13 + let r = Bitstream.Forward_reader.of_slice slice in 14 14 + let magic = Bitstream.Forward_reader.read_bits r 32 in 15 15 + let flags = Bitstream.Forward_reader.read_bits r 8 in 16 16 + 17 17 + (* Get remaining data as a slice (zero-copy) *) 18 18 + let remaining = Bitstream.Forward_reader.to_slice r in 19 19 + 20 20 + (* Backward reading - for FSE/ANS entropy decoding *) 21 21 + let r = Bitstream.Backward_reader.of_slice slice in 22 22 + let symbol = Bitstream.Backward_reader.read_bits r num_bits 23 23 + ]} 24 24 + 25 25 + {1 Bytesrw Compatibility} 26 26 + 27 27 + The {!Slice} type is structurally compatible with [Bytesrw.Bytes.Slice.t], 28 28 + enabling zero-copy integration with bytesrw-based streaming. All reader 29 29 + and writer constructors accept slices as the primary input type. 30 30 + 31 31 + {1 Error Handling} 32 32 + 33 33 + Operations raise exceptions on error: 34 34 + - {!End_of_stream}: Reading past end of stream 35 35 + - {!Invalid_state}: Operation requires specific state (e.g., byte alignment) 36 36 + - {!Corrupted_stream}: Malformed stream data *) 37 37 + 38 38 + (** {1 Slice Type} 39 39 + 40 40 + A slice is a view into a byte buffer. This type is structurally compatible 41 41 + with [Bytesrw.Bytes.Slice.t], enabling zero-copy interop. *) 42 42 + 43 43 + module Slice : sig 44 44 + type t = { 45 45 + bytes : bytes; 46 46 + first : int; 47 47 + length : int; 48 48 + } 49 49 + (** A slice referencing [length] bytes starting at [first] in [bytes]. 50 50 + This is structurally identical to [Bytesrw.Bytes.Slice.t]. *) 51 51 + 52 52 + val make : bytes -> first:int -> length:int -> t 53 53 + (** [make bytes ~first ~length] creates a slice. *) 54 54 + 55 55 + val of_bytes : ?first:int -> ?length:int -> bytes -> t 56 56 + (** [of_bytes bytes] creates a slice for the entire buffer. 57 57 + Optional [first] and [length] can restrict the range. *) 58 58 + 59 59 + val to_bytes : t -> bytes 60 60 + (** [to_bytes t] copies the slice contents to a new buffer. *) 61 61 + 62 62 + val is_empty : t -> bool 63 63 + (** [is_empty t] returns true if the slice has zero length. *) 64 64 + 65 65 + val sub : t -> first:int -> length:int -> t 66 66 + (** [sub t ~first ~length] creates a sub-slice. [first] is relative to [t]. *) 67 67 + end 68 68 + 69 69 + (** {1 Exceptions} *) 70 70 + 71 71 + exception End_of_stream 72 72 + (** Raised when attempting to read past the end of the stream. *) 73 73 + 74 74 + exception Invalid_state of string 75 75 + (** Raised when an operation requires a specific state (e.g., byte alignment). *) 76 76 + 77 77 + exception Corrupted_stream of string 78 78 + (** Raised when stream data is malformed (e.g., invalid padding marker). *) 79 79 + 80 80 + (** {1 Forward Bitstream Reader} *) 81 81 + 82 82 + module Forward_reader : sig 83 83 + (** Forward bitstream reader state. *) 84 84 + type t 85 85 + 86 86 + val of_slice : Slice.t -> t 87 87 + (** [of_slice slice] creates a reader from a slice. Zero-copy. *) 88 88 + 89 89 + val of_bytes : bytes -> t 90 90 + (** [of_bytes src] creates a reader for the entire byte buffer. *) 91 91 + 92 92 + val create : bytes -> pos:int -> len:int -> t 93 93 + (** [create src ~pos ~len] creates a reader for [len] bytes starting at [pos]. *) 94 94 + 95 95 + val remaining : t -> int 96 96 + (** [remaining t] returns the number of unread bits. *) 97 97 + 98 98 + val is_byte_aligned : t -> bool 99 99 + (** [is_byte_aligned t] returns true if the reader is at a byte boundary. *) 100 100 + 101 101 + val read_bits : t -> int -> int 102 102 + (** [read_bits t n] reads and returns [n] bits (1-57) in little-endian order. 103 103 + @raise End_of_stream if not enough data available. 104 104 + @raise Invalid_argument if [n > 57]. *) 105 105 + 106 106 + val read_byte : t -> int 107 107 + (** [read_byte t] reads and returns the next byte (0-255). 108 108 + @raise Invalid_state if not byte aligned. 109 109 + @raise End_of_stream if at end of stream. *) 110 110 + 111 111 + val rewind_bits : t -> int -> unit 112 112 + (** [rewind_bits t n] rewinds the stream by [n] bits. 113 113 + @raise End_of_stream if rewinding past the start. *) 114 114 + 115 115 + val align : t -> unit 116 116 + (** [align t] advances to the next byte boundary if not already aligned. *) 117 117 + 118 118 + val byte_position : t -> int 119 119 + (** [byte_position t] returns the current byte position. 120 120 + @raise Invalid_state if not byte aligned. *) 121 121 + 122 122 + val get_slice : t -> int -> Slice.t 123 123 + (** [get_slice t n] returns the next [n] bytes as a slice (zero-copy). 124 124 + The slice references the underlying buffer directly. 125 125 + @raise Invalid_state if not byte aligned. 126 126 + @raise End_of_stream if not enough data. *) 127 127 + 128 128 + val get_bytes : t -> int -> bytes 129 129 + (** [get_bytes t n] reads and returns the next [n] bytes as a new buffer. 130 130 + Equivalent to [Slice.to_bytes (get_slice t n)]. 131 131 + @raise Invalid_state if not byte aligned. 132 132 + @raise End_of_stream if not enough data. *) 133 133 + 134 134 + val to_slice : t -> Slice.t 135 135 + (** [to_slice t] returns the remaining data as a slice (zero-copy). 136 136 + @raise Invalid_state if not byte aligned. *) 137 137 + 138 138 + val advance : t -> int -> unit 139 139 + (** [advance t n] skips [n] bytes without returning them. 140 140 + @raise Invalid_state if not byte aligned. 141 141 + @raise End_of_stream if not enough data. *) 142 142 + 143 143 + val sub : t -> int -> t 144 144 + (** [sub t n] creates a sub-reader for the next [n] bytes and advances [t]. 145 145 + @raise Invalid_state if not byte aligned. 146 146 + @raise End_of_stream if not enough data. *) 147 147 + 148 148 + val remaining_bytes : t -> int 149 149 + (** [remaining_bytes t] returns the number of unread bytes. 150 150 + @raise Invalid_state if not byte aligned. *) 151 151 + 152 152 + val skip_bits : t -> int -> unit 153 153 + (** [skip_bits t n] skips [n] bits without returning them. 154 154 + @raise End_of_stream if not enough data. *) 155 155 + end 156 156 + 157 157 + (** {1 Backward Bitstream Reader} 158 158 + 159 159 + Reads bits from the end of a buffer towards the start. The stream format 160 160 + includes a padding marker: the highest 1-bit in the final byte indicates 161 161 + where actual data begins. 162 162 + 163 163 + This format is used by FSE and ANS entropy coders. *) 164 164 + 165 165 + module Backward_reader : sig 166 166 + (** Backward bitstream reader state. *) 167 167 + type t 168 168 + 169 169 + val of_slice : Slice.t -> t 170 170 + (** [of_slice slice] creates a backward reader from a slice. Zero-copy. 171 171 + @raise End_of_stream if slice is empty. 172 172 + @raise Corrupted_stream if padding marker is invalid. *) 173 173 + 174 174 + val of_bytes : bytes -> pos:int -> len:int -> t 175 175 + (** [of_bytes src ~pos ~len] creates a backward reader. 176 176 + The stream is read from position [pos + len - 1] towards [pos]. 177 177 + @raise End_of_stream if [len = 0]. 178 178 + @raise Corrupted_stream if padding marker is invalid. *) 179 179 + 180 180 + val remaining : t -> int 181 181 + (** [remaining t] returns the number of bits remaining. *) 182 182 + 183 183 + val is_empty : t -> bool 184 184 + (** [is_empty t] returns true if no more bits are available. *) 185 185 + 186 186 + val read_bits : t -> int -> int 187 187 + (** [read_bits t n] reads and returns [n] bits (1-57). 188 188 + Returns 0 bits when reading past the beginning. 189 189 + @raise Invalid_argument if [n > 57]. *) 190 190 + 191 191 + val peek_bits : t -> int -> int 192 192 + (** [peek_bits t n] returns the next [n] bits without consuming them. 193 193 + @raise Invalid_argument if [n > 57]. *) 194 194 + end 195 195 + 196 196 + (** {1 Forward Bitstream Writer} *) 197 197 + 198 198 + module Forward_writer : sig 199 199 + (** Forward bitstream writer state. *) 200 200 + type t 201 201 + 202 202 + val of_slice : Slice.t -> t 203 203 + (** [of_slice slice] creates a writer into a slice. Zero-copy. *) 204 204 + 205 205 + val of_bytes : bytes -> t 206 206 + (** [of_bytes dst] creates a writer starting at position 0. *) 207 207 + 208 208 + val create : bytes -> pos:int -> t 209 209 + (** [create dst ~pos] creates a writer starting at [pos] in buffer [dst]. *) 210 210 + 211 211 + val write_bits : t -> int -> int -> unit 212 212 + (** [write_bits t value n] writes the lower [n] bits (1-57) of [value] 213 213 + in little-endian order. 214 214 + @raise Invalid_argument if [n > 57]. *) 215 215 + 216 216 + val write_byte : t -> int -> unit 217 217 + (** [write_byte t value] writes a single byte. Flushes any partial bits first. *) 218 218 + 219 219 + val write_slice : t -> Slice.t -> unit 220 220 + (** [write_slice t slice] writes bytes from a slice. Flushes any partial bits first. *) 221 221 + 222 222 + val write_bytes : t -> bytes -> unit 223 223 + (** [write_bytes t src] writes all bytes from [src]. Flushes any partial bits first. *) 224 224 + 225 225 + val byte_position : t -> int 226 226 + (** [byte_position t] returns the current output position including any partial byte. *) 227 227 + 228 228 + val flush : t -> unit 229 229 + (** [flush t] writes any accumulated bits as a partial byte. *) 230 230 + 231 231 + val finalize : t -> int 232 232 + (** [finalize t] flushes and returns the total number of bytes written. *) 233 233 + 234 234 + val to_slice : t -> Slice.t 235 235 + (** [to_slice t] flushes and returns the written data as a slice (zero-copy). 236 236 + The slice references the underlying destination buffer. *) 237 237 + end 238 238 + 239 239 + (** {1 Backward Bitstream Writer} 240 240 + 241 241 + Accumulates bits to produce output that will be read backwards. 242 242 + Used for FSE and Huffman encoding. *) 243 243 + 244 244 + module Backward_writer : sig 245 245 + (** Backward bitstream writer state. *) 246 246 + type t 247 247 + 248 248 + val create : int -> t 249 249 + (** [create size] creates a writer with an internal buffer of [size] bytes. *) 250 250 + 251 251 + val write_bits : t -> int -> int -> unit 252 252 + (** [write_bits t value n] accumulates [n] bits from [value]. *) 253 253 + 254 254 + val flush_bytes : t -> unit 255 255 + (** [flush_bytes t] flushes complete bytes to the internal buffer. *) 256 256 + 257 257 + val finalize_to_slice : t -> Slice.t 258 258 + (** [finalize_to_slice t] adds the padding marker, flushes, and returns output 259 259 + as a slice (zero-copy). The slice references the internal buffer. *) 260 260 + 261 261 + val finalize : t -> bytes 262 262 + (** [finalize t] adds the padding marker, flushes, and returns the output. 263 263 + Equivalent to [Slice.to_bytes (finalize_to_slice t)]. *) 264 264 + 265 265 + val current_size : t -> int 266 266 + (** [current_size t] returns the current output size estimate. *) 267 267 + end

+4

src/dune

··· 1 1 + (library 2 2 + (name bitstream) 3 3 + (public_name bitstream) 4 4 + (ocamlopt_flags (:standard -O3)))

+3

test/dune

··· 1 1 + (test 2 2 + (name test_bitstream) 3 3 + (libraries bitstream alcotest))

+221

test/test_bitstream.ml

··· 1 1 + (** Tests for Bitstream library *) 2 2 + 3 3 + let test_forward_reader_bytes () = 4 4 + let data = Bytes.of_string "\x12\x34\x56\x78" in 5 5 + let r = Bitstream.Forward_reader.of_bytes data in 6 6 + Alcotest.(check int) "byte 0" 0x12 (Bitstream.Forward_reader.read_byte r); 7 7 + Alcotest.(check int) "byte 1" 0x34 (Bitstream.Forward_reader.read_byte r); 8 8 + Alcotest.(check int) "byte 2" 0x56 (Bitstream.Forward_reader.read_byte r); 9 9 + Alcotest.(check int) "byte 3" 0x78 (Bitstream.Forward_reader.read_byte r); 10 10 + (* Reading past end raises End_of_stream *) 11 11 + Alcotest.check_raises "past end" Bitstream.End_of_stream (fun () -> 12 12 + ignore (Bitstream.Forward_reader.read_byte r)) 13 13 + 14 14 + let test_forward_reader_bits () = 15 15 + (* 0x12 = 0001_0010, 0x34 = 0011_0100 in little-endian bits: 16 16 + Reading 4 bits: 0010 = 2 17 17 + Reading 4 bits: 0001 = 1 18 18 + Reading 8 bits: 0011_0100 = 0x34 *) 19 19 + let data = Bytes.of_string "\x12\x34" in 20 20 + let r = Bitstream.Forward_reader.of_bytes data in 21 21 + Alcotest.(check int) "4 bits" 0x2 (Bitstream.Forward_reader.read_bits r 4); 22 22 + Alcotest.(check int) "4 bits" 0x1 (Bitstream.Forward_reader.read_bits r 4); 23 23 + Alcotest.(check int) "8 bits" 0x34 (Bitstream.Forward_reader.read_bits r 8) 24 24 + 25 25 + let test_forward_reader_mixed () = 26 26 + let data = Bytes.of_string "\xFF\x00\xAB\xCD" in 27 27 + let r = Bitstream.Forward_reader.of_bytes data in 28 28 + Alcotest.(check int) "16 bits LE" 0x00FF (Bitstream.Forward_reader.read_bits r 16); 29 29 + Alcotest.(check int) "remaining bits" 16 (Bitstream.Forward_reader.remaining r); 30 30 + Alcotest.(check int) "remaining bytes" 2 (Bitstream.Forward_reader.remaining_bytes r); 31 31 + let rest = Bitstream.Forward_reader.get_bytes r 2 in 32 32 + Alcotest.(check string) "get_bytes" "\xAB\xCD" (Bytes.to_string rest) 33 33 + 34 34 + let test_forward_reader_32bit () = 35 35 + let data = Bytes.of_string "\x78\x56\x34\x12" in 36 36 + let r = Bitstream.Forward_reader.of_bytes data in 37 37 + Alcotest.(check int) "32 bits LE" 0x12345678 (Bitstream.Forward_reader.read_bits r 32) 38 38 + 39 39 + let test_forward_reader_rewind () = 40 40 + let data = Bytes.of_string "\x12\x34\x56" in 41 41 + let r = Bitstream.Forward_reader.of_bytes data in 42 42 + let v1 = Bitstream.Forward_reader.read_bits r 12 in 43 43 + Alcotest.(check int) "first read 12 bits" 0x412 v1; (* 0x12 + lower 4 of 0x34 *) 44 44 + Bitstream.Forward_reader.rewind_bits r 4; 45 45 + Alcotest.(check int) "remaining after rewind" 16 (Bitstream.Forward_reader.remaining r); 46 46 + let v2 = Bitstream.Forward_reader.read_bits r 8 in 47 47 + Alcotest.(check int) "read after rewind" 0x34 v2 (* now at byte 1 *) 48 48 + 49 49 + let test_forward_reader_align () = 50 50 + let data = Bytes.of_string "\xFF\xAA\xBB" in 51 51 + let r = Bitstream.Forward_reader.of_bytes data in 52 52 + let _ = Bitstream.Forward_reader.read_bits r 3 in 53 53 + Alcotest.(check bool) "not aligned" false (Bitstream.Forward_reader.is_byte_aligned r); 54 54 + Bitstream.Forward_reader.align r; 55 55 + Alcotest.(check bool) "aligned" true (Bitstream.Forward_reader.is_byte_aligned r); 56 56 + Alcotest.(check int) "after align" 0xAA (Bitstream.Forward_reader.read_byte r) 57 57 + 58 58 + let test_forward_reader_sub () = 59 59 + let data = Bytes.of_string "\x12\x34\x56\x78\x9A" in 60 60 + let r = Bitstream.Forward_reader.of_bytes data in 61 61 + let _ = Bitstream.Forward_reader.read_byte r in 62 62 + let sub = Bitstream.Forward_reader.sub r 2 in 63 63 + Alcotest.(check int) "sub byte 0" 0x34 (Bitstream.Forward_reader.read_byte sub); 64 64 + Alcotest.(check int) "sub byte 1" 0x56 (Bitstream.Forward_reader.read_byte sub); 65 65 + Alcotest.(check int) "parent continues" 0x78 (Bitstream.Forward_reader.read_byte r) 66 66 + 67 67 + let test_forward_writer_bytes () = 68 68 + let buf = Bytes.create 8 in 69 69 + let w = Bitstream.Forward_writer.of_bytes buf in 70 70 + Bitstream.Forward_writer.write_byte w 0x12; 71 71 + Bitstream.Forward_writer.write_byte w 0x34; 72 72 + let len = Bitstream.Forward_writer.finalize w in 73 73 + Alcotest.(check int) "length" 2 len; 74 74 + Alcotest.(check int) "byte 0" 0x12 (Bytes.get_uint8 buf 0); 75 75 + Alcotest.(check int) "byte 1" 0x34 (Bytes.get_uint8 buf 1) 76 76 + 77 77 + let test_forward_writer_bits () = 78 78 + let buf = Bytes.create 8 in 79 79 + let w = Bitstream.Forward_writer.of_bytes buf in 80 80 + Bitstream.Forward_writer.write_bits w 0x2 4; (* lower 4 bits *) 81 81 + Bitstream.Forward_writer.write_bits w 0x1 4; (* upper 4 bits *) 82 82 + Bitstream.Forward_writer.write_bits w 0x34 8; 83 83 + let len = Bitstream.Forward_writer.finalize w in 84 84 + Alcotest.(check int) "length" 2 len; 85 85 + Alcotest.(check int) "byte 0" 0x12 (Bytes.get_uint8 buf 0); 86 86 + Alcotest.(check int) "byte 1" 0x34 (Bytes.get_uint8 buf 1) 87 87 + 88 88 + let test_forward_writer_32bit () = 89 89 + let buf = Bytes.create 8 in 90 90 + let w = Bitstream.Forward_writer.of_bytes buf in 91 91 + Bitstream.Forward_writer.write_bits w 0x12345678 32; 92 92 + let len = Bitstream.Forward_writer.finalize w in 93 93 + Alcotest.(check int) "length" 4 len; 94 94 + Alcotest.(check int) "byte 0" 0x78 (Bytes.get_uint8 buf 0); 95 95 + Alcotest.(check int) "byte 1" 0x56 (Bytes.get_uint8 buf 1); 96 96 + Alcotest.(check int) "byte 2" 0x34 (Bytes.get_uint8 buf 2); 97 97 + Alcotest.(check int) "byte 3" 0x12 (Bytes.get_uint8 buf 3) 98 98 + 99 99 + let test_forward_roundtrip () = 100 100 + (* Write various bit patterns, then read them back *) 101 101 + let buf = Bytes.create 64 in 102 102 + let w = Bitstream.Forward_writer.of_bytes buf in 103 103 + Bitstream.Forward_writer.write_bits w 0b101 3; 104 104 + Bitstream.Forward_writer.write_bits w 0b11001 5; 105 105 + Bitstream.Forward_writer.write_bits w 0xABCD 16; 106 106 + Bitstream.Forward_writer.write_bits w 0b1111 4; 107 107 + let len = Bitstream.Forward_writer.finalize w in 108 108 + 109 109 + let r = Bitstream.Forward_reader.create buf ~pos:0 ~len in 110 110 + Alcotest.(check int) "3 bits" 0b101 (Bitstream.Forward_reader.read_bits r 3); 111 111 + Alcotest.(check int) "5 bits" 0b11001 (Bitstream.Forward_reader.read_bits r 5); 112 112 + Alcotest.(check int) "16 bits" 0xABCD (Bitstream.Forward_reader.read_bits r 16); 113 113 + Alcotest.(check int) "4 bits" 0b1111 (Bitstream.Forward_reader.read_bits r 4) 114 114 + 115 115 + let test_backward_roundtrip () = 116 116 + (* Backward streams are read in REVERSE order of writing. 117 117 + This matches FSE: encode in reverse order, decode in forward order. *) 118 118 + let w = Bitstream.Backward_writer.create 64 in 119 119 + Bitstream.Backward_writer.write_bits w 0b101 3; 120 120 + Bitstream.Backward_writer.write_bits w 0b11001 5; 121 121 + Bitstream.Backward_writer.write_bits w 0xAB 8; 122 122 + let data = Bitstream.Backward_writer.finalize w in 123 123 + 124 124 + let r = Bitstream.Backward_reader.of_bytes data ~pos:0 ~len:(Bytes.length data) in 125 125 + (* Read in REVERSE order (last written = first read) *) 126 126 + Alcotest.(check int) "8 bits (written last)" 0xAB (Bitstream.Backward_reader.read_bits r 8); 127 127 + Alcotest.(check int) "5 bits" 0b11001 (Bitstream.Backward_reader.read_bits r 5); 128 128 + Alcotest.(check int) "3 bits (written first)" 0b101 (Bitstream.Backward_reader.read_bits r 3) 129 129 + 130 130 + let test_backward_reader_peek () = 131 131 + (* For backward streams, bits are read from MSB to LSB within accumulated data. 132 132 + 0x5A = 0101_1010 binary. Reading 4 bits at a time gives: 133 133 + - First 4 bits (high nibble): 0101 = 0x5 134 134 + - Last 4 bits (low nibble): 1010 = 0xA *) 135 135 + let w = Bitstream.Backward_writer.create 64 in 136 136 + Bitstream.Backward_writer.write_bits w 0x5A 8; 137 137 + let data = Bitstream.Backward_writer.finalize w in 138 138 + 139 139 + let r = Bitstream.Backward_reader.of_bytes data ~pos:0 ~len:(Bytes.length data) in 140 140 + Alcotest.(check int) "peek 4 (high nibble)" 0x5 (Bitstream.Backward_reader.peek_bits r 4); 141 141 + Alcotest.(check int) "peek 4 again" 0x5 (Bitstream.Backward_reader.peek_bits r 4); 142 142 + Alcotest.(check int) "read 4" 0x5 (Bitstream.Backward_reader.read_bits r 4); 143 143 + Alcotest.(check int) "read 4 (low nibble)" 0xA (Bitstream.Backward_reader.read_bits r 4) 144 144 + 145 145 + let test_backward_is_empty () = 146 146 + let w = Bitstream.Backward_writer.create 64 in 147 147 + Bitstream.Backward_writer.write_bits w 0xFF 8; 148 148 + let data = Bitstream.Backward_writer.finalize w in 149 149 + 150 150 + let r = Bitstream.Backward_reader.of_bytes data ~pos:0 ~len:(Bytes.length data) in 151 151 + Alcotest.(check bool) "not empty" false (Bitstream.Backward_reader.is_empty r); 152 152 + let _ = Bitstream.Backward_reader.read_bits r 8 in 153 153 + Alcotest.(check bool) "empty after read" true (Bitstream.Backward_reader.is_empty r) 154 154 + 155 155 + let test_backward_empty_stream () = 156 156 + (* Empty stream should raise End_of_stream *) 157 157 + Alcotest.check_raises "empty stream" Bitstream.End_of_stream (fun () -> 158 158 + ignore (Bitstream.Backward_reader.of_bytes (Bytes.empty) ~pos:0 ~len:0)) 159 159 + 160 160 + let test_backward_invalid_padding () = 161 161 + (* Zero byte has no padding marker - should raise *) 162 162 + let data = Bytes.of_string "\x00" in 163 163 + Alcotest.check_raises "zero padding" (Bitstream.Corrupted_stream "invalid padding marker") (fun () -> 164 164 + ignore (Bitstream.Backward_reader.of_bytes data ~pos:0 ~len:1)) 165 165 + 166 166 + let test_edge_cases () = 167 167 + (* Zero bits *) 168 168 + let buf = Bytes.create 8 in 169 169 + let w = Bitstream.Forward_writer.of_bytes buf in 170 170 + Bitstream.Forward_writer.write_bits w 0 0; 171 171 + let len = Bitstream.Forward_writer.finalize w in 172 172 + Alcotest.(check int) "zero bits" 0 len; 173 173 + 174 174 + (* Read zero bits *) 175 175 + let data = Bytes.of_string "\xFF" in 176 176 + let r = Bitstream.Forward_reader.of_bytes data in 177 177 + Alcotest.(check int) "read 0 bits" 0 (Bitstream.Forward_reader.read_bits r 0); 178 178 + Alcotest.(check int) "byte still available" 0xFF (Bitstream.Forward_reader.read_byte r) 179 179 + 180 180 + let test_not_aligned_errors () = 181 181 + let data = Bytes.of_string "\xFF\xAA" in 182 182 + let r = Bitstream.Forward_reader.of_bytes data in 183 183 + let _ = Bitstream.Forward_reader.read_bits r 3 in 184 184 + 185 185 + Alcotest.check_raises "read_byte not aligned" 186 186 + (Bitstream.Invalid_state "read_byte: not byte aligned") 187 187 + (fun () -> ignore (Bitstream.Forward_reader.read_byte r)); 188 188 + 189 189 + Alcotest.check_raises "byte_position not aligned" 190 190 + (Bitstream.Invalid_state "byte_position: not byte aligned") 191 191 + (fun () -> ignore (Bitstream.Forward_reader.byte_position r)); 192 192 + 193 193 + Alcotest.check_raises "remaining_bytes not aligned" 194 194 + (Bitstream.Invalid_state "remaining_bytes: not byte aligned") 195 195 + (fun () -> ignore (Bitstream.Forward_reader.remaining_bytes r)) 196 196 + 197 197 + let tests = [ 198 198 + "forward reader bytes", `Quick, test_forward_reader_bytes; 199 199 + "forward reader bits", `Quick, test_forward_reader_bits; 200 200 + "forward reader mixed", `Quick, test_forward_reader_mixed; 201 201 + "forward reader 32bit", `Quick, test_forward_reader_32bit; 202 202 + "forward reader rewind", `Quick, test_forward_reader_rewind; 203 203 + "forward reader align", `Quick, test_forward_reader_align; 204 204 + "forward reader sub", `Quick, test_forward_reader_sub; 205 205 + "forward writer bytes", `Quick, test_forward_writer_bytes; 206 206 + "forward writer bits", `Quick, test_forward_writer_bits; 207 207 + "forward writer 32bit", `Quick, test_forward_writer_32bit; 208 208 + "forward roundtrip", `Quick, test_forward_roundtrip; 209 209 + "backward roundtrip", `Quick, test_backward_roundtrip; 210 210 + "backward reader peek", `Quick, test_backward_reader_peek; 211 211 + "backward is_empty", `Quick, test_backward_is_empty; 212 212 + "backward empty stream", `Quick, test_backward_empty_stream; 213 213 + "backward invalid padding", `Quick, test_backward_invalid_padding; 214 214 + "edge cases", `Quick, test_edge_cases; 215 215 + "not aligned errors", `Quick, test_not_aligned_errors; 216 216 + ] 217 217 + 218 218 + let () = 219 219 + Alcotest.run "Bitstream" [ 220 220 + "bitstream", tests; 221 221 + ]