Add dry-run mode to bushel sync command · anil.recoil.org/srcsetter@6c00f6d

+5 -82

bin/srcsetter.ml

··· 15 15 PERFORMANCE OF THIS SOFTWARE. 16 16 *) 17 17 18 - module SC = Srcsetter_cmd 19 - 20 - let min_interval = Some (Mtime.Span.of_uint64_ns 1000L) 21 - 22 - let stage1 { SC.img_exts; src_dir; _ } = 23 - let filter f = List.exists (Filename.check_suffix ("." ^ f)) img_exts in 24 - let fs = SC.file_seq ~filter src_dir in 25 - let total = Seq.length fs in 26 - Format.printf "[1/3] Scanned %d images from %a.\n%!" total Eio.Path.pp src_dir; 27 - fs 28 - 29 - let stage2 ({ SC.max_fibers; dst_dir; _ } as cfg) fs = 30 - let display = 31 - Progress.Display.start 32 - ~config:(Progress.Config.v ~persistent:false ~min_interval ()) 33 - (SC.main_bar_heading "[2/3] Processing images..." (Seq.length fs)) 34 - in 35 - let [ _; main_rep ] = Progress.Display.reporters display in 36 - let ents = ref [] in 37 - SC.iter_seq_p ~max_fibers 38 - (fun src -> 39 - let ent = SC.process_file cfg (display, main_rep) src in 40 - ents := ent :: !ents) 41 - fs; 42 - Progress.Display.finalise display; 43 - Format.printf "[2/3] Processed %d images to %a.\n%!" (List.length !ents) 44 - Eio.Path.pp dst_dir; 45 - !ents 46 - 47 - let stage3 ({ SC.dst_dir; max_fibers; _ } as cfg) ents = 48 - let ents_seq = List.to_seq ents in 49 - let oents = ref [] in 50 - let display = 51 - Progress.Display.start 52 - ~config:(Progress.Config.v ~persistent:false ~min_interval ()) 53 - (SC.main_bar_heading "[3/3] Verifying images..." (List.length ents)) 54 - in 55 - let [ _; rep ] = Progress.Display.reporters display in 56 - SC.iter_seq_p ~max_fibers 57 - (fun ent -> 58 - let w, h = SC.dims cfg Eio.Path.(dst_dir / Srcsetter.name ent) in 59 - let variants = 60 - Srcsetter.MS.bindings ent.variants 61 - |> List.map (fun (k, _) -> (k, SC.dims cfg Eio.Path.(dst_dir / k))) 62 - |> Srcsetter.MS.of_list 63 - in 64 - rep 1; 65 - oents := { ent with Srcsetter.dims = (w, h); variants } :: !oents) 66 - ents_seq; 67 - Progress.Display.finalise display; 68 - Printf.printf "[3/3] Verified %d generated image sizes.\n%!" 69 - (List.length ents); 70 - !oents 71 - 72 18 let _ = 73 19 (* TODO cmdliner *) 74 20 Eio_main.run @@ fun env -> 75 21 Eio.Switch.run @@ fun _ -> 22 + let fs = Eio.Stdenv.fs env in 76 23 let path_env p = 77 - if String.starts_with ~prefix:"/" p then Eio.(Path.(Stdenv.fs env / p)) 78 - else Eio.(Path.(Stdenv.cwd env / p)) 24 + if String.starts_with ~prefix:"/" p then Eio.Path.(fs / p) 25 + else Eio.Path.(Eio.Stdenv.cwd env / p) 79 26 in 80 27 let src_dir = path_env Sys.argv.(1) in 81 28 let dst_dir = path_env Sys.argv.(2) in 82 29 let proc_mgr = Eio.Stdenv.process_mgr env in 83 - let idx_file = "index.json" in 84 - let img_widths = 85 - [ 320; 480; 640; 768; 1024; 1280; 1440; 1600; 1920; 2560; 3840 ] 86 - in 87 - let img_exts = [ "png"; "webp"; "jpeg"; "jpg"; "bmp"; "heic"; "gif" ] in 88 - let img_widths = List.sort (fun a b -> compare b a) img_widths in 89 - let max_fibers = 8 in 90 - let cfg = 91 - { 92 - Srcsetter_cmd.dummy = false; 93 - preserve = true; 94 - proc_mgr; 95 - src_dir; 96 - dst_dir; 97 - idx_file; 98 - img_widths; 99 - img_exts; 100 - max_fibers; 101 - } 102 - in 103 - let fs = stage1 cfg in 104 - let ents = stage2 cfg fs in 105 - let oents = stage3 cfg ents in 106 - let j = Srcsetter.list_to_json oents |> Result.get_ok in 107 - let idx = Eio.Path.(dst_dir / idx_file) in 108 - Eio.Path.save ~append:false ~create:(`Or_truncate 0o644) idx j 30 + let _entries = Srcsetter_cmd.run ~proc_mgr ~src_dir ~dst_dir () in 31 + ()

+1 -1

lib/dune

··· 2 2 (name srcsetter_cmd) 3 3 (public_name srcsetter-cmd) 4 4 (modules srcsetter_cmd) 5 - (libraries srcsetter eio fpath progress)) 5 + (libraries srcsetter eio fpath progress mtime)) 6 6 7 7 (library 8 8 (name srcsetter)

+166 -21

lib/srcsetter_cmd.ml

··· 101 101 let output = Process.parse_out proc_mgr Buf_read.take_all args in 102 102 Scanf.sscanf output "%d %d" (fun w h -> (w, h)) 103 103 104 + (** [try_dims cfg path] returns [Some (w, h)] if identify succeeds, [None] otherwise. *) 105 + let try_dims cfg path = 106 + try Some (dims cfg path) 107 + with _ -> None 108 + 109 + (** [file_size path] returns the size of the file in bytes. *) 110 + let file_size path = 111 + let stat = Path.stat ~follow:true path in 112 + Optint.Int63.to_int stat.size 113 + 114 + (** [is_valid_image cfg path] returns true if the file exists, has non-zero size, 115 + and identify can read its dimensions. *) 116 + let is_valid_image cfg path = 117 + Path.is_file path && 118 + file_size path > 0 && 119 + Option.is_some (try_dims cfg path) 120 + 121 + (** [width_from_variant_name name] extracts the width from a variant filename. 122 + 123 + Variant filenames have the form "path/name.WIDTH.webp". Returns [None] for 124 + base images (no width suffix). *) 125 + let width_from_variant_name name = 126 + let base = Filename.chop_extension name in (* remove .webp *) 127 + let parts = String.split_on_char '.' base in 128 + match List.rev parts with 129 + | last :: _ -> ( 130 + match int_of_string_opt last with 131 + | Some w -> Some w 132 + | None -> None) 133 + | [] -> None 134 + 104 135 (** [run cfg args] executes a shell command unless in dummy mode. *) 105 136 let run { dummy; proc_mgr; _ } args = 106 137 if not dummy then Process.run proc_mgr args ··· 166 197 let dst = Path.(dst_dir / dst_file) in 167 198 (src_file, dst_file, w, needs_conversion ~preserve dst) 168 199 169 - (** [calc_needed cfg ~img_widths ~w src] computes which conversions are needed. 170 - 171 - Returns [(base, variants)] where each is tagged with [`Exists] or [`Todo]. *) 172 - let calc_needed { src_dir; dst_dir; preserve; _ } ~img_widths ~w src = 173 - let check_dst fname tw = 174 - let dst = Path.(dst_dir / fname) in 175 - let ent = (src, dst, tw) in 176 - if preserve && Path.is_file dst then `Exists ent else `Todo ent 177 - in 178 - let file = relativize_path src_dir src in 179 - let base_name = Filename.chop_extension file in 180 - let base = check_dst (Printf.sprintf "%s.webp" base_name) w in 181 - let variants = 182 - List.filter_map 183 - (fun tw -> 184 - if tw <= w then Some (check_dst (Printf.sprintf "%s.%d.webp" base_name tw) tw) 185 - else None) 186 - img_widths 187 - in 188 - (base, variants) 189 - 190 200 (** {1 Progress Bar Rendering} *) 191 201 192 202 (** [main_bar total] creates a progress bar for [total] items. *) ··· 269 279 main_rep 1 270 280 end; 271 281 ent 282 + 283 + (** {1 Pipeline Execution} *) 284 + 285 + let min_interval = Some (Mtime.Span.of_uint64_ns 1000L) 286 + 287 + (** [stage1 cfg] scans for images in the source directory. 288 + 289 + Returns a sequence of file paths matching the configured extensions. *) 290 + let stage1 { img_exts; src_dir; _ } = 291 + let filter f = List.exists (Filename.check_suffix ("." ^ f)) img_exts in 292 + let fs = file_seq ~filter src_dir in 293 + let total = Seq.length fs in 294 + Format.printf "[1/3] Scanned %d images from %a.\n%!" total Path.pp src_dir; 295 + fs 296 + 297 + (** [stage2 cfg fs] processes images, converting to WebP at multiple sizes. 298 + 299 + @return List of {!Srcsetter.t} entries with placeholder dimensions. *) 300 + let stage2 ({ max_fibers; dst_dir; _ } as cfg) fs = 301 + let display = 302 + Progress.Display.start 303 + ~config:(Progress.Config.v ~persistent:false ~min_interval ()) 304 + (main_bar_heading "[2/3] Processing images..." (Seq.length fs)) 305 + in 306 + let [ _; main_rep ] = Progress.Display.reporters display in 307 + let ents = ref [] in 308 + iter_seq_p ~max_fibers 309 + (fun src -> 310 + let ent = process_file cfg (display, main_rep) src in 311 + ents := ent :: !ents) 312 + fs; 313 + Progress.Display.finalise display; 314 + Format.printf "[2/3] Processed %d images to %a.\n%!" (List.length !ents) 315 + Path.pp dst_dir; 316 + !ents 317 + 318 + (** [stage3 cfg ents] verifies generated images and records their dimensions. 319 + 320 + Regenerates any images that have zero length or fail identify validation. 321 + 322 + @return List of {!Srcsetter.t} entries with actual dimensions. *) 323 + let stage3 ({ src_dir; dst_dir; max_fibers; _ } as cfg) ents = 324 + let ents_seq = List.to_seq ents in 325 + let oents = ref [] in 326 + let regenerated = ref 0 in 327 + let display = 328 + Progress.Display.start 329 + ~config:(Progress.Config.v ~persistent:false ~min_interval ()) 330 + (main_bar_heading "[3/3] Verifying images..." (List.length ents)) 331 + in 332 + let [ _; rep ] = Progress.Display.reporters display in 333 + iter_seq_p ~max_fibers 334 + (fun ent -> 335 + let src_path = Path.(src_dir / Srcsetter.origin ent) in 336 + let orig_w, _ = dims cfg src_path in 337 + (* Verify and regenerate base image if needed *) 338 + let base_path = Path.(dst_dir / Srcsetter.name ent) in 339 + if not (is_valid_image cfg base_path) then begin 340 + incr regenerated; 341 + convert cfg (Srcsetter.origin ent, Srcsetter.name ent, orig_w) 342 + end; 343 + let w, h = dims cfg base_path in 344 + (* Verify and regenerate variants if needed *) 345 + let variants = 346 + Srcsetter.MS.bindings ent.variants 347 + |> List.map (fun (k, _) -> 348 + let variant_path = Path.(dst_dir / k) in 349 + if not (is_valid_image cfg variant_path) then begin 350 + incr regenerated; 351 + let target_w = Option.value (width_from_variant_name k) ~default:orig_w in 352 + convert cfg (Srcsetter.origin ent, k, target_w) 353 + end; 354 + (k, dims cfg variant_path)) 355 + |> Srcsetter.MS.of_list 356 + in 357 + rep 1; 358 + oents := { ent with Srcsetter.dims = (w, h); variants } :: !oents) 359 + ents_seq; 360 + Progress.Display.finalise display; 361 + if !regenerated > 0 then 362 + Printf.printf "[3/3] Verified %d images, regenerated %d invalid outputs.\n%!" 363 + (List.length ents) !regenerated 364 + else 365 + Printf.printf "[3/3] Verified %d generated image sizes.\n%!" 366 + (List.length ents); 367 + !oents 368 + 369 + (** [run ~proc_mgr ~src_dir ~dst_dir ()] runs the full srcsetter pipeline. 370 + 371 + Scans [src_dir] for images, converts them to WebP format at multiple 372 + responsive sizes, and writes an index file to [dst_dir]. 373 + 374 + @param proc_mgr Eio process manager for running ImageMagick 375 + @param src_dir Source directory containing original images 376 + @param dst_dir Destination directory for generated images 377 + @param idx_file Name of the index file (default ["index.json"]) 378 + @param img_widths List of target widths (default common responsive breakpoints) 379 + @param img_exts List of extensions to process (default common image formats) 380 + @param max_fibers Maximum concurrent operations (default 8) 381 + @param dummy When true, skip actual conversions (default false) 382 + @param preserve When true, skip existing files (default true) 383 + @return List of {!Srcsetter.t} entries describing generated images *) 384 + let run 385 + ~proc_mgr 386 + ~src_dir 387 + ~dst_dir 388 + ?(idx_file = "index.json") 389 + ?(img_widths = [ 320; 480; 640; 768; 1024; 1280; 1440; 1600; 1920; 2560; 3840 ]) 390 + ?(img_exts = [ "png"; "webp"; "jpeg"; "jpg"; "bmp"; "heic"; "gif" ]) 391 + ?(max_fibers = 8) 392 + ?(dummy = false) 393 + ?(preserve = true) 394 + () 395 + = 396 + let img_widths = List.sort (fun a b -> compare b a) img_widths in 397 + let cfg = 398 + { 399 + dummy; 400 + preserve; 401 + proc_mgr; 402 + src_dir; 403 + dst_dir; 404 + idx_file; 405 + img_widths; 406 + img_exts; 407 + max_fibers; 408 + } 409 + in 410 + let fs = stage1 cfg in 411 + let ents = stage2 cfg fs in 412 + let oents = stage3 cfg ents in 413 + let j = Srcsetter.list_to_json oents |> Result.get_ok in 414 + let idx = Path.(dst_dir / idx_file) in 415 + Path.save ~append:false ~create:(`Or_truncate 0o644) idx j; 416 + oents

+122

lib/srcsetter_cmd.mli

··· 1 + (* Copyright (c) 2024, Anil Madhavapeddy <anil@recoil.org> 2 + 3 + Permission to use, copy, modify, and/or distribute this software for 4 + any purpose with or without fee is hereby granted, provided that the 5 + above copyright notice and this permission notice appear in all 6 + copies. 7 + 8 + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL 9 + WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED 10 + WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE 11 + AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL 12 + DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA 13 + OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER 14 + TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR 15 + PERFORMANCE OF THIS SOFTWARE. *) 16 + 17 + (** Command-line image processing operations for srcsetter. 18 + 19 + This module provides the core image processing pipeline including 20 + file discovery, image conversion, and progress reporting. 21 + 22 + {1 High-Level Pipeline} 23 + 24 + The simplest way to use this module is via {!run}, which executes 25 + the complete pipeline: 26 + 27 + {[ 28 + Srcsetter_cmd.run 29 + ~proc_mgr:(Eio.Stdenv.process_mgr env) 30 + ~src_dir:Eio.Path.(fs / "images/originals") 31 + ~dst_dir:Eio.Path.(fs / "images/output") 32 + () 33 + ]} 34 + 35 + {1 Configuration} *) 36 + 37 + (** Configuration for the image processing pipeline. *) 38 + type ('a, 'b) config = { 39 + dummy : bool; (** When true, skip actual image conversion (dry run) *) 40 + preserve : bool; (** When true, skip conversion if destination exists *) 41 + proc_mgr : 'a Eio.Process.mgr; (** Eio process manager for running ImageMagick *) 42 + src_dir : 'b Eio.Path.t; (** Source directory containing original images *) 43 + dst_dir : 'b Eio.Path.t; (** Destination directory for generated images *) 44 + img_widths : int list; (** List of target widths for responsive variants *) 45 + img_exts : string list; (** File extensions to process (e.g., ["jpg"; "png"]) *) 46 + idx_file : string; (** Name of the JSON index file to generate *) 47 + max_fibers : int; (** Maximum concurrent conversion operations *) 48 + } 49 + 50 + (** {1 File Operations} *) 51 + 52 + val file_seq : 53 + filter:(string -> bool) -> 54 + ([> Eio.Fs.dir_ty ] as 'a) Eio.Path.t -> 55 + 'a Eio.Path.t Seq.t 56 + (** [file_seq ~filter path] recursively enumerates files in [path]. 57 + 58 + Returns a sequence of file paths where [filter filename] is true. 59 + Directories are traversed depth-first. *) 60 + 61 + val iter_seq_p : ?max_fibers:int -> ('a -> unit) -> 'a Seq.t -> unit 62 + (** [iter_seq_p ?max_fibers fn seq] iterates [fn] over [seq] in parallel. 63 + 64 + @param max_fibers Optional limit on concurrent fibers. Must be positive. 65 + @raise Invalid_argument if [max_fibers] is not positive. *) 66 + 67 + (** {1 Image Operations} *) 68 + 69 + val dims : ('a, 'b) config -> 'b Eio.Path.t -> int * int 70 + (** [dims cfg path] returns the [(width, height)] dimensions of an image. 71 + 72 + Uses ImageMagick's [identify] command to read image metadata. *) 73 + 74 + val convert : ('a, 'b) config -> string * string * int -> unit 75 + (** [convert cfg (src, dst, size)] converts an image to WebP format. 76 + 77 + Creates the destination directory if needed, then uses ImageMagick 78 + to resize and convert the image with auto-orientation. 79 + 80 + @param src Source filename relative to [cfg.src_dir] 81 + @param dst Destination filename relative to [cfg.dst_dir] 82 + @param size Target width in pixels *) 83 + 84 + val convert_pdf : 85 + ('a, 'b) config -> 86 + size:string -> 87 + dst:'b Eio.Path.t -> 88 + src:'b Eio.Path.t -> 89 + unit 90 + (** [convert_pdf cfg ~size ~dst ~src] converts a PDF's first page to an image. 91 + 92 + Renders at 300 DPI, crops the top half, and resizes to the target width. *) 93 + 94 + (** {1 Pipeline Execution} *) 95 + 96 + val run : 97 + proc_mgr:'a Eio.Process.mgr -> 98 + src_dir:'b Eio.Path.t -> 99 + dst_dir:'b Eio.Path.t -> 100 + ?idx_file:string -> 101 + ?img_widths:int list -> 102 + ?img_exts:string list -> 103 + ?max_fibers:int -> 104 + ?dummy:bool -> 105 + ?preserve:bool -> 106 + unit -> 107 + Srcsetter.t list 108 + (** [run ~proc_mgr ~src_dir ~dst_dir ()] runs the full srcsetter pipeline. 109 + 110 + Scans [src_dir] for images, converts them to WebP format at multiple 111 + responsive sizes, and writes an index file to [dst_dir]. 112 + 113 + @param proc_mgr Eio process manager for running ImageMagick 114 + @param src_dir Source directory containing original images 115 + @param dst_dir Destination directory for generated images 116 + @param idx_file Name of the index file (default ["index.json"]) 117 + @param img_widths List of target widths (default common responsive breakpoints) 118 + @param img_exts List of extensions to process (default common image formats) 119 + @param max_fibers Maximum concurrent operations (default 8) 120 + @param dummy When true, skip actual conversions (default false) 121 + @param preserve When true, skip existing files (default true) 122 + @return List of {!Srcsetter.t} entries describing generated images *)