+160 -48

bin/main.ml

··· 318 318 let doc = "Path to vendor cache (overrides config and UNPAC_VENDOR_CACHE env var)." in 319 319 Arg.(value & opt (some string) None & info ["cache"] ~docv:"PATH" ~doc) 320 320 in 321 321 - let fast_arg = 322 322 - let doc = "Fast mode: skip history rewriting (useful for large repos like dune)." in 323 323 - Arg.(value & flag & info ["fast"] ~doc) 324 324 - in 325 325 - let run () pkg name_opt version_opt branch_opt solve cli_cache fast = 326 326 - let preserve_history = not fast in 321 321 + let run () pkg name_opt version_opt branch_opt solve cli_cache = 327 322 with_root @@ fun ~env:_ ~fs ~proc_mgr ~root -> 328 323 let config = load_config root in 329 324 let cache = resolve_cache ~proc_mgr ~fs ~config ~cli_cache in ··· 388 383 url; 389 384 branch = None; 390 385 } in 391 391 - match Unpac_opam.Opam.add_package ~preserve_history ~proc_mgr ~root ?cache info with 386 386 + match Unpac_opam.Opam.add_package ~proc_mgr ~root ?cache info with 392 387 | Unpac.Backend.Added { name = pkg_name; sha } -> 393 388 Format.printf "Added %s (%s)@." pkg_name (String.sub sha 0 7); 394 389 if List.length g.packages > 1 then ··· 447 442 url; 448 443 branch = branch_opt; 449 444 } in 450 450 - match Unpac_opam.Opam.add_package ~preserve_history ~proc_mgr ~root ?cache info with 445 445 + match Unpac_opam.Opam.add_package ~proc_mgr ~root ?cache info with 451 446 | Unpac.Backend.Added { name = pkg_name; sha } -> 452 447 Format.printf "Added %s (%s)@." pkg_name (String.sub sha 0 7); 453 448 Format.printf "@.Next steps:@."; ··· 461 456 end 462 457 in 463 458 let info = Cmd.info "add" ~doc in 464 464 - Cmd.v info Term.(const run $ logging_term $ pkg_arg $ name_arg $ version_arg $ branch_arg $ solve_arg $ cache_arg $ fast_arg) 459 459 + Cmd.v info Term.(const run $ logging_term $ pkg_arg $ name_arg $ version_arg $ branch_arg $ solve_arg $ cache_arg) 465 460 466 461 (* Opam list command *) 467 462 let opam_list_cmd = ··· 577 572 (* Opam merge command *) 578 573 let opam_merge_cmd = 579 574 let doc = "Merge vendored opam packages into a project. \ 580 580 - Use --all to merge all vendored packages." in 575 575 + Use --solve to merge a package and its dependencies, \ 576 576 + or --all to merge all vendored packages." in 581 577 let args = 582 582 - let doc = "With --all: PROJECT. Without --all: PACKAGE PROJECT." in 578 578 + let doc = "PACKAGE PROJECT (or just PROJECT with --all)." in 583 579 Arg.(value & pos_all string [] & info [] ~docv:"ARGS" ~doc) 584 580 in 585 581 let all_flag = 586 582 let doc = "Merge all vendored packages into the project." in 587 583 Arg.(value & flag & info ["all"] ~doc) 588 584 in 589 589 - let run () args all = 585 585 + let solve_flag = 586 586 + let doc = "Solve dependencies for PACKAGE and merge all solved packages into the project." in 587 587 + Arg.(value & flag & info ["solve"] ~doc) 588 588 + in 589 589 + let run () args all solve = 590 590 with_root @@ fun ~env:_ ~fs:_ ~proc_mgr ~root -> 591 591 - (* Parse arguments based on --all flag *) 592 592 - let pkg, project = if all then 593 593 - match args with 594 594 - | [project] -> None, project 595 595 - | _ -> 596 596 - Format.eprintf "Usage: unpac opam merge --all PROJECT@."; 597 597 - exit 1 598 598 - else 599 599 - match args with 600 600 - | [pkg; project] -> Some pkg, project 601 601 - | _ -> 602 602 - Format.eprintf "Usage: unpac opam merge PACKAGE PROJECT@."; 603 603 - exit 1 604 604 - in 605 605 - let merge_one pkg = 591 591 + let config = load_config root in 592 592 + 593 593 + let merge_one ~project pkg = 606 594 let patches_branch = Unpac_opam.Opam.patches_branch pkg in 607 595 match Unpac.Backend.merge_to_project ~proc_mgr ~root ~project ~patches_branch with 608 596 | Ok () -> 609 609 - Format.printf "Merged %s into project %s@." pkg project; 597 597 + Format.printf "Merged %s@." pkg; 610 598 true 611 599 | Error (`Conflict files) -> 612 600 Format.eprintf "Merge conflict in %s:@." pkg; 613 601 List.iter (Format.eprintf " %s@.") files; 614 602 false 615 603 in 616 616 - if all then begin 617 617 - (* Merge all vendored packages *) 618 618 - let packages = Unpac_opam.Opam.list_packages ~proc_mgr ~root in 619 619 - if packages = [] then begin 620 620 - Format.eprintf "No vendored packages to merge.@."; 621 621 - exit 1 622 622 - end; 604 604 + 605 605 + let merge_packages packages project = 623 606 Format.printf "Merging %d packages into project %s...@." (List.length packages) project; 624 607 let (successes, failures) = List.fold_left (fun (s, f) pkg -> 625 625 - if merge_one pkg then (s + 1, f) else (s, f + 1) 608 608 + if merge_one ~project pkg then (s + 1, f) else (s, f + 1) 626 609 ) (0, 0) packages in 627 610 Format.printf "@.Done: %d merged" successes; 628 611 if failures > 0 then Format.printf ", %d had conflicts" failures; ··· 632 615 exit 1 633 616 end else 634 617 Format.printf "Next: Build your project in project/%s@." project 635 635 - end else begin 636 636 - match pkg with 637 637 - | Some pkg -> 638 638 - if merge_one pkg then 639 639 - Format.printf "@.Next: Build your project in project/%s@." project 640 640 - else begin 641 641 - Format.eprintf "Resolve conflicts in project/%s and commit.@." project; 618 618 + in 619 619 + 620 620 + if solve then begin 621 621 + (* Solve dependencies and merge all solved packages that are vendored *) 622 622 + let pkg, project = match args with 623 623 + | [pkg; project] -> pkg, project 624 624 + | _ -> 625 625 + Format.eprintf "Usage: unpac opam merge --solve PACKAGE PROJECT@."; 642 626 exit 1 643 643 - end 644 644 - | None -> 645 645 - Format.eprintf "Error: Either provide a package name or use --all@."; 627 627 + in 628 628 + let repos = config.opam.repositories in 629 629 + if repos = [] then begin 630 630 + Format.eprintf "No repositories configured. Add one with: unpac opam repo add <name> <path>@."; 631 631 + exit 1 632 632 + end; 633 633 + let ocaml_version = match Unpac.Config.get_compiler config with 634 634 + | Some v -> v 635 635 + | None -> 636 636 + Format.eprintf "No compiler version configured.@."; 637 637 + Format.eprintf "Set one with: unpac opam config compiler 5.2.0@."; 638 638 + exit 1 639 639 + in 640 640 + let repo_paths = List.map (fun (r : Unpac.Config.repo_config) -> 641 641 + match r.source with 642 642 + | Unpac.Config.Local p -> p 643 643 + | Unpac.Config.Remote u -> u 644 644 + ) repos in 645 645 + Format.printf "Solving dependencies for %s...@." pkg; 646 646 + match Unpac_opam.Solver.solve ~repos:repo_paths ~ocaml_version ~packages:[pkg] with 647 647 + | Error msg -> 648 648 + Format.eprintf "Dependency solving failed:@.%s@." msg; 646 649 exit 1 650 650 + | Ok result -> 651 651 + (* Group by dev-repo to get canonical names *) 652 652 + let groups = group_packages_by_dev_repo ~config result.packages in 653 653 + let canonical_names = List.map (fun (g : package_group) -> g.canonical_name) groups in 654 654 + (* Filter to only vendored packages *) 655 655 + let vendored = Unpac_opam.Opam.list_packages ~proc_mgr ~root in 656 656 + let to_merge = List.filter (fun name -> List.mem name vendored) canonical_names in 657 657 + if to_merge = [] then begin 658 658 + Format.eprintf "No vendored packages match the solved dependencies.@."; 659 659 + Format.eprintf "Run 'unpac opam add %s --solve' first to vendor them.@." pkg; 660 660 + exit 1 661 661 + end; 662 662 + Format.printf "Found %d vendored packages to merge.@.@." (List.length to_merge); 663 663 + merge_packages to_merge project 664 664 + end else if all then begin 665 665 + (* Merge all vendored packages *) 666 666 + let project = match args with 667 667 + | [project] -> project 668 668 + | _ -> 669 669 + Format.eprintf "Usage: unpac opam merge --all PROJECT@."; 670 670 + exit 1 671 671 + in 672 672 + let packages = Unpac_opam.Opam.list_packages ~proc_mgr ~root in 673 673 + if packages = [] then begin 674 674 + Format.eprintf "No vendored packages to merge.@."; 675 675 + exit 1 676 676 + end; 677 677 + merge_packages packages project 678 678 + end else begin 679 679 + (* Single package mode *) 680 680 + let pkg, project = match args with 681 681 + | [pkg; project] -> pkg, project 682 682 + | _ -> 683 683 + Format.eprintf "Usage: unpac opam merge PACKAGE PROJECT@."; 684 684 + exit 1 685 685 + in 686 686 + if merge_one ~project pkg then 687 687 + Format.printf "@.Next: Build your project in project/%s@." project 688 688 + else begin 689 689 + Format.eprintf "Resolve conflicts in project/%s and commit.@." project; 690 690 + exit 1 691 691 + end 647 692 end 648 693 in 649 694 let info = Cmd.info "merge" ~doc in 650 650 - Cmd.v info Term.(const run $ logging_term $ args $ all_flag) 695 695 + Cmd.v info Term.(const run $ logging_term $ args $ all_flag $ solve_flag) 651 696 652 697 (* Opam info command *) 653 698 let opam_info_cmd = ··· 849 894 let info = Cmd.info "push" ~doc in 850 895 Cmd.v info Term.(const run $ logging_term $ remote_arg $ force_arg $ dry_run_arg) 851 896 897 897 + (* Vendor status command *) 898 898 + let vendor_status_cmd = 899 899 + let doc = "Show status of all vendored packages." in 900 900 + let run () = 901 901 + with_root @@ fun ~env:_ ~fs:_ ~proc_mgr ~root -> 902 902 + let git = Unpac.Worktree.git_dir root in 903 903 + 904 904 + (* Get all vendored packages *) 905 905 + let packages = Unpac_opam.Opam.list_packages ~proc_mgr ~root in 906 906 + if packages = [] then begin 907 907 + Format.printf "No vendored packages.@."; 908 908 + exit 0 909 909 + end; 910 910 + 911 911 + (* Get all project branches *) 912 912 + let all_branches = Unpac.Git.run_lines ~proc_mgr ~cwd:git 913 913 + ["branch"; "--format=%(refname:short)"] in 914 914 + let project_branches = List.filter (fun b -> 915 915 + String.starts_with ~prefix:"project/" b 916 916 + ) all_branches in 917 917 + let project_names = List.map (fun b -> 918 918 + String.sub b 8 (String.length b - 8) (* Remove "project/" prefix *) 919 919 + ) project_branches in 920 920 + 921 921 + (* Print header *) 922 922 + Format.printf "%-25s %8s %s@." "Package" "Patches" "Merged into"; 923 923 + Format.printf "%s@." (String.make 70 '-'); 924 924 + 925 925 + (* For each package, get patch count and merge status *) 926 926 + List.iter (fun pkg -> 927 927 + let vendor_branch = Unpac_opam.Opam.vendor_branch pkg in 928 928 + let patches_branch = Unpac_opam.Opam.patches_branch pkg in 929 929 + 930 930 + (* Count commits on patches that aren't on vendor *) 931 931 + let patch_count = 932 932 + let output = Unpac.Git.run_exn ~proc_mgr ~cwd:git 933 933 + ["rev-list"; "--count"; vendor_branch ^ ".." ^ patches_branch] in 934 934 + int_of_string (String.trim output) 935 935 + in 936 936 + 937 937 + (* Check which projects contain this package's patches *) 938 938 + let merged_into = List.filter (fun proj_name -> 939 939 + let proj_branch = "project/" ^ proj_name in 940 940 + (* Check if patches branch is an ancestor of project branch *) 941 941 + match Unpac.Git.run ~proc_mgr ~cwd:git 942 942 + ["merge-base"; "--is-ancestor"; patches_branch; proj_branch] with 943 943 + | Ok _ -> true 944 944 + | Error _ -> false 945 945 + ) project_names in 946 946 + 947 947 + let merged_str = if merged_into = [] then "-" 948 948 + else String.concat ", " merged_into in 949 949 + 950 950 + Format.printf "%-25s %8d %s@." pkg patch_count merged_str 951 951 + ) packages; 952 952 + 953 953 + Format.printf "@.Total: %d packages@." (List.length packages) 954 954 + in 955 955 + let info = Cmd.info "status" ~doc in 956 956 + Cmd.v info Term.(const run $ logging_term) 957 957 + 958 958 + (* Vendor command group *) 959 959 + let vendor_cmd = 960 960 + let doc = "Vendor status and management commands." in 961 961 + let info = Cmd.info "vendor" ~doc in 962 962 + Cmd.group info [vendor_status_cmd] 963 963 + 852 964 (* Main command *) 853 965 let main_cmd = 854 966 let doc = "Multi-backend vendoring tool using git worktrees." in 855 967 let info = Cmd.info "unpac" ~version:"0.1.0" ~doc in 856 856 - Cmd.group info [init_cmd; project_cmd; opam_cmd; push_cmd] 968 968 + Cmd.group info [init_cmd; project_cmd; opam_cmd; vendor_cmd; push_cmd] 857 969 858 970 let () = exit (Cmd.eval main_cmd)

+9 -22

lib/git.ml

··· 396 396 Log.debug (fun m -> m "Cleaning untracked files"); 397 397 run_exn ~proc_mgr ~cwd ["clean"; "-fd"] |> ignore 398 398 399 399 - let filter_branch_to_subdirectory ~proc_mgr ~cwd ~branch ~subdirectory = 399 399 + let filter_repo_to_subdirectory ~proc_mgr ~cwd ~branch ~subdirectory = 400 400 Log.info (fun m -> m "Rewriting history of %s into subdirectory %s..." branch subdirectory); 401 401 - (* Use filter-branch with index-filter to rewrite all paths into subdirectory. 402 402 - This preserves full history with paths prefixed. 401 401 + (* Use git-filter-repo with --to-subdirectory-filter to rewrite all paths into subdirectory. 402 402 + This preserves full history with paths prefixed. Much faster than filter-branch. 403 403 404 404 - For bare repositories, we need to create a temporary worktree, run filter-branch 404 404 + For bare repositories, we need to create a temporary worktree, run filter-repo 405 405 there, and then update the branch in the bare repo. *) 406 406 407 407 (* Create a unique temporary worktree name using the branch name *) ··· 422 422 (* Create worktree for the branch *) 423 423 run_exn ~proc_mgr ~cwd ["worktree"; "add"; temp_wt_relpath; branch] |> ignore; 424 424 425 425 - (* Run filter-branch in the worktree *) 426 426 - let script = Printf.sprintf 427 427 - {|git ls-files -s | sed "s,\t,&%s/," | GIT_INDEX_FILE=$GIT_INDEX_FILE.new git update-index --index-info && mv "$GIT_INDEX_FILE.new" "$GIT_INDEX_FILE"|} 428 428 - subdirectory 429 429 - in 430 430 - 431 431 - (* Set environment to suppress the warning *) 432 432 - let old_env = try Some (Unix.getenv "FILTER_BRANCH_SQUELCH_WARNING") with Not_found -> None in 433 433 - Unix.putenv "FILTER_BRANCH_SQUELCH_WARNING" "1"; 434 434 - 425 425 + (* Run git-filter-repo in the worktree *) 435 426 let result = run ~proc_mgr ~cwd:temp_wt [ 436 436 - "filter-branch"; "-f"; 437 437 - "--index-filter"; script; 438 438 - "--"; "HEAD" 427 427 + "filter-repo"; 428 428 + "--to-subdirectory-filter"; subdirectory; 429 429 + "--force"; 430 430 + "--refs"; "HEAD" 439 431 ] in 440 440 - 441 441 - (* Restore environment *) 442 442 - (match old_env with 443 443 - | Some v -> Unix.putenv "FILTER_BRANCH_SQUELCH_WARNING" v 444 444 - | None -> (try Unix.putenv "FILTER_BRANCH_SQUELCH_WARNING" "" with _ -> ())); 445 432 446 433 (* Handle result: get the new SHA, cleanup worktree, then update branch *) 447 434 (match result with

+3 -3

lib/git.mli

··· 345 345 unit 346 346 (** [clean_fd] removes untracked files and directories. *) 347 347 348 348 - val filter_branch_to_subdirectory : 348 348 + val filter_repo_to_subdirectory : 349 349 proc_mgr:proc_mgr -> 350 350 cwd:path -> 351 351 branch:string -> 352 352 subdirectory:string -> 353 353 unit 354 354 - (** [filter_branch_to_subdirectory ~proc_mgr ~cwd ~branch ~subdirectory] 354 354 + (** [filter_repo_to_subdirectory ~proc_mgr ~cwd ~branch ~subdirectory] 355 355 rewrites the history of [branch] so all files are moved into [subdirectory]. 356 356 - This preserves full commit history with paths prefixed. *) 356 356 + Uses git-filter-repo for fast history rewriting. Preserves full commit history. *)

+12 -40

lib/opam/opam.ml

··· 66 66 end 67 67 ) 68 68 69 69 - let add_package ?(preserve_history=true) ~proc_mgr ~root ?cache (info : Backend.package_info) = 69 69 + let add_package ~proc_mgr ~root ?cache (info : Backend.package_info) = 70 70 let pkg = info.name in 71 71 let git = Worktree.git_dir root in 72 72 ··· 105 105 Git.branch_force ~proc_mgr ~cwd:git 106 106 ~name:(upstream_branch pkg) ~point:ref_point; 107 107 108 108 - let vendor_sha = 109 109 - if preserve_history then begin 110 110 - (* Step 2a: Create vendor branch from upstream (preserves history) *) 111 111 - Git.branch_force ~proc_mgr ~cwd:git 112 112 - ~name:(vendor_branch pkg) ~point:(upstream_branch pkg); 113 113 - 114 114 - (* Rewrite vendor branch history to move all files into vendor/opam/<pkg>/ *) 115 115 - Git.filter_branch_to_subdirectory ~proc_mgr ~cwd:git 116 116 - ~branch:(vendor_branch pkg) 117 117 - ~subdirectory:(vendor_path pkg); 118 118 - 119 119 - (* Get the vendor SHA after rewriting *) 120 120 - match Git.rev_parse ~proc_mgr ~cwd:git (vendor_branch pkg) with 121 121 - | Some sha -> sha 122 122 - | None -> failwith "Vendor branch not found after filter-branch" 123 123 - end else begin 124 124 - (* Step 2b: Fast mode - create orphan vendor branch without history *) 125 125 - Worktree.ensure ~proc_mgr root (upstream_kind pkg); 126 126 - let upstream_wt = Worktree.path root (upstream_kind pkg) in 127 127 - 128 128 - Worktree.ensure_orphan ~proc_mgr root (vendor_kind pkg); 129 129 - let vendor_wt = Worktree.path root (vendor_kind pkg) in 130 130 - 131 131 - (* Copy files with vendor/opam/<pkg>/ prefix *) 132 132 - copy_with_prefix 133 133 - ~src_dir:upstream_wt 134 134 - ~dst_dir:vendor_wt 135 135 - ~prefix:(vendor_path pkg); 108 108 + (* Step 2: Create vendor branch from upstream and rewrite history *) 109 109 + Git.branch_force ~proc_mgr ~cwd:git 110 110 + ~name:(vendor_branch pkg) ~point:(upstream_branch pkg); 136 111 137 137 - (* Commit vendor branch *) 138 138 - Git.add_all ~proc_mgr ~cwd:vendor_wt; 139 139 - Git.commit ~proc_mgr ~cwd:vendor_wt 140 140 - ~message:(Printf.sprintf "Vendor %s" pkg); 141 141 - 142 142 - let sha = Git.current_head ~proc_mgr ~cwd:vendor_wt in 112 112 + (* Rewrite vendor branch history to move all files into vendor/opam/<pkg>/ *) 113 113 + Git.filter_repo_to_subdirectory ~proc_mgr ~cwd:git 114 114 + ~branch:(vendor_branch pkg) 115 115 + ~subdirectory:(vendor_path pkg); 143 116 144 144 - (* Cleanup worktrees *) 145 145 - Worktree.remove ~proc_mgr root (upstream_kind pkg); 146 146 - Worktree.remove ~proc_mgr root (vendor_kind pkg); 147 147 - sha 148 148 - end 117 117 + (* Get the vendor SHA after rewriting *) 118 118 + let vendor_sha = match Git.rev_parse ~proc_mgr ~cwd:git (vendor_branch pkg) with 119 119 + | Some sha -> sha 120 120 + | None -> failwith "Vendor branch not found after filter-repo" 149 121 in 150 122 151 123 (* Step 3: Create patches branch from vendor *)

+2 -5

lib/opam/opam.mli

··· 31 31 (** {1 Package Operations} *) 32 32 33 33 val add_package : 34 34 - ?preserve_history:bool -> 35 34 proc_mgr:Unpac.Git.proc_mgr -> 36 35 root:Unpac.Worktree.root -> 37 36 ?cache:Unpac.Vendor_cache.t -> ··· 40 39 (** [add_package ~proc_mgr ~root ?cache info] vendors a single package. 41 40 42 41 1. Fetches upstream into opam/upstream/<pkg> (via cache if provided) 43 43 - 2. Creates opam/vendor/<pkg> with vendor/opam/<pkg>/ prefix 42 42 + 2. Creates opam/vendor/<pkg> with vendor/opam/<pkg>/ prefix (preserving history) 44 43 3. Creates opam/patches/<pkg> from vendor 45 44 46 46 - @param preserve_history If true (default), rewrites git history to preserve 47 47 - full commit history in the vendor branch. Set to false for faster 48 48 - vendoring without history (useful for large repositories). 45 45 + Uses git-filter-repo for fast history rewriting. 49 46 @param cache Optional vendor cache for shared fetches across projects. *) 50 47 51 48 val update_package :