anil.recoil.org
forked from
mtelvers.tunbury.org/repo-tool
A CLI tool that generates an opam repository and monorepo from a list of git repos
Initial commit
+133
README.md
+133
README.md
···
1
+
# repo-tool
2
+
3
+
A CLI tool that generates an opam repository and monorepo from a list of git repositories.
4
+
5
+
## Overview
6
+
7
+
`repo-tool` reads a text file containing git repository URLs and:
8
+
9
+
1. Clones each repository into a `vendor/` directory
10
+
2. Generates an opam repository structure in `opam-repository/`
11
+
3. Creates a `setup.sh` script to pin packages and install dependencies
12
+
4. Sets up dune to build everything as a monorepo
13
+
14
+
This is useful for creating a local development environment with multiple interdependent packages that may not yet be published to opam.
15
+
16
+
## Installation
17
+
18
+
```bash
19
+
opam install . --deps-only
20
+
dune build
21
+
dune install
22
+
```
23
+
24
+
Or run directly:
25
+
26
+
```bash
27
+
dune exec repo-tool -- <args>
28
+
```
29
+
30
+
## Usage
31
+
32
+
```bash
33
+
repo-tool INPUT_FILE [-o OUTPUT_DIR] [-v]
34
+
```
35
+
36
+
### Arguments
37
+
38
+
- `INPUT_FILE` - Path to a text file containing git repository URLs (one per line)
39
+
- `-o, --output DIR` - Output directory (default: `opam-repository`)
40
+
- `-v, --verbose` - Enable verbose output
41
+
42
+
### Input File Format
43
+
44
+
```
45
+
# Comments start with #
46
+
https://github.com/user/repo1.git
47
+
https://github.com/user/repo2.git main
48
+
https://tangled.org/user/repo3
49
+
```
50
+
51
+
Each line contains a git URL, optionally followed by a branch name.
52
+
53
+
## Output Structure
54
+
55
+
```
56
+
output-dir/
57
+
├── dune-project # Dune project file
58
+
├── dune # Top-level dune config
59
+
├── setup.sh # Setup script for opam switch
60
+
├── opam-repository/
61
+
│ ├── repo # opam repository metadata
62
+
│ └── packages/
63
+
│ └── <pkg>/
64
+
│ └── <pkg>.dev/
65
+
│ └── opam # Package opam file with url stanza
66
+
└── vendor/
67
+
├── dune # Lists vendor subdirectories
68
+
├── repo1/ # Cloned source code
69
+
├── repo2/
70
+
└── ...
71
+
```
72
+
73
+
## Setting Up the Monorepo
74
+
75
+
After running `repo-tool`, set up the development environment:
76
+
77
+
```bash
78
+
cd output-dir
79
+
./setup.sh
80
+
```
81
+
82
+
The setup script will:
83
+
1. Create a local opam switch with OCaml 5.4.0
84
+
2. Pin all vendor packages
85
+
3. Install dependencies (including test dependencies)
86
+
4. Run `dune build`
87
+
88
+
Alternatively, run the steps manually:
89
+
90
+
```bash
91
+
cd output-dir
92
+
opam switch create . 5.4.0 -y
93
+
opam pin add -ny <pkg1> vendor/<repo1>
94
+
opam pin add -ny <pkg2> vendor/<repo2>
95
+
# ... for each package
96
+
opam install -y --deps-only --with-test <pkg1> <pkg2> ...
97
+
opam exec -- dune build --root .
98
+
```
99
+
100
+
## Using the opam Repository as an Overlay
101
+
102
+
You can also use just the generated opam repository as an overlay:
103
+
104
+
```bash
105
+
opam repository add local /path/to/output-dir/opam-repository
106
+
opam update
107
+
opam install <package-name>
108
+
```
109
+
110
+
## Incremental Updates
111
+
112
+
Running `repo-tool` again on an existing output directory will:
113
+
- Update existing repositories with `git pull`
114
+
- Clone any new repositories
115
+
- Regenerate the opam repository and setup script
116
+
117
+
## Example
118
+
119
+
```bash
120
+
# Create a repos.txt file
121
+
cat > repos.txt << EOF
122
+
https://github.com/user/ocaml-foo
123
+
https://github.com/user/ocaml-bar
124
+
https://tangled.org/user/ocaml-baz
125
+
EOF
126
+
127
+
# Generate the monorepo
128
+
repo-tool repos.txt -o my-monorepo -v
129
+
130
+
# Set up and build
131
+
cd my-monorepo
132
+
./setup.sh
133
+
```
+4
bin/dune
+4
bin/dune
+45
bin/main.ml
+45
bin/main.ml
···
1
+
open Cmdliner
2
+
3
+
let input_file =
4
+
let doc = "Path to the input file containing git repository URLs (one per line)." in
5
+
Arg.(required & pos 0 (some file) None & info [] ~docv:"INPUT_FILE" ~doc)
6
+
7
+
let output_dir =
8
+
let doc = "Output directory for the generated opam repository." in
9
+
Arg.(value & opt string "opam-repository" & info [ "o"; "output" ] ~docv:"DIR" ~doc)
10
+
11
+
let verbose =
12
+
let doc = "Enable verbose output." in
13
+
Arg.(value & flag & info [ "v"; "verbose" ] ~doc)
14
+
15
+
let run input_file output_dir verbose =
16
+
let exit_code = Repo_tool.run ~input_file ~output_dir ~verbose in
17
+
exit exit_code
18
+
19
+
let run_t = Term.(const run $ input_file $ output_dir $ verbose)
20
+
21
+
let cmd =
22
+
let doc = "Generate an opam repository from git repositories" in
23
+
let man =
24
+
[
25
+
`S Manpage.s_description;
26
+
`P
27
+
"$(tname) reads a text file containing a list of git repository URLs \
28
+
and generates an opam repository structure. Each line in the input \
29
+
file should contain a git URL, optionally followed by a branch name.";
30
+
`S Manpage.s_examples;
31
+
`P "Create an opam repository from repos.txt:";
32
+
`Pre " $(tname) repos.txt -o my-opam-repo";
33
+
`P "Input file format:";
34
+
`Pre
35
+
" https://github.com/user/repo1.git\n\
36
+
\ https://github.com/user/repo2.git main\n\
37
+
\ # This is a comment";
38
+
`S Manpage.s_bugs;
39
+
`P "Report bugs at https://github.com/mtelvers/repo-tool/issues";
40
+
]
41
+
in
42
+
let info = Cmd.info "repo-tool" ~version:"0.1.0" ~doc ~man in
43
+
Cmd.v info run_t
44
+
45
+
let () = exit (Cmd.eval cmd)
+1
dune
+1
dune
···
1
+
(dirs :standard \ monorepo)
+24
dune-project
+24
dune-project
···
1
+
(lang dune 3.0)
2
+
3
+
(name repo_tool)
4
+
5
+
(generate_opam_files true)
6
+
7
+
(source
8
+
(github mtelvers/repo-tool))
9
+
10
+
(authors "Mark Elvers")
11
+
12
+
(maintainers "Mark Elvers")
13
+
14
+
(license MIT)
15
+
16
+
(package
17
+
(name repo_tool)
18
+
(synopsis "Generate opam repository from git repositories")
19
+
(description
20
+
"A CLI tool that reads a text file containing git repository URLs and generates an opam repository structure that can be overlaid on the official opam repository.")
21
+
(depends
22
+
(ocaml (>= 4.14))
23
+
dune
24
+
cmdliner))
+285
lib/repo_tool.ml
+285
lib/repo_tool.ml
···
1
+
type repo_entry = {
2
+
url : string;
3
+
branch : string option;
4
+
}
5
+
6
+
let verbose = ref false
7
+
8
+
let log fmt =
9
+
if !verbose then Printf.eprintf (fmt ^^ "\n%!")
10
+
else Printf.ifprintf stderr fmt
11
+
12
+
let parse_repo_line line =
13
+
let line = String.trim line in
14
+
if String.length line = 0 || line.[0] = '#' then None
15
+
else
16
+
match String.split_on_char ' ' line with
17
+
| [] -> None
18
+
| [ url ] -> Some { url; branch = None }
19
+
| url :: branch :: _ -> Some { url; branch = Some branch }
20
+
21
+
let read_lines filename =
22
+
let ic = open_in filename in
23
+
let rec loop acc =
24
+
match input_line ic with
25
+
| line -> loop (line :: acc)
26
+
| exception End_of_file ->
27
+
close_in ic;
28
+
List.rev acc
29
+
in
30
+
loop []
31
+
32
+
let read_repo_file path =
33
+
let lines = read_lines path in
34
+
List.filter_map parse_repo_line lines
35
+
36
+
let extract_repo_name url =
37
+
let url = String.trim url in
38
+
let url =
39
+
if String.ends_with ~suffix:".git" url then
40
+
String.sub url 0 (String.length url - 4)
41
+
else url
42
+
in
43
+
(* Remove trailing slash if present *)
44
+
let url =
45
+
if String.ends_with ~suffix:"/" url then
46
+
String.sub url 0 (String.length url - 1)
47
+
else url
48
+
in
49
+
match String.split_on_char '/' url |> List.rev with
50
+
| name :: _ -> name
51
+
| [] -> "unknown"
52
+
53
+
let run_command cmd =
54
+
let exit_code = Sys.command cmd in
55
+
if exit_code = 0 then Ok ()
56
+
else Error (Printf.sprintf "Command failed with exit code %d: %s" exit_code cmd)
57
+
58
+
let rec mkdir_p path =
59
+
if Sys.file_exists path then ()
60
+
else begin
61
+
mkdir_p (Filename.dirname path);
62
+
Unix.mkdir path 0o755
63
+
end
64
+
65
+
let clone_or_update_repo ~vendor_dir entry =
66
+
let name = extract_repo_name entry.url in
67
+
let target = Filename.concat vendor_dir name in
68
+
if Sys.file_exists target then begin
69
+
log "Updating %s in %s" entry.url target;
70
+
let cmd = Printf.sprintf "git -C %s pull --ff-only 2>/dev/null" target in
71
+
match run_command cmd with
72
+
| Ok () -> Ok target
73
+
| Error _ ->
74
+
(* If pull fails, try a fresh clone *)
75
+
log "Pull failed, re-cloning %s" entry.url;
76
+
let rm_cmd = Printf.sprintf "rm -rf %s" target in
77
+
ignore (run_command rm_cmd);
78
+
let branch_args =
79
+
match entry.branch with
80
+
| Some b -> Printf.sprintf "--branch %s" b
81
+
| None -> ""
82
+
in
83
+
let clone_cmd =
84
+
Printf.sprintf "git clone --depth 1 %s %s %s 2>/dev/null"
85
+
branch_args entry.url target
86
+
in
87
+
match run_command clone_cmd with
88
+
| Ok () -> Ok target
89
+
| Error msg ->
90
+
Printf.eprintf "Failed to clone %s: %s\n%!" entry.url msg;
91
+
Error msg
92
+
end
93
+
else begin
94
+
log "Cloning %s to %s" entry.url target;
95
+
let branch_args =
96
+
match entry.branch with
97
+
| Some b -> Printf.sprintf "--branch %s" b
98
+
| None -> ""
99
+
in
100
+
let cmd =
101
+
Printf.sprintf "git clone --depth 1 %s %s %s 2>/dev/null"
102
+
branch_args entry.url target
103
+
in
104
+
match run_command cmd with
105
+
| Ok () -> Ok target
106
+
| Error msg ->
107
+
Printf.eprintf "Failed to clone %s: %s\n%!" entry.url msg;
108
+
Error msg
109
+
end
110
+
111
+
let find_opam_files repo_path =
112
+
let files = Sys.readdir repo_path in
113
+
Array.to_list files
114
+
|> List.filter (fun f -> String.ends_with ~suffix:".opam" f)
115
+
|> List.map (fun f -> Filename.concat repo_path f)
116
+
117
+
let read_file path =
118
+
let ic = open_in path in
119
+
let n = in_channel_length ic in
120
+
let content = really_input_string ic n in
121
+
close_in ic;
122
+
content
123
+
124
+
let write_file path content =
125
+
let oc = open_out path in
126
+
output_string oc content;
127
+
close_out oc
128
+
129
+
let extract_version_from_opam content =
130
+
let lines = String.split_on_char '\n' content in
131
+
let version_line =
132
+
List.find_opt
133
+
(fun line ->
134
+
let trimmed = String.trim line in
135
+
String.length trimmed > 8 && String.sub trimmed 0 8 = "version:")
136
+
lines
137
+
in
138
+
match version_line with
139
+
| Some line -> (
140
+
let parts = String.split_on_char '"' line in
141
+
match parts with _ :: ver :: _ -> Some ver | _ -> None)
142
+
| None -> None
143
+
144
+
let package_name_from_opam_path path =
145
+
let basename = Filename.basename path in
146
+
if String.ends_with ~suffix:".opam" basename then
147
+
String.sub basename 0 (String.length basename - 5)
148
+
else basename
149
+
150
+
let generate_repo_structure ~output_dir ~repo_path ~git_url =
151
+
let opam_files = find_opam_files repo_path in
152
+
let packages_dir = Filename.concat output_dir "packages" in
153
+
mkdir_p packages_dir;
154
+
List.iter
155
+
(fun opam_path ->
156
+
let pkg_name = package_name_from_opam_path opam_path in
157
+
let content = read_file opam_path in
158
+
let version =
159
+
match extract_version_from_opam content with
160
+
| Some v -> v
161
+
| None -> "dev"
162
+
in
163
+
let pkg_dir =
164
+
Filename.concat packages_dir
165
+
(Filename.concat pkg_name (pkg_name ^ "." ^ version))
166
+
in
167
+
log "Creating package %s.%s" pkg_name version;
168
+
mkdir_p pkg_dir;
169
+
let opam_target = Filename.concat pkg_dir "opam" in
170
+
let url_section =
171
+
Printf.sprintf "\nurl {\n src: \"%s\"\n dev-repo: \"git+%s\"\n}\n"
172
+
git_url git_url
173
+
in
174
+
let lines = String.split_on_char '\n' content in
175
+
let filtered =
176
+
List.filter
177
+
(fun line ->
178
+
let trimmed = String.trim line in
179
+
not
180
+
(String.length trimmed > 8
181
+
&& String.sub trimmed 0 8 = "version:"))
182
+
lines
183
+
in
184
+
let final_content = String.concat "\n" filtered ^ url_section in
185
+
write_file opam_target final_content)
186
+
opam_files
187
+
188
+
let create_repo_file output_dir =
189
+
let repo_path = Filename.concat output_dir "repo" in
190
+
write_file repo_path "opam-version: \"2.0\"\n"
191
+
192
+
let collect_all_packages vendor_dirs =
193
+
List.concat_map
194
+
(fun vendor_path ->
195
+
let opam_files = find_opam_files vendor_path in
196
+
List.map
197
+
(fun opam_path ->
198
+
let pkg_name = package_name_from_opam_path opam_path in
199
+
(pkg_name, vendor_path))
200
+
opam_files)
201
+
vendor_dirs
202
+
203
+
let create_setup_script output_dir packages =
204
+
let script_path = Filename.concat output_dir "setup.sh" in
205
+
let buf = Buffer.create 1024 in
206
+
Buffer.add_string buf "#!/bin/sh\n";
207
+
Buffer.add_string buf "# Auto-generated setup script for monorepo\n\n";
208
+
Buffer.add_string buf "set -e\n\n";
209
+
Buffer.add_string buf "echo \"Creating opam switch...\"\n";
210
+
Buffer.add_string buf "opam switch create . 5.4.0 -y\n\n";
211
+
Buffer.add_string buf "echo \"Pinning vendor packages...\"\n";
212
+
List.iter
213
+
(fun (pkg_name, vendor_path) ->
214
+
let rel_path = "vendor/" ^ Filename.basename vendor_path in
215
+
Buffer.add_string buf
216
+
(Printf.sprintf "opam pin add -ny %s %s\n" pkg_name rel_path))
217
+
packages;
218
+
Buffer.add_string buf "\necho \"Installing dependencies...\"\n";
219
+
let pkg_names = List.map fst packages |> String.concat " " in
220
+
Buffer.add_string buf (Printf.sprintf "opam install -y --deps-only --with-test %s\n" pkg_names);
221
+
Buffer.add_string buf "\necho \"Building...\"\n";
222
+
Buffer.add_string buf "opam exec -- dune build --root .\n";
223
+
Buffer.add_string buf "\necho \"Done!\"\n";
224
+
write_file script_path (Buffer.contents buf);
225
+
Unix.chmod script_path 0o755
226
+
227
+
let create_dune_project output_dir vendor_dirs =
228
+
let dune_project_path = Filename.concat output_dir "dune-project" in
229
+
let content = "(lang dune 3.0)\n" in
230
+
write_file dune_project_path content;
231
+
(* Create top-level dune file *)
232
+
let dune_path = Filename.concat output_dir "dune" in
233
+
let dune_content = "(dirs vendor opam-repository)\n" in
234
+
write_file dune_path dune_content;
235
+
(* Create vendor dune file that lists subdirs *)
236
+
let vendor_dir = Filename.concat output_dir "vendor" in
237
+
let vendor_dune_path = Filename.concat vendor_dir "dune" in
238
+
let subdirs =
239
+
List.map (fun d -> Filename.basename d) vendor_dirs
240
+
|> String.concat " "
241
+
in
242
+
let vendor_dune_content = Printf.sprintf "(dirs %s)\n" subdirs in
243
+
write_file vendor_dune_path vendor_dune_content;
244
+
(* Create setup script *)
245
+
let packages = collect_all_packages vendor_dirs in
246
+
create_setup_script output_dir packages
247
+
248
+
let run ~input_file ~output_dir ~verbose:v =
249
+
verbose := v;
250
+
Printf.printf "Reading repository list from %s\n%!" input_file;
251
+
let entries = read_repo_file input_file in
252
+
Printf.printf "Found %d repositories to process\n%!" (List.length entries);
253
+
mkdir_p output_dir;
254
+
let opam_repo_dir = Filename.concat output_dir "opam-repository" in
255
+
mkdir_p opam_repo_dir;
256
+
create_repo_file opam_repo_dir;
257
+
let vendor_dir = Filename.concat output_dir "vendor" in
258
+
mkdir_p vendor_dir;
259
+
log "Using vendor directory %s" vendor_dir;
260
+
let results =
261
+
List.map
262
+
(fun entry ->
263
+
match clone_or_update_repo ~vendor_dir entry with
264
+
| Ok repo_path ->
265
+
generate_repo_structure ~output_dir:opam_repo_dir ~repo_path ~git_url:entry.url;
266
+
Ok repo_path
267
+
| Error msg -> Error msg)
268
+
entries
269
+
in
270
+
let vendor_dirs =
271
+
List.filter_map (function Ok p -> Some p | Error _ -> None) results
272
+
in
273
+
let errors =
274
+
List.filter_map (function Error e -> Some e | Ok _ -> None) results
275
+
in
276
+
create_dune_project output_dir vendor_dirs;
277
+
if List.length errors > 0 then begin
278
+
Printf.eprintf "Completed with %d errors:\n%!" (List.length errors);
279
+
List.iter (fun e -> Printf.eprintf " - %s\n%!" e) errors
280
+
end;
281
+
Printf.printf "Output written to %s\n%!" output_dir;
282
+
Printf.printf " opam-repository/ - opam package definitions\n%!";
283
+
Printf.printf " vendor/ - source code\n%!";
284
+
Printf.printf " setup.sh - run to pin packages and install deps\n%!";
285
+
if List.length errors > 0 then 1 else 0
+31
repo_tool.opam
+31
repo_tool.opam
···
1
+
# This file is generated by dune, edit dune-project instead
2
+
opam-version: "2.0"
3
+
synopsis: "Generate opam repository from git repositories"
4
+
description:
5
+
"A CLI tool that reads a text file containing git repository URLs and generates an opam repository structure that can be overlaid on the official opam repository."
6
+
maintainer: ["Mark Elvers"]
7
+
authors: ["Mark Elvers"]
8
+
license: "MIT"
9
+
homepage: "https://github.com/mtelvers/repo-tool"
10
+
bug-reports: "https://github.com/mtelvers/repo-tool/issues"
11
+
depends: [
12
+
"ocaml" {>= "4.14"}
13
+
"dune" {>= "3.0"}
14
+
"cmdliner"
15
+
"odoc" {with-doc}
16
+
]
17
+
build: [
18
+
["dune" "subst"] {dev}
19
+
[
20
+
"dune"
21
+
"build"
22
+
"-p"
23
+
name
24
+
"-j"
25
+
jobs
26
+
"@install"
27
+
"@runtest" {with-test}
28
+
"@doc" {with-doc}
29
+
]
30
+
]
31
+
dev-repo: "git+https://github.com/mtelvers/repo-tool.git"
+17
repos.txt
+17
repos.txt
···
1
+
https://tangled.org/anil.recoil.org/ocaml-crockford
2
+
https://tangled.org/anil.recoil.org/ocaml-jsonfeed
3
+
https://tangled.org/anil.recoil.org/xdge
4
+
https://tangled.org/anil.recoil.org/ocaml-claudeio
5
+
https://tangled.org/anil.recoil.org/ocaml-bytesrw-eio
6
+
https://tangled.org/anil.recoil.org/ocaml-yamlrw
7
+
https://tangled.org/anil.recoil.org/ocaml-yamlt
8
+
https://tangled.org/anil.recoil.org/sortal
9
+
https://tangled.org/anil.recoil.org/sortal-term
10
+
https://tangled.org/anil.recoil.org/ocaml-punycode
11
+
https://tangled.org/anil.recoil.org/ocaml-conpool
12
+
https://tangled.org/anil.recoil.org/ocaml-requests
13
+
https://tangled.org/anil.recoil.org/ocaml-karakeep
14
+
https://tangled.org/anil.recoil.org/ocaml-html5rw
15
+
https://github.com/tmattio/mosaic
16
+
https://tangled.org/anil.recoil.org/ocaml-cookeio
17
+
https://tangled.org/anil.recoil.org/ocaml-publicsuffix