jon.recoil.org
A fork of mtelver's day10 project
1# Makefile for running health-checks on all available OPAM packages in parallel
2# Usage: make -j<N> all (where N is the number of parallel jobs)
3# make OUTPUT_DIR=/path/to/output all (to specify custom output directory)
4# make OPAM_REPO=/path/to/packages all (to specify custom opam repository)
5# make clean (to remove markdown files)
6
7# OS target
8SYSTEM := debian-12
9
10# Compiler versions - can be overridden on command line
11#COMPILERS := ocaml.4.08.1 ocaml.4.09.1 ocaml.4.10.2 ocaml.4.11.2 ocaml.4.12.1 ocaml.4.13.1 ocaml.4.14.2 ocaml.5.0.0 ocaml.5.1.1 ocaml.5.2.1 ocaml.5.3.0
12#COMPILERS := ocaml.4.08.2 ocaml.4.09.2 ocaml.4.10.3 ocaml.4.11.3 ocaml.4.12.2 ocaml.4.13.2 ocaml.4.14.3 ocaml.5.0.1 ocaml.5.1.2 ocaml.5.2.2 ocaml.5.3.1
13COMPILERS := ocaml-base-compiler.5.4.0~beta2
14
15# Output directory - can be overridden on command line: make OUTPUT_DIR=/path/to/output
16#OUTPUT_DIR := output
17#OUTPUT_DIR := relocatable
18OUTPUT_DIR := output
19
20# Path to the opam repository root (for git operations) - can be overridden
21OPAM_REPO := /home/mtelvers/opam-repository
22
23# Output directory - can be overridden on command line: make OUTPUT_DIR=/path/to/output
24CACHE_DIR := /home/mtelvers/cache2
25
26# Get the git commit SHA of the opam repository
27OPAM_SHA := $(shell git -C "$(OPAM_REPO)" rev-parse HEAD 2>/dev/null || echo "unknown")
28
29# Get the list of packages from opam
30PACKAGES := $(shell ./_build/install/default/bin/day10 list --opam-repository "$(OPAM_REPO)")
31# PACKAGES := 0install.2.18 diffast-api.0.2 alcotest.1.9.0 bos.0.2.1 ansi.0.7.0
32
33# --opam-repository /home/mtelvers/opam-repository-relocatable \
34
35# Template to generate rules for each compiler version
36define COMPILER_TEMPLATE
37$$(OUTPUT_DIR)/$$(OPAM_SHA)/$$(SYSTEM)/$(1)/%.json: | $$(CACHE_DIR)
38 @mkdir -p $$(OUTPUT_DIR)/$$(OPAM_SHA)/$$(SYSTEM)/$(1)
39 ./_build/install/default/bin/day10 health-check \
40 --cache-dir "$$(CACHE_DIR)" \
41 --opam-repository "$$(OPAM_REPO)" \
42 --ocaml-version $(1) \
43 --json $$@ $$(basename $$(notdir $$@))
44endef
45
46# Generate pattern rules for each compiler
47$(foreach compiler,$(COMPILERS),$(eval $(call COMPILER_TEMPLATE,$(compiler))))
48
49# Generate all targets for all compiler/package combinations
50# Order by package first, then compiler (better resource distribution)
51TARGETS := $(foreach package,$(PACKAGES),$(foreach compiler,$(COMPILERS),$(OUTPUT_DIR)/$(OPAM_SHA)/$(SYSTEM)/$(compiler)/$(package).json))
52
53# Default target - depends on all package health-checks for all compilers
54all: $(TARGETS)
55
56$(CACHE_DIR):
57 mkdir -p $(CACHE_DIR)
58
59$(OUTPUT_DIR)/commits.json:
60 @echo "[]" > $@.tmp
61 @for dir in $(OUTPUT_DIR)/*/; do \
62 if [ -d "$$dir" ]; then \
63 sha=$$(basename "$$dir"); \
64 echo "Processing SHA: $$sha"; \
65 git -C $(OPAM_REPO) show --pretty=format:'%H%x00%aI%x00%s%x00' -s "$$sha" 2>/dev/null | \
66 jq -R -s 'if . == "" then empty else split("\n")[0] | split("\u0000") | {"sha": .[0], "date": .[1], "message": .[2]} end' | \
67 jq -s 'if length > 0 then .[0] else {"sha": "'$$sha'", "date": null, "message": "Unknown commit"} end' > $@.entry && \
68 jq --slurpfile entry $@.entry '. += $$entry' $@.tmp > $@.tmp2 && \
69 mv $@.tmp2 $@.tmp; \
70 rm -f $@.entry; \
71 fi; \
72 done
73 @mv $@.tmp $@
74 @echo "JSON file generated: $@"
75
76$(OUTPUT_DIR)/%/commit.json:
77 @echo "Generating flattened $@"
78 @{ \
79 sha=$$(basename $(@D)); \
80 for os_dir in $(@D)/*/; do \
81 if [ -d "$$os_dir" ]; then \
82 os=$$(basename "$$os_dir"); \
83 for compiler_dir in "$$os_dir"*/; do \
84 if [ -d "$$compiler_dir" ]; then \
85 compiler=$$(basename "$$compiler_dir"); \
86 json_files="$$compiler_dir"*.json; \
87 if ls $$json_files >/dev/null 2>&1; then \
88 cat $$json_files | jq --arg os "$$os" --arg compiler "$$compiler" --arg sha "$$sha" \
89 '. + {"os": $$os, "compiler": $$compiler, "sha": $$sha}'; \
90 fi; \
91 fi; \
92 done; \
93 fi; \
94 done; \
95 } | jq -s '.' > $@
96
97json: $(OUTPUT_DIR)/commits.json $(foreach dir,$(wildcard output/*),$(dir)/commit.json)
98
99$(OUTPUT_DIR)/%/commit.parquet: $(OUTPUT_DIR)/%/commit.json
100 @echo "Converting $< to Parquet format"
101 clickhouse local --query "SELECT * FROM file('$<', 'JSONEachRow') INTO OUTFILE '$@' FORMAT Parquet"
102
103$(OUTPUT_DIR)/%/commit-with-logs.json:
104 @echo "Generating flattened $@ with build logs using Python"
105 python3 process_with_logs.py $(@D) --cache-dir $(CACHE_DIR) --output-json $@
106
107$(OUTPUT_DIR)/%/commit-with-logs.parquet: $(OUTPUT_DIR)/%/commit-with-logs.json
108 @echo "Converting $< to Parquet format"
109 clickhouse local --query "SELECT * FROM file('$<', 'JSONEachRow') INTO OUTFILE '$@' FORMAT Parquet"
110
111# Combined target to generate both JSON and Parquet with build logs
112$(OUTPUT_DIR)/%/commit-with-logs: $(OUTPUT_DIR)/%/commit-with-logs.json $(OUTPUT_DIR)/%/commit-with-logs.parquet
113 @echo "Generated both JSON and Parquet files with build logs for $(@D)"
114
115copy:
116 @find $(CACHE_DIR) -maxdepth 2 \( -name "layer.json" -o -name "build.log" \) -print0 | \
117 xargs -0 -P $(shell nproc) -I {} sh -c 'd=$${1%/*}; d=$${d##*/}; mkdir -p $(OUTPUT_DIR)/cache/$$d; cp -l "$$1" $(OUTPUT_DIR)/cache/$$d/' _ {}
118
119# Clean up json files for all compilers
120clean:
121 rm -rf $(foreach compiler,$(COMPILERS),$(OUTPUT_DIR)/$(OPAM_SHA)/$(SYSTEM)/$(compiler))
122
123# Show the list of packages that will be processed for each compiler
124list:
125 @echo "Packages to process (from $(OPAM_REPO)/packages):"
126 @$(foreach compiler,$(COMPILERS),echo "Compiler $(compiler): $(OUTPUT_DIR)/$(OPAM_SHA)/$(SYSTEM)/$(compiler)";)
127 @echo "Packages:"
128 @echo $(PACKAGES) | tr ' ' '\n'
129
130# Count total packages across all compilers
131count:
132 @echo "Total packages per compiler: $(words $(PACKAGES))"
133 @echo "Total compilers: $(words $(COMPILERS))"
134 @echo "Total targets: $(words $(TARGETS))"
135
136# Targets for building with specific compilers
137$(foreach compiler,$(COMPILERS),$(eval $(compiler): $(addprefix $(OUTPUT_DIR)/$(OPAM_SHA)/$(SYSTEM)/$(compiler)/, $(addsuffix .json, $(PACKAGES)))))
138
139next:
140 git -C $(OPAM_REPO) fetch --all
141 next_merge=$$(git -C $(OPAM_REPO) log --merges --format="%H" --reverse HEAD..upstream/master | head -1); \
142 if [ -z "$$next_merge" ]; then \
143 echo "No merge commits found ahead of current position in upstream/master"; \
144 exit 1; \
145 fi; \
146 echo "Moving to next merge commit: $$next_merge"; \
147 git -C $(OPAM_REPO) log --oneline -1 $$next_merge; \
148 git -C $(OPAM_REPO) checkout $$next_merge
149
150parquet: $(foreach dir,$(wildcard $(OUTPUT_DIR)/*),$(dir)/commit.parquet)
151
152.PHONY: all clean list count parquet $(COMPILERS)