anil.recoil.org
My aggregated monorepo of OCaml code, automaintained
1(*---------------------------------------------------------------------------
2 Copyright (c) 2026 Anil Madhavapeddy <anil@recoil.org>. All rights reserved.
3 SPDX-License-Identifier: ISC
4 ---------------------------------------------------------------------------*)
5
6(** Daily changes with per-day-per-repo structure.
7
8 This module provides an immutable data structure for loading and querying
9 daily changes from per-day-per-repo JSON files. Files are named
10 [<repo>-<YYYY-MM-DD>.json] and contain timestamped entries for real-time
11 tracking. *)
12
13type commit_range = {
14 from_hash : string;
15 to_hash : string;
16 count : int;
17}
18
19type entry = {
20 repository : string;
21 hour : int;
22 timestamp : Ptime.t;
23 summary : string;
24 changes : string list;
25 commit_range : commit_range;
26 contributors : string list;
27 repo_url : string option;
28}
29
30type day = {
31 repository : string;
32 date : string;
33 entries : entry list;
34}
35
36module String_map = Map.Make(String)
37
38type t = {
39 by_repo : day list String_map.t;
40 by_date : day list String_map.t;
41 all_entries : entry list;
42}
43
44(* JSON codecs for the per-day file format *)
45
46let commit_range_jsont =
47 let make from_hash to_hash count = { from_hash; to_hash; count } in
48 Jsont.Object.map ~kind:"commit_range" make
49 |> Jsont.Object.mem "from" Jsont.string ~enc:(fun r -> r.from_hash)
50 |> Jsont.Object.mem "to" Jsont.string ~enc:(fun r -> r.to_hash)
51 |> Jsont.Object.mem "count" Jsont.int ~enc:(fun r -> r.count)
52 |> Jsont.Object.finish
53
54let ptime_jsont =
55 let enc t = Ptime.to_rfc3339 t ~tz_offset_s:0 in
56 let dec s =
57 match Ptime.of_rfc3339 s with
58 | Ok (t, _, _) -> t
59 | Error _ -> failwith ("Invalid timestamp: " ^ s)
60 in
61 Jsont.map ~dec ~enc Jsont.string
62
63(* Entry codec for the file format (without repository, added during load) *)
64type file_entry = {
65 hour : int;
66 timestamp : Ptime.t;
67 summary : string;
68 changes : string list;
69 commit_range : commit_range;
70 contributors : string list;
71 repo_url : string option;
72}
73
74let file_entry_jsont =
75 let make hour timestamp summary changes commit_range contributors repo_url =
76 { hour; timestamp; summary; changes; commit_range; contributors; repo_url }
77 in
78 let default_hour = 0 in
79 let default_timestamp = Ptime.epoch in
80 Jsont.Object.map ~kind:"daily_entry" make
81 |> Jsont.Object.mem "hour" Jsont.int ~dec_absent:default_hour ~enc:(fun e -> e.hour)
82 |> Jsont.Object.mem "timestamp" ptime_jsont ~dec_absent:default_timestamp ~enc:(fun e -> e.timestamp)
83 |> Jsont.Object.mem "summary" Jsont.string ~enc:(fun e -> e.summary)
84 |> Jsont.Object.mem "changes" (Jsont.list Jsont.string) ~enc:(fun e -> e.changes)
85 |> Jsont.Object.mem "commit_range" commit_range_jsont ~enc:(fun e -> e.commit_range)
86 |> Jsont.Object.mem "contributors" (Jsont.list Jsont.string) ~dec_absent:[] ~enc:(fun e -> e.contributors)
87 |> Jsont.Object.mem "repo_url" (Jsont.option Jsont.string) ~dec_absent:None ~enc:(fun e -> e.repo_url)
88 |> Jsont.Object.finish
89
90type json_file = {
91 json_repository : string;
92 json_entries : file_entry list;
93}
94
95let json_file_jsont =
96 let make json_repository json_entries = { json_repository; json_entries } in
97 Jsont.Object.map ~kind:"daily_changes_file" make
98 |> Jsont.Object.mem "repository" Jsont.string ~enc:(fun f -> f.json_repository)
99 |> Jsont.Object.mem "entries" (Jsont.list file_entry_jsont) ~enc:(fun f -> f.json_entries)
100 |> Jsont.Object.finish
101
102(* Parse date from filename: <repo>-<YYYY-MM-DD>.json *)
103let parse_daily_filename filename =
104 (* Check for pattern: ends with -YYYY-MM-DD.json *)
105 let len = String.length filename in
106 if len < 16 || not (String.ends_with ~suffix:".json" filename) then
107 None
108 else
109 (* Try to extract date: last 15 chars are -YYYY-MM-DD.json *)
110 let date_start = len - 15 in
111 let potential_date = String.sub filename (date_start + 1) 10 in
112 (* Validate date format YYYY-MM-DD *)
113 if String.length potential_date = 10 &&
114 potential_date.[4] = '-' && potential_date.[7] = '-' then
115 let repo = String.sub filename 0 date_start in
116 Some (repo, potential_date)
117 else
118 None
119
120(* Load a single daily file *)
121let load_file ~fs ~changes_dir ~repo ~date : entry list =
122 let filename = repo ^ "-" ^ date ^ ".json" in
123 let file_path = Eio.Path.(fs / Fpath.to_string changes_dir / filename) in
124 match Eio.Path.kind ~follow:true file_path with
125 | `Regular_file -> (
126 let content = Eio.Path.load file_path in
127 match Jsont_bytesrw.decode_string json_file_jsont content with
128 | Ok jf ->
129 List.map (fun (fe : file_entry) : entry ->
130 { repository = repo;
131 hour = fe.hour;
132 timestamp = fe.timestamp;
133 summary = fe.summary;
134 changes = fe.changes;
135 commit_range = fe.commit_range;
136 contributors = fe.contributors;
137 repo_url = fe.repo_url;
138 }) jf.json_entries
139 | Error _ -> [])
140 | _ -> []
141 | exception Eio.Io _ -> []
142
143let empty = {
144 by_repo = String_map.empty;
145 by_date = String_map.empty;
146 all_entries = [];
147}
148
149let list_repos ~fs ~changes_dir =
150 let dir_path = Eio.Path.(fs / Fpath.to_string changes_dir) in
151 match Eio.Path.kind ~follow:true dir_path with
152 | `Directory ->
153 let files = Eio.Path.read_dir dir_path in
154 files
155 |> List.filter_map parse_daily_filename
156 |> List.map fst
157 |> List.sort_uniq String.compare
158 | _ -> []
159 | exception Eio.Io _ -> []
160
161let list_dates ~fs ~changes_dir ~repo =
162 let dir_path = Eio.Path.(fs / Fpath.to_string changes_dir) in
163 match Eio.Path.kind ~follow:true dir_path with
164 | `Directory ->
165 let files = Eio.Path.read_dir dir_path in
166 files
167 |> List.filter_map (fun filename ->
168 match parse_daily_filename filename with
169 | Some (r, date) when r = repo -> Some date
170 | _ -> None)
171 |> List.sort (fun a b -> String.compare b a) (* descending *)
172 | _ -> []
173 | exception Eio.Io _ -> []
174
175let load_repo_day ~fs ~changes_dir ~repo ~date =
176 load_file ~fs ~changes_dir ~repo ~date
177
178let load_repo_all ~fs ~changes_dir ~repo =
179 let dates = list_dates ~fs ~changes_dir ~repo in
180 List.concat_map (fun date -> load_file ~fs ~changes_dir ~repo ~date) dates
181
182let load_all ~fs ~changes_dir =
183 let dir_path = Eio.Path.(fs / Fpath.to_string changes_dir) in
184 match Eio.Path.kind ~follow:true dir_path with
185 | `Directory ->
186 let files = Eio.Path.read_dir dir_path in
187 let parsed_files = List.filter_map parse_daily_filename files in
188
189 (* Load all files and build days *)
190 let days : day list = List.filter_map (fun (repo, date) ->
191 let loaded_entries : entry list = load_file ~fs ~changes_dir ~repo ~date in
192 if loaded_entries = [] then None
193 else
194 let sorted_entries : entry list = List.sort (fun (e1 : entry) (e2 : entry) ->
195 Ptime.compare e1.timestamp e2.timestamp) loaded_entries
196 in
197 Some ({ repository = repo; date; entries = sorted_entries } : day)
198 ) parsed_files in
199
200 (* Build by_repo map *)
201 let by_repo : day list String_map.t = List.fold_left (fun acc (d : day) ->
202 let existing = String_map.find_opt d.repository acc |> Option.value ~default:[] in
203 String_map.add d.repository (d :: existing) acc
204 ) String_map.empty days in
205
206 (* Sort each repo's days by date descending *)
207 let by_repo : day list String_map.t = String_map.map (fun (ds : day list) ->
208 List.sort (fun (d1 : day) (d2 : day) -> String.compare d2.date d1.date) ds
209 ) by_repo in
210
211 (* Build by_date map *)
212 let by_date : day list String_map.t = List.fold_left (fun acc (d : day) ->
213 let existing = String_map.find_opt d.date acc |> Option.value ~default:[] in
214 String_map.add d.date (d :: existing) acc
215 ) String_map.empty days in
216
217 (* Sort each date's days by repo name *)
218 let by_date : day list String_map.t = String_map.map (fun (ds : day list) ->
219 List.sort (fun (d1 : day) (d2 : day) -> String.compare d1.repository d2.repository) ds
220 ) by_date in
221
222 (* Collect all entries sorted by timestamp *)
223 let all_entries : entry list =
224 days
225 |> List.concat_map (fun (d : day) -> d.entries)
226 |> List.sort (fun (e1 : entry) (e2 : entry) -> Ptime.compare e1.timestamp e2.timestamp)
227 in
228
229 { by_repo; by_date; all_entries }
230
231 | _ -> empty
232 | exception Eio.Io _ -> empty
233
234let since (t : t) (timestamp : Ptime.t) : entry list =
235 List.filter (fun (e : entry) -> Ptime.compare e.timestamp timestamp > 0) t.all_entries
236
237let for_repo t repo =
238 String_map.find_opt repo t.by_repo |> Option.value ~default:[]
239
240let for_date t date =
241 String_map.find_opt date t.by_date |> Option.value ~default:[]
242
243let repos t =
244 String_map.bindings t.by_repo |> List.map fst
245
246let dates t =
247 String_map.bindings t.by_date
248 |> List.map fst
249 |> List.sort (fun a b -> String.compare b a) (* descending *)
250
251let entries_since ~fs ~changes_dir ~since:timestamp =
252 let t = load_all ~fs ~changes_dir in
253 since t timestamp