slices.network
Auto-indexing service and GraphQL API for AT Protocol Records
quickslice.slices.network/
atproto
gleam
graphql
chadtmiller.com
1import database/executor.{type Executor}
2import database/repositories/lexicons
3import gleam/dict
4import gleam/dynamic/decode
5import gleam/json
6import gleam/list
7import gleam/result
8import gleam/string
9import honk
10import logging
11import simplifile
12import zip_helper
13
14pub type ImportStats {
15 ImportStats(total: Int, imported: Int, failed: Int, errors: List(String))
16}
17
18/// Imports lexicons from a directory into the database
19pub fn import_lexicons_from_directory(
20 directory: String,
21 db: Executor,
22) -> Result(ImportStats, String) {
23 // Scan directory for JSON files
24 logging.log(logging.Info, "[import] Scanning directory recursively...")
25 use file_paths <- result.try(scan_directory_recursive(directory))
26
27 logging.log(
28 logging.Info,
29 "[import] Found "
30 <> string.inspect(list.length(file_paths))
31 <> " .json files",
32 )
33 logging.log(logging.Info, "")
34 logging.log(logging.Info, "[import] Reading all lexicon files...")
35
36 // Read all files first to get their content
37 let file_contents =
38 file_paths
39 |> list.filter_map(fn(file_path) {
40 case simplifile.read(file_path) {
41 Ok(content) -> Ok(#(file_path, content))
42 Error(_) -> Error(Nil)
43 }
44 })
45
46 logging.log(logging.Info, "[import] Validating all lexicons together...")
47
48 // Extract all JSON strings and parse them to Json objects
49 let all_json_strings = list.map(file_contents, fn(pair) { pair.1 })
50 let all_json_results =
51 honk.parse_json_strings(all_json_strings)
52 |> result.map_error(fn(_) { "Failed to parse JSON" })
53
54 let validation_result = case all_json_results {
55 Ok(all_jsons) ->
56 case honk.validate(all_jsons) {
57 Ok(_) -> {
58 logging.log(
59 logging.Info,
60 "[import] All lexicons validated successfully",
61 )
62 Ok(Nil)
63 }
64 Error(err_map) -> {
65 let error_details = format_validation_errors(err_map)
66 logging.log(
67 logging.Error,
68 "[import] Validation failed: " <> error_details,
69 )
70 Error("Validation failed: " <> error_details)
71 }
72 }
73 Error(_) -> {
74 logging.log(logging.Error, "[import] Failed to parse JSON")
75 Error("Failed to parse JSON")
76 }
77 }
78
79 logging.log(logging.Info, "")
80 logging.log(logging.Info, "[import] Importing lexicons to database...")
81
82 // If validation failed, return error immediately
83 use _ <- result.try(validation_result)
84
85 // Wipe existing lexicons before importing new set
86 let _ = lexicons.delete_all(db)
87
88 // Validation succeeded, import each lexicon
89 let results =
90 file_contents
91 |> list.map(fn(pair) {
92 let #(file_path, json_content) = pair
93 import_validated_lexicon(db, file_path, json_content)
94 })
95
96 // Calculate stats
97 let total = list.length(results)
98 let imported =
99 results
100 |> list.filter(fn(r) {
101 case r {
102 Ok(_) -> True
103 Error(_) -> False
104 }
105 })
106 |> list.length
107
108 let failed = total - imported
109
110 let errors =
111 results
112 |> list.filter_map(fn(r) {
113 case r {
114 Error(err) -> Ok(err)
115 Ok(_) -> Error(Nil)
116 }
117 })
118
119 Ok(ImportStats(
120 total: total,
121 imported: imported,
122 failed: failed,
123 errors: errors,
124 ))
125}
126
127/// Scans a directory recursively for JSON files
128pub fn scan_directory_recursive(path: String) -> Result(List(String), String) {
129 case simplifile.is_directory(path) {
130 Ok(False) -> Error("Path is not a directory: " <> path)
131 Error(_) -> Error("Failed to access directory: " <> path)
132 Ok(True) -> {
133 case simplifile.read_directory(path) {
134 Error(_) -> Error("Failed to read directory: " <> path)
135 Ok(entries) -> {
136 entries
137 |> list.filter_map(fn(entry) {
138 // Skip macOS metadata directories and hidden files
139 case
140 string.starts_with(entry, "__MACOSX")
141 || string.starts_with(entry, ".")
142 {
143 True -> Error(Nil)
144 False -> {
145 let entry_path = path <> "/" <> entry
146
147 case simplifile.is_directory(entry_path) {
148 Ok(True) -> {
149 // Recursively scan subdirectory
150 case scan_directory_recursive(entry_path) {
151 Ok(paths) -> Ok(paths)
152 Error(_) -> Error(Nil)
153 }
154 }
155 _ -> {
156 // Check if it's a .json file
157 case string.ends_with(entry, ".json") {
158 True -> Ok([entry_path])
159 False -> Error(Nil)
160 }
161 }
162 }
163 }
164 }
165 })
166 |> list.flatten
167 |> Ok
168 }
169 }
170 }
171 }
172}
173
174/// Parses and validates a lexicon file
175pub fn parse_and_validate_lexicon(
176 file_path: String,
177) -> Result(#(String, String), String) {
178 // Read file content
179 use json_content <- result.try(case simplifile.read(file_path) {
180 Ok(content) -> Ok(content)
181 Error(_) -> Error("Failed to read file")
182 })
183
184 // Extract lexicon ID from JSON
185 use lexicon_id <- result.try(extract_lexicon_id(json_content))
186
187 // Validate using lexicon package
188 use json_obj <- result.try(
189 honk.parse_json_string(json_content)
190 |> result.map_error(fn(_) { "Failed to parse JSON" }),
191 )
192
193 case honk.validate([json_obj]) {
194 Ok(_) -> Ok(#(lexicon_id, json_content))
195 Error(err_map) ->
196 Error("Validation failed: " <> format_validation_errors(err_map))
197 }
198}
199
200/// Extracts the lexicon ID from JSON content
201fn extract_lexicon_id(json_content: String) -> Result(String, String) {
202 // Try to decode "id" field first
203 let id_decoder = {
204 use id <- decode.field("id", decode.string)
205 decode.success(id)
206 }
207
208 // Try to decode "lexicon" field as fallback
209 let lexicon_decoder = {
210 use lex <- decode.field("lexicon", decode.string)
211 decode.success(lex)
212 }
213
214 case json.parse(json_content, id_decoder) {
215 Ok(id) -> Ok(id)
216 Error(_) ->
217 case json.parse(json_content, lexicon_decoder) {
218 Ok(id) -> Ok(id)
219 Error(_) ->
220 Error("Missing 'id' or 'lexicon' field - not a valid lexicon schema")
221 }
222 }
223}
224
225/// Formats validation errors from error map into readable strings
226fn format_validation_errors(
227 error_map: dict.Dict(String, List(String)),
228) -> String {
229 error_map
230 |> dict.to_list
231 |> list.flat_map(fn(entry) {
232 let #(_key, errors) = entry
233 errors
234 })
235 |> string.join(", ")
236}
237
238/// Imports a single lexicon file (with validation)
239pub fn import_single_lexicon(
240 conn: Executor,
241 file_path: String,
242) -> Result(String, String) {
243 let file_name = case string.split(file_path, "/") |> list.last {
244 Ok(name) -> name
245 Error(_) -> file_path
246 }
247
248 case parse_and_validate_lexicon(file_path) {
249 Ok(#(lexicon_id, json_content)) -> {
250 case lexicons.insert(conn, lexicon_id, json_content) {
251 Ok(_) -> {
252 logging.log(logging.Info, "[import] " <> lexicon_id)
253 Ok(lexicon_id)
254 }
255 Error(_) -> {
256 let err_msg = file_name <> ": Database insertion failed"
257 logging.log(logging.Error, "[import] " <> err_msg)
258 Error(err_msg)
259 }
260 }
261 }
262 Error(err) -> {
263 let err_msg = file_name <> ": " <> err
264 logging.log(logging.Error, "[import] " <> err_msg)
265 Error(err_msg)
266 }
267 }
268}
269
270/// Imports a lexicon that has already been validated
271/// Used when importing multiple lexicons that were validated together
272fn import_validated_lexicon(
273 conn: Executor,
274 file_path: String,
275 json_content: String,
276) -> Result(String, String) {
277 let file_name = case string.split(file_path, "/") |> list.last {
278 Ok(name) -> name
279 Error(_) -> file_path
280 }
281
282 case extract_lexicon_id(json_content) {
283 Ok(lexicon_id) -> {
284 case lexicons.insert(conn, lexicon_id, json_content) {
285 Ok(_) -> {
286 logging.log(logging.Info, "[import] " <> lexicon_id)
287 Ok(lexicon_id)
288 }
289 Error(_) -> {
290 let err_msg = file_name <> ": Database insertion failed"
291 logging.log(logging.Error, "[import] " <> err_msg)
292 Error(err_msg)
293 }
294 }
295 }
296 Error(err) -> {
297 let err_msg = file_name <> ": " <> err
298 logging.log(logging.Error, "[import] " <> err_msg)
299 Error(err_msg)
300 }
301 }
302}
303
304/// Decode base64 string to bit array using Erlang FFI
305@external(erlang, "base64", "decode")
306fn decode_base64(base64: String) -> BitArray
307
308/// Import lexicons from a base64-encoded ZIP file
309/// Returns ImportStats on success, error message on failure
310pub fn import_lexicons_from_base64_zip(
311 zip_base64: String,
312 db: Executor,
313) -> Result(ImportStats, String) {
314 // Decode base64 to binary
315 let zip_binary = decode_base64(zip_base64)
316
317 // Create temporary directory for extraction
318 let temp_dir = "/tmp/lexicons_" <> string.inspect(erlang_timestamp())
319 use _ <- result.try(case simplifile.create_directory(temp_dir) {
320 Ok(_) -> Ok(Nil)
321 Error(_) -> Error("Failed to create temporary directory")
322 })
323
324 // Write ZIP file to temp location
325 let zip_path = temp_dir <> "/lexicons.zip"
326 use _ <- result.try(case simplifile.write_bits(zip_path, zip_binary) {
327 Ok(_) -> Ok(Nil)
328 Error(_) -> Error("Failed to write ZIP file")
329 })
330
331 // Extract ZIP to temp directory
332 let extract_dir = temp_dir <> "/extracted"
333 use _ <- result.try(case simplifile.create_directory(extract_dir) {
334 Ok(_) -> Ok(Nil)
335 Error(_) -> Error("Failed to create extraction directory")
336 })
337
338 use _ <- result.try(zip_helper.extract_zip(zip_path, extract_dir))
339
340 // Import lexicons from extracted directory
341 let import_result = import_lexicons_from_directory(extract_dir, db)
342
343 // Clean up temp directory
344 let _ = simplifile.delete(zip_path)
345 let _ = delete_directory_recursive(extract_dir)
346 let _ = simplifile.delete(temp_dir)
347
348 import_result
349}
350
351/// Recursively delete a directory and its contents
352fn delete_directory_recursive(path: String) -> Result(Nil, Nil) {
353 case simplifile.is_directory(path) {
354 Ok(True) -> {
355 case simplifile.read_directory(path) {
356 Ok(entries) -> {
357 // Delete all entries first
358 list.each(entries, fn(entry) {
359 let entry_path = path <> "/" <> entry
360 let _ = delete_directory_recursive(entry_path)
361 Nil
362 })
363
364 // Then delete the directory itself
365 case simplifile.delete(path) {
366 Ok(_) -> Ok(Nil)
367 Error(_) -> Error(Nil)
368 }
369 }
370 Error(_) -> Error(Nil)
371 }
372 }
373 _ -> {
374 // Not a directory, try to delete as file
375 case simplifile.delete(path) {
376 Ok(_) -> Ok(Nil)
377 Error(_) -> Error(Nil)
378 }
379 }
380 }
381}
382
383/// Get current Erlang timestamp (for temp directory naming)
384@external(erlang, "erlang", "system_time")
385fn erlang_timestamp() -> Int