···3* opam-version: "2.1" must appear at most once and as the first non-comment
4 item. If opam-version is at the start and is greater than the library version,
5 `OpamLexer.Error` and `Parsing.Parse_error` are no longer raised; instead the
6- items parsed so far are returned. [#43 @dra27]
0782.1.2 [07 Jan 2021]
9* Some hash-consing for strings [#27 @AltGr]
···3* opam-version: "2.1" must appear at most once and as the first non-comment
4 item. If opam-version is at the start and is greater than the library version,
5 `OpamLexer.Error` and `Parsing.Parse_error` are no longer raised; instead the
6+ opam-version variable is returned along with an invalid group to signal the
7+ parsing error, giving the client enough information to act. [#43, #44 @dra27]
892.1.2 [07 Jan 2021]
10* Some hash-consing for strings [#27 @AltGr]
···1516(** Opam config file generic type parser *)
1718-let get_pos_full ?(s=1) n =
19- let spos = Parsing.rhs_start_pos s in
20- let epos = Parsing.rhs_end_pos n in
21 Lexing.({
22- filename = spos.pos_fname;
23- start = spos.pos_lnum, spos.pos_cnum - spos.pos_bol;
24- stop = epos.pos_lnum, epos.pos_cnum - epos.pos_bol;
25- })
26-27-let get_pos n = get_pos_full ~s:n n
2829-let parsed_so_far = ref []
03031-let record_token t =
32- parsed_so_far := t :: !parsed_so_far; t
3334(* This must match up with the package's version; checked by the build system *)
35let version = (2, 1)
···69%%
7071main:
72-| items EOF { parsed_so_far := []; fun file_name ->
73 { file_contents = $1; file_name } }
74;
75···8081item:
82| IDENT COLON value {
83- record_token
84 { pos = get_pos_full 3;
85 pelem =
86 Variable ({ pos = get_pos 1; pelem = $1 }, $3);
87 }
88}
89| IDENT LBRACE items RBRACE {
90- record_token
91 { pos = get_pos_full 4;
92 pelem =
93 Section ({section_kind = { pos = get_pos 1; pelem = $1 };
···98 }
99}
100| IDENT STRING LBRACE items RBRACE {
101- record_token
102 { pos = get_pos_full 4;
103 pelem =
104 Section ({section_kind = { pos = get_pos 1; pelem = $1 };
···165 Parsing.clear_parser ();
166 raise e
167168-exception Nothing
169-170-let reset_lexbuf l file_name (start_line, start_col) (end_line, end_col) =
171 let open Lexing in
172 l.lex_start_p <- {pos_fname = file_name; pos_lnum = start_line; pos_bol = 0; pos_cnum = start_col};
173 l.lex_curr_p <- {pos_fname = file_name; pos_lnum = end_line; pos_bol = 0; pos_cnum = end_col};
174- true
175176-let main t l file_name =
177- (* Always return a result from parsing/lexing, but note if an exception
178- occurred. *)
179- let parsing_exception = ref Lexing.(Nothing, dummy_pos, dummy_pos) in
180- let raise_if_parsing_failed = function
181- | false -> ()
182- | true ->
183- match parsing_exception with
184- | {contents = (Nothing, _, _)} -> ()
185- | {contents = (e, start, curr)} ->
186- let open Lexing in
187- l.lex_start_p <- start;
188- l.lex_curr_p <- curr;
189- raise e
000000000000000000000000000000000000000000000000000000000000190 in
191- let t l =
192- try t l
193- with
194- | Sys.Break
195- | Assert_failure _
196- | Match_failure _ as e -> raise e
197- | e -> parsing_exception := Lexing.(e, l.lex_start_p, l.lex_curr_p); EOF
0000000198 in
199- let r =
200- try with_clear_parser (main t l) file_name
201- with Parsing.Parse_error as e ->
202- parsing_exception := Lexing.(e, l.lex_start_p, l.lex_curr_p);
203- (* Record the tokens captured so far *)
204- let r = {file_contents = List.rev !parsed_so_far; file_name} in
205- parsed_so_far := [];
206- r
207- in
208- match r with
209- | {file_contents = {pelem = Variable({pelem = "opam-version"; _}, {pelem = String ver; _}); _}::items; _}
210- when nopatch ver >= (2, 1) ->
211- let opam_version_variable = function
212- | {pelem = Variable({pelem = "opam-version"; _}, _); pos = {start; stop; _}} ->
213- reset_lexbuf l file_name start stop
214- | _ -> false
215 in
216- (* For opam-version: 2.1 and later, there must be no other opam-version
217- fields. *)
218- if List.exists opam_version_variable items then
219- raise Parsing.Parse_error;
220- (* Parsing and lexing errors from future versions of opam are ignored:
221- the intent is that the tool will abort/ignore because of the
222- opam-version field rather than through lexer/parser errors. *)
223- raise_if_parsing_failed (nopatch ver <= version);
224- r
225- | {file_contents = items; _} ->
226- let opam_version_greater_2_0 = function
227- | {pelem = Variable({pelem = "opam-version"; _}, {pelem = String ver; _}); pos = {start; stop; _}}
228- when nopatch ver > (2, 0) ->
229- reset_lexbuf l file_name start stop
230- | _ -> false
231 in
232- (* opam-version: 2.1 or later must be the first item. *)
233- if List.exists opam_version_greater_2_0 items then
234- raise Parsing.Parse_error;
235- (* If no opam-version field was given, all exceptions must be
236- raised. *)
237- raise_if_parsing_failed true;
238- r
000000239240-let value t l =
241- try
242- let r = value t l in
243- Parsing.clear_parser ();
244- r
245- with
246- | e ->
247- Parsing.clear_parser ();
248- raise e
···1516(** Opam config file generic type parser *)
1718+let pos_of_lexing_pos spos epos =
0019 Lexing.({
20+ filename = spos.pos_fname;
21+ start = spos.pos_lnum, spos.pos_cnum - spos.pos_bol;
22+ stop = epos.pos_lnum, epos.pos_cnum - epos.pos_bol;
23+ })
002425+let get_pos_full ?(s=1) n =
26+ pos_of_lexing_pos (Parsing.rhs_start_pos s) (Parsing.rhs_end_pos n)
2728+let get_pos n = get_pos_full ~s:n n
02930(* This must match up with the package's version; checked by the build system *)
31let version = (2, 1)
···65%%
6667main:
68+| items EOF { fun file_name ->
69 { file_contents = $1; file_name } }
70;
71···7677item:
78| IDENT COLON value {
079 { pos = get_pos_full 3;
80 pelem =
81 Variable ({ pos = get_pos 1; pelem = $1 }, $3);
82 }
83}
84| IDENT LBRACE items RBRACE {
085 { pos = get_pos_full 4;
86 pelem =
87 Section ({section_kind = { pos = get_pos 1; pelem = $1 };
···92 }
93}
94| IDENT STRING LBRACE items RBRACE {
095 { pos = get_pos_full 4;
96 pelem =
97 Section ({section_kind = { pos = get_pos 1; pelem = $1 };
···158 Parsing.clear_parser ();
159 raise e
160161+(* Update a lexbuf with position information prior to raising an exception *)
162+let reset_lexbuf_and_abort l file_name (start_line, start_col) (end_line, end_col) exn =
0163 let open Lexing in
164 l.lex_start_p <- {pos_fname = file_name; pos_lnum = start_line; pos_bol = 0; pos_cnum = start_col};
165 l.lex_curr_p <- {pos_fname = file_name; pos_lnum = end_line; pos_bol = 0; pos_cnum = end_col};
166+ exn ()
167168+(* cf. OpamStd.fatal - always allow standard exceptions to propagate. *)
169+let not_fatal = function
170+| Sys.Break
171+| Assert_failure _
172+| Match_failure _ -> false
173+| _ -> true
174+175+let get_three_tokens lexer lexbuf =
176+ let open Lexing in
177+ try
178+ let p0 = lexbuf.lex_start_p, lexbuf.lex_curr_p in
179+ let t1 = lexer lexbuf in
180+ let p1 = lexbuf.lex_start_p, lexbuf.lex_curr_p in
181+ let t2 = lexer lexbuf in
182+ let p2 = lexbuf.lex_start_p, lexbuf.lex_curr_p in
183+ let t3 = lexer lexbuf in
184+ let p3 = lexbuf.lex_start_p, lexbuf.lex_curr_p in
185+ ((p0, p1, p2, p3), (t1, t2, t3))
186+ with
187+ | e when not_fatal e -> raise Parsing.Parse_error
188+189+(* Wrap the ocamlyacc parser *)
190+let main lexer lexbuf file_name =
191+ (* Extract the exceptions for opam-version not at the top of the file and
192+ opam-version duplicated. OpamLexer has special cases for these two
193+ constants. If OpamLexer.token isn't used, raise Parse_error instead. *)
194+ let exn_not_first () =
195+ let _ = lexer (Lexing.from_string "version: \"42\"\nopam-version: \"2.1\"") in
196+ raise Parsing.Parse_error
197+ and exn_duplicate () =
198+ let _ = lexer (Lexing.from_string "opam-version: \"2.1\"\nopam-version: \"z\"") in
199+ raise Parsing.Parse_error
200+ and restore_pos (start, curr) =
201+ let open Lexing in
202+ lexbuf.lex_start_p <- start;
203+ lexbuf.lex_curr_p <- curr
204+ in
205+ (* Raises the exn_not_first or exn_duplicate exceptions if an invalid
206+ opam-version variable is found in the result. *)
207+ let scan_opam_version_variable format_2_1_or_greater = function
208+ | {pelem = Variable({pelem = "opam-version"; _}, {pelem = String ver; _}); pos = {start; stop; _}} ->
209+ if format_2_1_or_greater then
210+ (* [opam-version] can only appear once for 2.1+ *)
211+ reset_lexbuf_and_abort lexbuf file_name start stop exn_duplicate
212+ else if nopatch ver > (2, 0) then
213+ (* Only [opam-version: "2.0"] can appear after the first non-comment/whitespace line of the file *)
214+ reset_lexbuf_and_abort lexbuf file_name start stop exn_not_first
215+ else
216+ ()
217+ | _ -> ()
218+ in
219+ (* Now parse the header of the file manually. The smallest valid opam file
220+ is `ident: atom`, so if we can't read three tokens we have a parse error *)
221+ let ((((_, p0) as initial_pos), ((_, p1) as pos1), ((_, p2) as pos2), ((_, p3) as pos3)), (t1, t2, t3)) =
222+ get_three_tokens lexer lexbuf
223+ in
224+ (* Parse those three tokens if they are [opam-version: ver] *)
225+ let (header, format_2_1_or_greater, trap_exceptions) =
226+ match (t1, t2, t3) with
227+ | (IDENT "opam-version", COLON, STRING ver) ->
228+ let header =
229+ (* Parsing or lexing errors immediate following opam-version may cause
230+ an exception to be raised before the element has been fully parsed.
231+ In this case, we generate a single opam-version Variable to return.
232+ *)
233+ {pelem = Variable({pelem = "opam-version"; pos = pos_of_lexing_pos p0 p1},
234+ {pelem = String ver; pos = pos_of_lexing_pos p2 p3});
235+ pos = pos_of_lexing_pos p0 p3}
236+ in
237+ (header, (nopatch ver >= (2, 1)), (nopatch ver > version))
238+ | _ ->
239+ (* Default is [opam-version: "2.0"] *)
240+ let pos = {filename = ""; start = (0, 0); stop = (0, 0)} in
241+ ({pelem = Variable ({pelem = ""; pos}, {pelem = Int 42; pos}); pos}, false, false)
242 in
243+ (* The parser will use position information from the lexbuf, so replay the
244+ positions, even if we're not actually reading anything. *)
245+ restore_pos initial_pos;
246+ (* Wrap the lexer to simulate reading those three tokens a second time *)
247+ let lexer =
248+ let tokens = ref [t1, pos1; t2, pos2; t3, pos3] in
249+ fun lexbuf ->
250+ match tokens with
251+ | {contents = (t, p)::rest} ->
252+ tokens := rest;
253+ restore_pos p;
254+ t
255+ | {contents = []} ->
256+ lexer lexbuf
257 in
258+ let result =
259+ try with_clear_parser (main lexer lexbuf) file_name
260+ with e when trap_exceptions && not_fatal e ->
261+ (* Append a syntactically invalid sentinel section "#" to the version
262+ header which was manually parsed. That is then sufficient
263+ information for a client to determine that the file was invalid.
264+ If OpamBaseParser.version = (2, 1), this would allow
265+ `opam-version: "2.2"`, containing no lexer or parser changes, still to
266+ report syntax errors in opam 2.2, by using this sentinel group to
267+ detect the parsing error. *)
268+ let sentinel =
269+ let pos =
270+ Lexing.(pos_of_lexing_pos lexbuf.lex_start_p lexbuf.lex_curr_p)
000271 in
272+ let section =
273+ {section_kind = {pelem = "#"; pos};
274+ section_name = None;
275+ section_items = {pelem = []; pos}}
00000000000276 in
277+ {pelem = Section section; pos}
278+ in
279+ {file_contents = [header; sentinel]; file_name}
280+ in
281+ begin
282+ match result with
283+ | {file_contents = _::items; _} ->
284+ (* Ensure that there are no `opam-version` fields with a value >= "2.1"
285+ further down the file. *)
286+ List.iter (scan_opam_version_variable format_2_1_or_greater) items
287+ | _ -> ()
288+ end;
289+ result
290291+let value t l = with_clear_parser (value t) l
00000000
+7
vendor/opam/opam-file-format/src/opamLexer.mll
···138| pfxop { PFXOP (FullPos.pfxop (Lexing.lexeme lexbuf)) }
139| envop { ENVOP (FullPos.env_update_op (Lexing.lexeme lexbuf)) }
140| eof { EOF }
0000000141| _ { let token = Lexing.lexeme lexbuf in
142 error "'%s' is not a valid token" token }
143
···138| pfxop { PFXOP (FullPos.pfxop (Lexing.lexeme lexbuf)) }
139| envop { ENVOP (FullPos.env_update_op (Lexing.lexeme lexbuf)) }
140| eof { EOF }
141+(* OpamBaseParser can't directly access OpamLexer.Error so it uses these
142+ constants (which would parse that way) to extract the exception values.
143+ *)
144+| "opam-version: \"2.1\"\nopam-version: \"z\"" eof
145+ { error "opam-version cannot be repeated" }
146+| "version: \"42\"\nopam-version: \"2.1\"" eof
147+ { error "opam-version must be the first non-comment line" }
148| _ { let token = Lexing.lexeme lexbuf in
149 error "'%s' is not a valid token" token }
150
+74
vendor/opam/opam-file-format/src/opamPrinter.ml
···1112open OpamParserTypes
13000000000000000000000000000000000000014let relop = function
15 | `Eq -> "="
16 | `Neq -> "!="
···134 Format.pp_print_newline fmt ()
135136let items l =
00137 format_items Format.str_formatter l; Format.flush_str_formatter ()
138139let opamfile f =
···223 (String.concat "\n" (List.map item s.section_items))
224225 let item_order a b = match a,b with
0000226 | Section _, Variable _ -> 1
227 | Variable _, Section _ -> -1
228 | Variable (_,i,_), Variable (_,j,_) -> String.compare i j
···232 else compare s.section_name t.section_name
233234 let items its =
00235 let its = List.sort item_order its in
236 String.concat "\n" (List.map item its) ^ "\n"
237···240241module Preserved = struct
242 let items txt orig f =
00243 let pos_index =
244 let lines_index =
245 let rec aux acc s =
···329330 open OpamParserTypes.FullPos
33100000000000000000332 let relop_kind = relop
333 let relop r = relop_kind r.pelem
334 let logop_kind = logop
···433 Format.pp_print_newline fmt ()
434435 let items l =
00436 format_items Format.str_formatter l; Format.flush_str_formatter ()
437438 let opamfile f =
···548549 let item_order a b =
550 match a.pelem ,b.pelem with
0000551 | Section _, Variable _ -> 1
552 | Variable _, Section _ -> -1
553 | Variable (i,_), Variable (j,_) -> String.compare i.pelem j.pelem
···557 else compare s.section_name t.section_name
558559 let items its =
00560 let its = List.sort item_order its in
561 String.concat "\n" (List.map item its) ^ "\n"
562···565566 module Preserved = struct
567 let items txt orig f =
00568 let pos_index =
569 let lines_index =
570 let rec aux acc s =
···1112open OpamParserTypes
1314+(* The code duplication with OpamBaseParser is irritating, but can't be solved
15+ without introducing another module. *)
16+let nopatch v =
17+ let s =
18+ try
19+ let i = String.index v '.' in
20+ let i = String.index_from v (i+1) '.' in
21+ (String.sub v 0 i)
22+ with Not_found ->
23+ let rec f i =
24+ if i >= String.length v then v
25+ else match String.get v i with
26+ | '0'..'9' | '.' -> f (i+1)
27+ | _ -> String.sub v 0 i
28+ in
29+ f 0
30+ in
31+ try Scanf.sscanf s "%u.%u" (fun maj min -> (maj, min))
32+ with Scanf.Scan_failure _ -> (0, 0)
33+34+let valid_opamfile_contents = function
35+| (Variable (_, "opam-version", String (_, ver)))::items
36+ when nopatch ver >= (2, 1) ->
37+ let opam_version_field = function
38+ | Variable (_, "opam-version", _) -> true
39+ | _ -> false
40+ in
41+ not (List.exists opam_version_field items)
42+| _::items ->
43+ let greater_2_0_opam_version_field = function
44+ | Variable (_, "opam-version", String (_, ver))
45+ when nopatch ver >= (2, 1) -> true
46+ | _ -> false
47+ in
48+ not (List.exists greater_2_0_opam_version_field items)
49+| [] -> true
50+51let relop = function
52 | `Eq -> "="
53 | `Neq -> "!="
···171 Format.pp_print_newline fmt ()
172173let items l =
174+ if not (valid_opamfile_contents l) then
175+ invalid_arg "OpamPrinter.items";
176 format_items Format.str_formatter l; Format.flush_str_formatter ()
177178let opamfile f =
···262 (String.concat "\n" (List.map item s.section_items))
263264 let item_order a b = match a,b with
265+ | Variable (_, "opam-version", String (_, ver)), _
266+ when nopatch ver >= (2, 1) -> -1
267+ | _, Variable (_, "opam-version", String (_, ver))
268+ when nopatch ver >= (2, 1) -> 1
269 | Section _, Variable _ -> 1
270 | Variable _, Section _ -> -1
271 | Variable (_,i,_), Variable (_,j,_) -> String.compare i j
···275 else compare s.section_name t.section_name
276277 let items its =
278+ if not (valid_opamfile_contents its) then
279+ invalid_arg "OpamPrinter.Normalise.items";
280 let its = List.sort item_order its in
281 String.concat "\n" (List.map item its) ^ "\n"
282···285286module Preserved = struct
287 let items txt orig f =
288+ if not (valid_opamfile_contents f) then
289+ invalid_arg "OpamPrinter.Preserved.items";
290 let pos_index =
291 let lines_index =
292 let rec aux acc s =
···376377 open OpamParserTypes.FullPos
378379+ let valid_opamfile_contents = function
380+ | {pelem = Variable ({pelem = "opam-version"; _}, {pelem = String ver; _}); _}::items
381+ when nopatch ver >= (2, 1) ->
382+ let opam_version_field = function
383+ | {pelem = Variable ({pelem = "opam-version"; _}, _); _} -> true
384+ | _ -> false
385+ in
386+ not (List.exists opam_version_field items)
387+ | _::items ->
388+ let greater_2_0_opam_version_field = function
389+ | {pelem = Variable ({pelem = "opam-version"; _}, {pelem = String ver; _}); _}
390+ when nopatch ver >= (2, 1) -> true
391+ | _ -> false
392+ in
393+ not (List.exists greater_2_0_opam_version_field items)
394+ | [] -> true
395+396 let relop_kind = relop
397 let relop r = relop_kind r.pelem
398 let logop_kind = logop
···497 Format.pp_print_newline fmt ()
498499 let items l =
500+ if not (valid_opamfile_contents l) then
501+ invalid_arg "OpamPrinter.FullPos.items";
502 format_items Format.str_formatter l; Format.flush_str_formatter ()
503504 let opamfile f =
···614615 let item_order a b =
616 match a.pelem ,b.pelem with
617+ | Variable ({pelem = "opam-version"; _}, {pelem = String ver; _}), _
618+ when nopatch ver >= (2, 1) -> -1
619+ | _, Variable ({pelem = "opam-version"; _}, {pelem = String ver; _})
620+ when nopatch ver >= (2, 1) -> 1
621 | Section _, Variable _ -> 1
622 | Variable _, Section _ -> -1
623 | Variable (i,_), Variable (j,_) -> String.compare i.pelem j.pelem
···627 else compare s.section_name t.section_name
628629 let items its =
630+ if not (valid_opamfile_contents its) then
631+ invalid_arg "OpamPrinter.Normalise.items";
632 let its = List.sort item_order its in
633 String.concat "\n" (List.map item its) ^ "\n"
634···637638 module Preserved = struct
639 let items txt orig f =
640+ if not (valid_opamfile_contents f) then
641+ invalid_arg "OpamPrinter.Preserved.items";
642 let pos_index =
643 let lines_index =
644 let rec aux acc s =
+22-4
vendor/opam/opam-file-format/src/opamPrinter.mli
···61 newlines}. *)
6263 val items: opamfile_item list -> string
00006465 val opamfile: opamfile -> string
66- (** Converts an {!opamfile} to a string. *)
0006768 val format_opamfile: Format.formatter -> opamfile -> unit
69 (** Writes an {!opamfile} to a [Format.formatter]. The function ensures that all
70 newlines are sent using [Format]'s break instructions (and so ultimately are
71 processed with the [out_newline] function of the formatter) but it is the
72 responsibility of the caller to ensure that the formatter is configured for
73- the required output, if necessary. *)
0007475 (** {2 Normalised output for opam syntax files} *)
76···106 (** [items str orig_its its] converts [its] to string, while attempting to
107 preserve the layout and comments of the original [str] for unmodified
108 elements. The function assumes that [str] parses to the items
109- [orig_its]. *)
000110111 val opamfile: ?format_from:file_name -> opamfile -> string
112 (** [opamfile f] converts [f] to string, respecting the layout and comments in
113 the corresponding on-disk file for unmodified items. [format_from] can be
114- specified instead of using the filename specified in [f]. *)
00000115 end
116117 (** {2 Random utility functions} *)
···61 newlines}. *)
6263 val items: opamfile_item list -> string
64+ (** Converts a list of opam field/sections to a string.
65+66+ @raise Invalid_argument if ["opam-version"] is greater than "2.0"
67+ and not solely the first item. *)
6869 val opamfile: opamfile -> string
70+ (** Converts an {!opamfile} to a string.
71+72+ @raise Invalid_argument if ["opam-version"] is greater than "2.0"
73+ and not solely the first item. *)
7475 val format_opamfile: Format.formatter -> opamfile -> unit
76 (** Writes an {!opamfile} to a [Format.formatter]. The function ensures that all
77 newlines are sent using [Format]'s break instructions (and so ultimately are
78 processed with the [out_newline] function of the formatter) but it is the
79 responsibility of the caller to ensure that the formatter is configured for
80+ the required output, if necessary.
81+82+ @raise Invalid_argument if ["opam-version"] is greater than "2.0"
83+ and not solely the first item. *)
8485 (** {2 Normalised output for opam syntax files} *)
86···116 (** [items str orig_its its] converts [its] to string, while attempting to
117 preserve the layout and comments of the original [str] for unmodified
118 elements. The function assumes that [str] parses to the items
119+ [orig_its].
120+121+ @raise Invalid_argument if ["opam-version"] is greater than "2.0"
122+ and not solely the first item in either list. *)
123124 val opamfile: ?format_from:file_name -> opamfile -> string
125 (** [opamfile f] converts [f] to string, respecting the layout and comments in
126 the corresponding on-disk file for unmodified items. [format_from] can be
127+ specified instead of using the filename specified in [f].
128+129+ @raise Invalid_argument if ["opam-version"] is greater than "2.0"
130+ and not solely the first item in the list. Note that
131+ any errors in the file raise {!OpamLexer.Error} as
132+ normal. *)
133 end
134135 (** {2 Random utility functions} *)