···33* opam-version: "2.1" must appear at most once and as the first non-comment
44 item. If opam-version is at the start and is greater than the library version,
55 `OpamLexer.Error` and `Parsing.Parse_error` are no longer raised; instead the
66- items parsed so far are returned. [#43 @dra27]
66+ opam-version variable is returned along with an invalid group to signal the
77+ parsing error, giving the client enough information to act. [#43, #44 @dra27]
78892.1.2 [07 Jan 2021]
910* Some hash-consing for strings [#27 @AltGr]
···15151616(** Opam config file generic type parser *)
17171818-let get_pos_full ?(s=1) n =
1919- let spos = Parsing.rhs_start_pos s in
2020- let epos = Parsing.rhs_end_pos n in
1818+let pos_of_lexing_pos spos epos =
2119 Lexing.({
2222- filename = spos.pos_fname;
2323- start = spos.pos_lnum, spos.pos_cnum - spos.pos_bol;
2424- stop = epos.pos_lnum, epos.pos_cnum - epos.pos_bol;
2525- })
2626-2727-let get_pos n = get_pos_full ~s:n n
2020+ filename = spos.pos_fname;
2121+ start = spos.pos_lnum, spos.pos_cnum - spos.pos_bol;
2222+ stop = epos.pos_lnum, epos.pos_cnum - epos.pos_bol;
2323+ })
28242929-let parsed_so_far = ref []
2525+let get_pos_full ?(s=1) n =
2626+ pos_of_lexing_pos (Parsing.rhs_start_pos s) (Parsing.rhs_end_pos n)
30273131-let record_token t =
3232- parsed_so_far := t :: !parsed_so_far; t
2828+let get_pos n = get_pos_full ~s:n n
33293430(* This must match up with the package's version; checked by the build system *)
3531let version = (2, 1)
···6965%%
70667167main:
7272-| items EOF { parsed_so_far := []; fun file_name ->
6868+| items EOF { fun file_name ->
7369 { file_contents = $1; file_name } }
7470;
7571···80768177item:
8278| IDENT COLON value {
8383- record_token
8479 { pos = get_pos_full 3;
8580 pelem =
8681 Variable ({ pos = get_pos 1; pelem = $1 }, $3);
8782 }
8883}
8984| IDENT LBRACE items RBRACE {
9090- record_token
9185 { pos = get_pos_full 4;
9286 pelem =
9387 Section ({section_kind = { pos = get_pos 1; pelem = $1 };
···9892 }
9993}
10094| IDENT STRING LBRACE items RBRACE {
101101- record_token
10295 { pos = get_pos_full 4;
10396 pelem =
10497 Section ({section_kind = { pos = get_pos 1; pelem = $1 };
···165158 Parsing.clear_parser ();
166159 raise e
167160168168-exception Nothing
169169-170170-let reset_lexbuf l file_name (start_line, start_col) (end_line, end_col) =
161161+(* Update a lexbuf with position information prior to raising an exception *)
162162+let reset_lexbuf_and_abort l file_name (start_line, start_col) (end_line, end_col) exn =
171163 let open Lexing in
172164 l.lex_start_p <- {pos_fname = file_name; pos_lnum = start_line; pos_bol = 0; pos_cnum = start_col};
173165 l.lex_curr_p <- {pos_fname = file_name; pos_lnum = end_line; pos_bol = 0; pos_cnum = end_col};
174174- true
166166+ exn ()
175167176176-let main t l file_name =
177177- (* Always return a result from parsing/lexing, but note if an exception
178178- occurred. *)
179179- let parsing_exception = ref Lexing.(Nothing, dummy_pos, dummy_pos) in
180180- let raise_if_parsing_failed = function
181181- | false -> ()
182182- | true ->
183183- match parsing_exception with
184184- | {contents = (Nothing, _, _)} -> ()
185185- | {contents = (e, start, curr)} ->
186186- let open Lexing in
187187- l.lex_start_p <- start;
188188- l.lex_curr_p <- curr;
189189- raise e
168168+(* cf. OpamStd.fatal - always allow standard exceptions to propagate. *)
169169+let not_fatal = function
170170+| Sys.Break
171171+| Assert_failure _
172172+| Match_failure _ -> false
173173+| _ -> true
174174+175175+let get_three_tokens lexer lexbuf =
176176+ let open Lexing in
177177+ try
178178+ let p0 = lexbuf.lex_start_p, lexbuf.lex_curr_p in
179179+ let t1 = lexer lexbuf in
180180+ let p1 = lexbuf.lex_start_p, lexbuf.lex_curr_p in
181181+ let t2 = lexer lexbuf in
182182+ let p2 = lexbuf.lex_start_p, lexbuf.lex_curr_p in
183183+ let t3 = lexer lexbuf in
184184+ let p3 = lexbuf.lex_start_p, lexbuf.lex_curr_p in
185185+ ((p0, p1, p2, p3), (t1, t2, t3))
186186+ with
187187+ | e when not_fatal e -> raise Parsing.Parse_error
188188+189189+(* Wrap the ocamlyacc parser *)
190190+let main lexer lexbuf file_name =
191191+ (* Extract the exceptions for opam-version not at the top of the file and
192192+ opam-version duplicated. OpamLexer has special cases for these two
193193+ constants. If OpamLexer.token isn't used, raise Parse_error instead. *)
194194+ let exn_not_first () =
195195+ let _ = lexer (Lexing.from_string "version: \"42\"\nopam-version: \"2.1\"") in
196196+ raise Parsing.Parse_error
197197+ and exn_duplicate () =
198198+ let _ = lexer (Lexing.from_string "opam-version: \"2.1\"\nopam-version: \"z\"") in
199199+ raise Parsing.Parse_error
200200+ and restore_pos (start, curr) =
201201+ let open Lexing in
202202+ lexbuf.lex_start_p <- start;
203203+ lexbuf.lex_curr_p <- curr
204204+ in
205205+ (* Raises the exn_not_first or exn_duplicate exceptions if an invalid
206206+ opam-version variable is found in the result. *)
207207+ let scan_opam_version_variable format_2_1_or_greater = function
208208+ | {pelem = Variable({pelem = "opam-version"; _}, {pelem = String ver; _}); pos = {start; stop; _}} ->
209209+ if format_2_1_or_greater then
210210+ (* [opam-version] can only appear once for 2.1+ *)
211211+ reset_lexbuf_and_abort lexbuf file_name start stop exn_duplicate
212212+ else if nopatch ver > (2, 0) then
213213+ (* Only [opam-version: "2.0"] can appear after the first non-comment/whitespace line of the file *)
214214+ reset_lexbuf_and_abort lexbuf file_name start stop exn_not_first
215215+ else
216216+ ()
217217+ | _ -> ()
218218+ in
219219+ (* Now parse the header of the file manually. The smallest valid opam file
220220+ is `ident: atom`, so if we can't read three tokens we have a parse error *)
221221+ let ((((_, p0) as initial_pos), ((_, p1) as pos1), ((_, p2) as pos2), ((_, p3) as pos3)), (t1, t2, t3)) =
222222+ get_three_tokens lexer lexbuf
223223+ in
224224+ (* Parse those three tokens if they are [opam-version: ver] *)
225225+ let (header, format_2_1_or_greater, trap_exceptions) =
226226+ match (t1, t2, t3) with
227227+ | (IDENT "opam-version", COLON, STRING ver) ->
228228+ let header =
229229+ (* Parsing or lexing errors immediate following opam-version may cause
230230+ an exception to be raised before the element has been fully parsed.
231231+ In this case, we generate a single opam-version Variable to return.
232232+ *)
233233+ {pelem = Variable({pelem = "opam-version"; pos = pos_of_lexing_pos p0 p1},
234234+ {pelem = String ver; pos = pos_of_lexing_pos p2 p3});
235235+ pos = pos_of_lexing_pos p0 p3}
236236+ in
237237+ (header, (nopatch ver >= (2, 1)), (nopatch ver > version))
238238+ | _ ->
239239+ (* Default is [opam-version: "2.0"] *)
240240+ let pos = {filename = ""; start = (0, 0); stop = (0, 0)} in
241241+ ({pelem = Variable ({pelem = ""; pos}, {pelem = Int 42; pos}); pos}, false, false)
190242 in
191191- let t l =
192192- try t l
193193- with
194194- | Sys.Break
195195- | Assert_failure _
196196- | Match_failure _ as e -> raise e
197197- | e -> parsing_exception := Lexing.(e, l.lex_start_p, l.lex_curr_p); EOF
243243+ (* The parser will use position information from the lexbuf, so replay the
244244+ positions, even if we're not actually reading anything. *)
245245+ restore_pos initial_pos;
246246+ (* Wrap the lexer to simulate reading those three tokens a second time *)
247247+ let lexer =
248248+ let tokens = ref [t1, pos1; t2, pos2; t3, pos3] in
249249+ fun lexbuf ->
250250+ match tokens with
251251+ | {contents = (t, p)::rest} ->
252252+ tokens := rest;
253253+ restore_pos p;
254254+ t
255255+ | {contents = []} ->
256256+ lexer lexbuf
198257 in
199199- let r =
200200- try with_clear_parser (main t l) file_name
201201- with Parsing.Parse_error as e ->
202202- parsing_exception := Lexing.(e, l.lex_start_p, l.lex_curr_p);
203203- (* Record the tokens captured so far *)
204204- let r = {file_contents = List.rev !parsed_so_far; file_name} in
205205- parsed_so_far := [];
206206- r
207207- in
208208- match r with
209209- | {file_contents = {pelem = Variable({pelem = "opam-version"; _}, {pelem = String ver; _}); _}::items; _}
210210- when nopatch ver >= (2, 1) ->
211211- let opam_version_variable = function
212212- | {pelem = Variable({pelem = "opam-version"; _}, _); pos = {start; stop; _}} ->
213213- reset_lexbuf l file_name start stop
214214- | _ -> false
258258+ let result =
259259+ try with_clear_parser (main lexer lexbuf) file_name
260260+ with e when trap_exceptions && not_fatal e ->
261261+ (* Append a syntactically invalid sentinel section "#" to the version
262262+ header which was manually parsed. That is then sufficient
263263+ information for a client to determine that the file was invalid.
264264+ If OpamBaseParser.version = (2, 1), this would allow
265265+ `opam-version: "2.2"`, containing no lexer or parser changes, still to
266266+ report syntax errors in opam 2.2, by using this sentinel group to
267267+ detect the parsing error. *)
268268+ let sentinel =
269269+ let pos =
270270+ Lexing.(pos_of_lexing_pos lexbuf.lex_start_p lexbuf.lex_curr_p)
215271 in
216216- (* For opam-version: 2.1 and later, there must be no other opam-version
217217- fields. *)
218218- if List.exists opam_version_variable items then
219219- raise Parsing.Parse_error;
220220- (* Parsing and lexing errors from future versions of opam are ignored:
221221- the intent is that the tool will abort/ignore because of the
222222- opam-version field rather than through lexer/parser errors. *)
223223- raise_if_parsing_failed (nopatch ver <= version);
224224- r
225225- | {file_contents = items; _} ->
226226- let opam_version_greater_2_0 = function
227227- | {pelem = Variable({pelem = "opam-version"; _}, {pelem = String ver; _}); pos = {start; stop; _}}
228228- when nopatch ver > (2, 0) ->
229229- reset_lexbuf l file_name start stop
230230- | _ -> false
272272+ let section =
273273+ {section_kind = {pelem = "#"; pos};
274274+ section_name = None;
275275+ section_items = {pelem = []; pos}}
231276 in
232232- (* opam-version: 2.1 or later must be the first item. *)
233233- if List.exists opam_version_greater_2_0 items then
234234- raise Parsing.Parse_error;
235235- (* If no opam-version field was given, all exceptions must be
236236- raised. *)
237237- raise_if_parsing_failed true;
238238- r
277277+ {pelem = Section section; pos}
278278+ in
279279+ {file_contents = [header; sentinel]; file_name}
280280+ in
281281+ begin
282282+ match result with
283283+ | {file_contents = _::items; _} ->
284284+ (* Ensure that there are no `opam-version` fields with a value >= "2.1"
285285+ further down the file. *)
286286+ List.iter (scan_opam_version_variable format_2_1_or_greater) items
287287+ | _ -> ()
288288+ end;
289289+ result
239290240240-let value t l =
241241- try
242242- let r = value t l in
243243- Parsing.clear_parser ();
244244- r
245245- with
246246- | e ->
247247- Parsing.clear_parser ();
248248- raise e
291291+let value t l = with_clear_parser (value t) l
+7
vendor/opam/opam-file-format/src/opamLexer.mll
···138138| pfxop { PFXOP (FullPos.pfxop (Lexing.lexeme lexbuf)) }
139139| envop { ENVOP (FullPos.env_update_op (Lexing.lexeme lexbuf)) }
140140| eof { EOF }
141141+(* OpamBaseParser can't directly access OpamLexer.Error so it uses these
142142+ constants (which would parse that way) to extract the exception values.
143143+ *)
144144+| "opam-version: \"2.1\"\nopam-version: \"z\"" eof
145145+ { error "opam-version cannot be repeated" }
146146+| "version: \"42\"\nopam-version: \"2.1\"" eof
147147+ { error "opam-version must be the first non-comment line" }
141148| _ { let token = Lexing.lexeme lexbuf in
142149 error "'%s' is not a valid token" token }
143150
+74
vendor/opam/opam-file-format/src/opamPrinter.ml
···11111212open OpamParserTypes
13131414+(* The code duplication with OpamBaseParser is irritating, but can't be solved
1515+ without introducing another module. *)
1616+let nopatch v =
1717+ let s =
1818+ try
1919+ let i = String.index v '.' in
2020+ let i = String.index_from v (i+1) '.' in
2121+ (String.sub v 0 i)
2222+ with Not_found ->
2323+ let rec f i =
2424+ if i >= String.length v then v
2525+ else match String.get v i with
2626+ | '0'..'9' | '.' -> f (i+1)
2727+ | _ -> String.sub v 0 i
2828+ in
2929+ f 0
3030+ in
3131+ try Scanf.sscanf s "%u.%u" (fun maj min -> (maj, min))
3232+ with Scanf.Scan_failure _ -> (0, 0)
3333+3434+let valid_opamfile_contents = function
3535+| (Variable (_, "opam-version", String (_, ver)))::items
3636+ when nopatch ver >= (2, 1) ->
3737+ let opam_version_field = function
3838+ | Variable (_, "opam-version", _) -> true
3939+ | _ -> false
4040+ in
4141+ not (List.exists opam_version_field items)
4242+| _::items ->
4343+ let greater_2_0_opam_version_field = function
4444+ | Variable (_, "opam-version", String (_, ver))
4545+ when nopatch ver >= (2, 1) -> true
4646+ | _ -> false
4747+ in
4848+ not (List.exists greater_2_0_opam_version_field items)
4949+| [] -> true
5050+1451let relop = function
1552 | `Eq -> "="
1653 | `Neq -> "!="
···134171 Format.pp_print_newline fmt ()
135172136173let items l =
174174+ if not (valid_opamfile_contents l) then
175175+ invalid_arg "OpamPrinter.items";
137176 format_items Format.str_formatter l; Format.flush_str_formatter ()
138177139178let opamfile f =
···223262 (String.concat "\n" (List.map item s.section_items))
224263225264 let item_order a b = match a,b with
265265+ | Variable (_, "opam-version", String (_, ver)), _
266266+ when nopatch ver >= (2, 1) -> -1
267267+ | _, Variable (_, "opam-version", String (_, ver))
268268+ when nopatch ver >= (2, 1) -> 1
226269 | Section _, Variable _ -> 1
227270 | Variable _, Section _ -> -1
228271 | Variable (_,i,_), Variable (_,j,_) -> String.compare i j
···232275 else compare s.section_name t.section_name
233276234277 let items its =
278278+ if not (valid_opamfile_contents its) then
279279+ invalid_arg "OpamPrinter.Normalise.items";
235280 let its = List.sort item_order its in
236281 String.concat "\n" (List.map item its) ^ "\n"
237282···240285241286module Preserved = struct
242287 let items txt orig f =
288288+ if not (valid_opamfile_contents f) then
289289+ invalid_arg "OpamPrinter.Preserved.items";
243290 let pos_index =
244291 let lines_index =
245292 let rec aux acc s =
···329376330377 open OpamParserTypes.FullPos
331378379379+ let valid_opamfile_contents = function
380380+ | {pelem = Variable ({pelem = "opam-version"; _}, {pelem = String ver; _}); _}::items
381381+ when nopatch ver >= (2, 1) ->
382382+ let opam_version_field = function
383383+ | {pelem = Variable ({pelem = "opam-version"; _}, _); _} -> true
384384+ | _ -> false
385385+ in
386386+ not (List.exists opam_version_field items)
387387+ | _::items ->
388388+ let greater_2_0_opam_version_field = function
389389+ | {pelem = Variable ({pelem = "opam-version"; _}, {pelem = String ver; _}); _}
390390+ when nopatch ver >= (2, 1) -> true
391391+ | _ -> false
392392+ in
393393+ not (List.exists greater_2_0_opam_version_field items)
394394+ | [] -> true
395395+332396 let relop_kind = relop
333397 let relop r = relop_kind r.pelem
334398 let logop_kind = logop
···433497 Format.pp_print_newline fmt ()
434498435499 let items l =
500500+ if not (valid_opamfile_contents l) then
501501+ invalid_arg "OpamPrinter.FullPos.items";
436502 format_items Format.str_formatter l; Format.flush_str_formatter ()
437503438504 let opamfile f =
···548614549615 let item_order a b =
550616 match a.pelem ,b.pelem with
617617+ | Variable ({pelem = "opam-version"; _}, {pelem = String ver; _}), _
618618+ when nopatch ver >= (2, 1) -> -1
619619+ | _, Variable ({pelem = "opam-version"; _}, {pelem = String ver; _})
620620+ when nopatch ver >= (2, 1) -> 1
551621 | Section _, Variable _ -> 1
552622 | Variable _, Section _ -> -1
553623 | Variable (i,_), Variable (j,_) -> String.compare i.pelem j.pelem
···557627 else compare s.section_name t.section_name
558628559629 let items its =
630630+ if not (valid_opamfile_contents its) then
631631+ invalid_arg "OpamPrinter.Normalise.items";
560632 let its = List.sort item_order its in
561633 String.concat "\n" (List.map item its) ^ "\n"
562634···565637566638 module Preserved = struct
567639 let items txt orig f =
640640+ if not (valid_opamfile_contents f) then
641641+ invalid_arg "OpamPrinter.Preserved.items";
568642 let pos_index =
569643 let lines_index =
570644 let rec aux acc s =
+22-4
vendor/opam/opam-file-format/src/opamPrinter.mli
···6161 newlines}. *)
62626363 val items: opamfile_item list -> string
6464+ (** Converts a list of opam field/sections to a string.
6565+6666+ @raise Invalid_argument if ["opam-version"] is greater than "2.0"
6767+ and not solely the first item. *)
64686569 val opamfile: opamfile -> string
6666- (** Converts an {!opamfile} to a string. *)
7070+ (** Converts an {!opamfile} to a string.
7171+7272+ @raise Invalid_argument if ["opam-version"] is greater than "2.0"
7373+ and not solely the first item. *)
67746875 val format_opamfile: Format.formatter -> opamfile -> unit
6976 (** Writes an {!opamfile} to a [Format.formatter]. The function ensures that all
7077 newlines are sent using [Format]'s break instructions (and so ultimately are
7178 processed with the [out_newline] function of the formatter) but it is the
7279 responsibility of the caller to ensure that the formatter is configured for
7373- the required output, if necessary. *)
8080+ the required output, if necessary.
8181+8282+ @raise Invalid_argument if ["opam-version"] is greater than "2.0"
8383+ and not solely the first item. *)
74847585 (** {2 Normalised output for opam syntax files} *)
7686···106116 (** [items str orig_its its] converts [its] to string, while attempting to
107117 preserve the layout and comments of the original [str] for unmodified
108118 elements. The function assumes that [str] parses to the items
109109- [orig_its]. *)
119119+ [orig_its].
120120+121121+ @raise Invalid_argument if ["opam-version"] is greater than "2.0"
122122+ and not solely the first item in either list. *)
110123111124 val opamfile: ?format_from:file_name -> opamfile -> string
112125 (** [opamfile f] converts [f] to string, respecting the layout and comments in
113126 the corresponding on-disk file for unmodified items. [format_from] can be
114114- specified instead of using the filename specified in [f]. *)
127127+ specified instead of using the filename specified in [f].
128128+129129+ @raise Invalid_argument if ["opam-version"] is greater than "2.0"
130130+ and not solely the first item in the list. Note that
131131+ any errors in the file raise {!OpamLexer.Error} as
132132+ normal. *)
115133 end
116134117135 (** {2 Random utility functions} *)
+64-12
vendor/opam/opam-file-format/tests/versions.ml
···10101111module A = Alcotest
12121313+exception IA
1414+1515+let tests_corrupt =
1616+ let minimal =
1717+ {|
1818+opam-version: "2.1"
1919+version: "2.1"
2020+ |}
2121+ in
2222+ let opamfile = OpamParser.FullPos.string minimal "corrupt.opam" in
2323+ let corrupt = OpamParserTypes.FullPos.({opamfile with file_contents = List.rev opamfile.file_contents}) in
2424+ [
2525+ "OpamPrinter.FullPos.opamfile", OpamPrinter.FullPos.opamfile;
2626+ "OpamPrinter.FullPos.Normalise.opamfile", OpamPrinter.FullPos.Normalise.opamfile
2727+] |> List.map (fun (name, f) ->
2828+ name, (fun () ->
2929+ A.check_raises name IA (fun () ->
3030+ try ignore (f corrupt) with Invalid_argument _ -> raise IA)))
3131+1332let tests_exn = [
1414- "opam-version > 2.0 not at start 1",
3333+ "opam-version > 2.0 not at start 1", OpamLexer.Error("opam-version must be the first non-comment line"),
1534 {|
1635version: "2.1"
1736opam-version: "2.1"
1837 |};
1919- "opam-version > 2.1 repeated",
3838+ "opam-version > 2.1 repeated", OpamLexer.Error("opam-version cannot be repeated"),
2039 {|
2140opam-version: "2.1"
2241opam-version: "2.1"
2342 |};
2424- "no opam-version and parsing error",
4343+ "no opam-version and parsing error", Parsing.Parse_error,
2544 {|
2645build: [ "echo"
2746 |};
2828- "opam-version 2.1 and parsing error",
4747+ "opam-version 2.1 and lexing error", OpamLexer.Error "'@' is not a valid token",
4848+ {|
4949+opam-version: "2.1"
5050+@
5151+ |};
5252+ "opam-version 2.1 and parsing error", Parsing.Parse_error,
2953 {|
3054opam-version: "2.1"
3155build: [ "echo"
3256 |};
3333-] |> List.map (fun (name, content) ->
5757+ "opam-version 2.1 and immediate parsing error", Parsing.Parse_error,
5858+ {|
5959+opam-version: "2.1"
6060+!!
6161+ |};
6262+] |> List.map (fun (name, exn, content) ->
3463 name, (fun () ->
3535- A.check_raises name Parsing.Parse_error (fun () ->
6464+ A.check_raises name exn (fun () ->
3665 OpamParser.FullPos.string content "broken.opam" |> ignore)))
6666+6767+let has_sentinel =
6868+ let open OpamParserTypes.FullPos in
6969+ fun {file_contents; _} ->
7070+ match List.rev file_contents with
7171+ | {pelem = Section {section_kind = {pelem = "#"; _}; _}; _}::_ -> true
7272+ | _ -> false
37733874let tests_noexn = [
3939- "opam-version 2.2 and parsing error",
7575+ "opam-version 42.0 and parsing error",
4076 {|
4141-opam-version: "2.2"
4242-build: [ "echo"
7777+opam-version: "42.0"
7878+version: "42.0"
7979+!!
8080+ |};
8181+ "opam-version 42.0 and evil parsing error",
8282+ {|
8383+opam-version: "42.0" <
8484+ |};
8585+ "opam-version 42.0 and immediate parsing error",
8686+ {|
8787+opam-version: "42.0"
8888+!!
8989+ |};
9090+ "opam-version 42.0 and lexing error",
9191+ {|
9292+opam-version: "42.0"
9393+@
4394 |};
4495] |> List.map (fun (name, content) ->
4596 name, (fun () ->
4646- A.check A.unit name ()
4747- (OpamParser.FullPos.string content "broken.opam" |> ignore)))
9797+ A.check A.bool name true
9898+ (OpamParser.FullPos.string content "broken.opam"
9999+ |> has_sentinel)))
4810049101let tests =
5050- ["opam-version", tests_exn @ tests_noexn]
102102+ ["opam-version", tests_corrupt @ tests_exn @ tests_noexn]