Merge remote-tracking branch 'origin/master' into 1.3

Conflicts:
	configure
	configure.ac
	opam
	src/client/opamArg.ml
	src/core/opamGlobals.ml
	src/format/opamFile.ml

CHANGES

@@ -10,6 +10,9 @@
 * Added numbers to lint checks, and some new checks
 * Restored the handling of a simple path to an `aspcud`-compatible executable
   in variable OPAMEXTERNALSOLVER (#2085)
+* Added package universe output to new PEF format for diagnostics
+* Prioritise newer versions even when the latest can't be installed (#2109)
+* Automatically install plugins on `opam plugin-name` (#2092)
 
 1.2.1
 * Non-system compiler definitions without source are now allowed
@@ -138,6 +141,7 @@
 * New shell completion scripts
 * Added 'opam lint' to perform checks on opam files
 * Use the published version of jsonm rather than include it (#1574)
+* Changed findlib package name from 'opam' to 'opam-lib'
 * Hundreds of smaller fixes and UI improvements
 
 1.1.2

doc/dev-manual/dev-manual.tex

@@ -285,6 +285,7 @@ \subsubsection{Package Specification files: {\tt opam}}
     ?ocaml-version: [ <and-formula(constraint)>+ ]
     ?libraries:     [ STRING+ ]
     ?syntax:        [ STRING+ ]
+    ?flags:         [ IDENT+ ]
     ?features:      [ (IDENT STRING <filter>)+ ]
 
 <argument>       := STRING
@@ -519,6 +520,20 @@ \subsubsection{Package Specification files: {\tt opam}}
   libraries and syntax extensions defined by the package. See
   Section~\ref{section:config} for more details.
 
+\item {\tt flags} allow a limited set of idents that give special handling
+  instructions for the package:
+  \begin{description}
+  \item{\tt light-uninstall} the package's uninstall instructions don't require the
+    package source. This is currently inferred when the only uninstall
+    instructions have the form `ocamlfind remove...`, but making it explicit is
+    preferred (since OPAM 1.2.0).
+  \item{\tt verbose} when this is present, the stdout of the package's build and
+    install instructions will be printed to the user (since \OPAM\ 1.2.1).
+  \item{\tt plugin} the package installs a program named \verb+opam-$NAME+. OPAM
+    will hand over to it when run with subcommand \verb+$NAME+, automatically
+    installing it if necessary.
+  \end{description}
+
 \item {\tt features} is currently experimental and shouldn't be used on the main
   package repository. It allows to define custom variables that better document
   what {\em features} are available in a given package build. Each feature is

doc/pages/Manual.md

@@ -620,6 +620,18 @@ recommended to check the validity and quality of your `opam` files.
     current switch state. An unavailable package won't generally be seen on the
     system, except with `opam list -A`.
 
+- `flags: [ <ident> ... ]`: specify package flags that may alter package
+  behaviour. Currently available flags are:
+
+    - `light-uninstall`: the package's uninstall instructions don't require
+      the package source. This is currently inferred when the only uninstall
+      instructions have the form `ocamlfind remove...`, but making it explicit
+      is preferred (since OPAM 1.2.0).
+    - `verbose`: when this is present, the stdout of the package's build and
+      install instructions will be printed to the user (since OPAM 1.2.1).
+    - `plugin`: the package installs a program named `opam-<name>` and may be
+      auto-installed and run with `opam <name>` (since OPAM 1.2.2)
+
 - `features: [ <ident> <string> { <filter> } ... ]` (EXPERIMENTAL): this field
   is currently experimental and shouldn't be used on the main package
   repository. It allows to define custom variables that better document what

src/client/opamArg.ml

@@ -2014,9 +2014,11 @@ let check_and_run_external_commands () =
     (* No such command, check if there is a matching plugin *)
     let command = opam ^ "-" ^ name in
     OpamStd.Config.init ();
+    OpamFormatConfig.init ();
+    let root_dir = OpamStateConfig.opamroot () in
+    let conf_file = OpamStateConfig.load root_dir in
     let env =
-      let root_dir = OpamStateConfig.opamroot () in
-      match OpamStateConfig.load root_dir with
+      match conf_file with
       | None -> Unix.environment ()
       | Some c ->
         let current_switch = OpamFile.Config.switch c in
@@ -2029,6 +2031,30 @@ let check_and_run_external_commands () =
     if OpamSystem.command_exists ~env command then
       let argv = Array.of_list (command :: args) in
       raise (OpamStd.Sys.Exec (command, argv, env))
+    else if conf_file <> None then
+      (* Look for a corresponding package *)
+      let t =
+        OpamState.load_state "plugins-inst" OpamStateConfig.(!r.current_switch)
+      in
+      let open OpamState.Types in
+      try
+        let pkgname = OpamPackage.Name.of_string name in
+        let candidates = Lazy.force t.available_packages in
+        let nv = OpamPackage.max_version candidates pkgname in
+        let flags = OpamFile.OPAM.flags (OpamPackage.Map.find nv t.opams) in
+        if List.mem Pkgflag_Plugin flags &&
+           not (OpamState.is_name_installed t pkgname) &&
+           OpamConsole.confirm "OPAM plugin %s is not installed. \
+                                Install it on the current switch?"
+             name
+        then
+          (Client.install [pkgname,None] None false;
+           OpamConsole.header_msg "Carrying on to \"%s\""
+             (String.concat " " (Array.to_list Sys.argv));
+           OpamConsole.msg "\n";
+           let argv = Array.of_list (command :: args) in
+           raise (OpamStd.Sys.Exec (command, argv, env)))
+      with Not_found -> ()
 
 let run default commands =
   OpamStd.Option.iter OpamVersion.set_git OpamGitVersion.version;

src/core/opamSystem.ml

@@ -516,13 +516,15 @@ end
 let is_tar_archive = Tar.is_archive
 
 let extract file dst =
-  let kind, extract_function =
+  let _, extract_function =
     if Zip.is_archive file then "zip", Zip.extract_function
     else "tar", Tar.extract_function
   in
   with_tmp_dir (fun tmp_dir ->
     match extract_function file with
-    | None   -> internal_error "%s is not a valid %s archive." file kind
+    | None   ->
+      mkdir dst;
+      copy file (dst/Filename.basename file)
     | Some f ->
       f tmp_dir;
       if Sys.file_exists dst then

src/format/opamFile.ml

@@ -118,8 +118,13 @@ module X = struct
         already_warned := true;
         false
 
+    (* Prints warnings or fails in strict mode; returns true if permissive mode
+       is enabled *)
     let check ?allow_major ?(versioned=true) =
       fun f fields ->
+        if not OpamFormatConfig.(!r.strict) && not (OpamConsole.debug ())
+        then true
+        else
         let f_opam_version =
           if List.mem s_opam_version fields then
             OpamFormat.assoc_option f.file_contents s_opam_version
@@ -149,15 +154,16 @@ module X = struct
             List.partition (fun x -> List.mem x fields) invalids
           in
           if too_many <> [] then
-            OpamConsole.warning "duplicate fields in %s: %s"
+            OpamConsole.warning "In %s:\n  duplicate fields %s"
               f.file_name
               (OpamStd.List.to_string (fun x -> x) too_many);
           let is_, s_ =
             if List.length invalids <= 1 then "is an", "" else "are", "s" in
           if invalids <> [] then
-            OpamConsole.warning "%s %s unknown field%s in %s."
+            OpamConsole.warning "In %s:\n  %s %s unknown field%s."
+              f.file_name
               (OpamStd.Format.pretty_list invalids)
-              is_ s_ f.file_name;
+              is_ s_;
           if OpamFormatConfig.(!r.strict) then
             OpamConsole.error_and_exit "Strict mode: bad fields in %s"
               f.file_name;
@@ -486,7 +492,7 @@ module X = struct
                 map,
               i+1
             | s ->
-              OpamConsole.error "At %s:%d: skipped invalid line %S"
+              OpamConsole.error "At %s:%d:\n  skipped invalid line %S"
                 (OpamFilename.prettify filename) i (String.concat " " s);
               map, i+1
           ) (empty,1) lines in
@@ -1266,7 +1272,12 @@ module X = struct
 
     let of_syntax ?(permissive=false) ?(conservative=false) f nv =
       let safe default f x y z =
-        try f x y z with OpamFormat.Bad_format _ when permissive -> default
+        try f x y z with
+        | OpamFormat.Bad_format _ as bf
+          when not conservative && not OpamFormatConfig.(!r.strict) ->
+          if not permissive then
+            OpamConsole.warning "%s" (OpamFormat.string_of_bad_format bf);
+          default
       in
       let assoc_option x y z = safe None OpamFormat.assoc_option x y z in
       let assoc_list x y z = safe [] OpamFormat.assoc_list x y z in
@@ -1300,7 +1311,7 @@ module X = struct
                   (function Depflag_Unknown _ -> false | _ -> true) flags in
               if not permissive && known_flags <> flags then
                 OpamConsole.warning
-                  "At %s: Unknown flags %s ignored for dependency %s"
+                  "At %s:\n  Unknown flags %s ignored for dependency %s"
                   (string_of_pos pos)
                   (OpamStd.Format.pretty_list (OpamStd.List.filter_map (function
                        | Depflag_Unknown s -> Some s
@@ -1398,7 +1409,7 @@ module X = struct
               allflags in
           if not permissive && known_flags <> allflags then
             OpamConsole.warning
-              "At %s: Unknown package flags %s ignored"
+              "At %s:\n  Unknown package flags %s ignored"
               (string_of_pos (OpamFormat.value_pos v))
               (OpamStd.Format.pretty_list (OpamStd.List.filter_map (function
                    | Pkgflag_Unknown s -> Some s
@@ -2351,13 +2362,6 @@ module Make (F : F) = struct
     write_files := filename :: !write_files;
     log "Wrote %s in %.3fs" filename (chrono ())
 
-  let string_of_backtrace_list = function
-    | [] | _ when not (Printexc.backtrace_status ()) -> ""
-    | btl -> List.fold_left (fun s bts ->
-        let bt_lines = OpamStd.String.split bts '\n' in
-        "\n  Backtrace:\n    "^(String.concat "\n    " bt_lines)^s
-      ) "" btl
-
   let read f =
     let filename = OpamFilename.prettify f in
     read_files := filename :: !read_files;
@@ -2379,13 +2383,7 @@ module Make (F : F) = struct
         else raise e (* Message already printed *)
       | e ->
         OpamStd.Exn.fatal e;
-        let pos,msg,btl = match e with
-          | OpamFormat.Bad_format (Some pos, btl, msg) -> pos, ":\n  "^msg, btl
-          | OpamFormat.Bad_format (None, btl, msg) -> (f,-1,-1), ":\n  "^msg, btl
-          | _ -> (f,-1,-1),"",[] in
-        let e = OpamFormat.add_pos pos e in
-        OpamConsole.error "At %s%s%s"
-          (string_of_pos pos) msg (string_of_backtrace_list btl);
+        OpamConsole.error "%s" (OpamFormat.string_of_bad_format ~file:f e);
         if OpamFormatConfig.(!r.strict) then OpamStd.Sys.exit 66
         else raise e
 
@@ -2403,14 +2401,8 @@ module Make (F : F) = struct
 
   let read_from_channel ic =
     try F.of_channel dummy_file ic with
-    | OpamFormat.Bad_format (Some pos, btl, msg) as e ->
-      OpamConsole.error "At %s: %s%s"
-        (string_of_pos pos) msg (string_of_backtrace_list btl);
-      if OpamFormatConfig.(!r.strict) then
-        OpamConsole.error_and_exit "Strict mode: aborting"
-      else raise e
-    | OpamFormat.Bad_format (None, btl, msg) as e ->
-      OpamConsole.error "Input error: %s%s" msg (string_of_backtrace_list btl);
+    | OpamFormat.Bad_format _ as e ->
+      OpamConsole.error "%s" (OpamFormat.string_of_bad_format e);
       if OpamFormatConfig.(!r.strict) then
         OpamConsole.error_and_exit "Strict mode: aborting"
       else raise e

src/format/opamFormat.ml

@@ -38,6 +38,26 @@ let add_pos pos = function
     Bad_format (Some pos, backtrace::btl, msg)
   | e -> e
 
+let string_of_backtrace_list = function
+  | [] | _ when not (Printexc.backtrace_status ()) -> ""
+  | btl -> List.fold_left (fun s bts ->
+      let bt_lines = OpamStd.String.split bts '\n' in
+      "\n  Backtrace:\n    "^(String.concat "\n    " bt_lines)^s
+    ) "" btl
+
+let string_of_bad_format ?file e =
+  match e, file with
+  | Bad_format (Some pos, btl, msg), _ ->
+    Printf.sprintf "At %s:\n  %s%s"
+      (string_of_pos pos) msg (string_of_backtrace_list btl)
+  | Bad_format (None, btl, msg), Some f ->
+    Printf.sprintf "In %s:\n  %s%s"
+      (OpamFilename.to_string f) msg (string_of_backtrace_list btl)
+  | Bad_format (None, btl, msg), None ->
+    Printf.sprintf "Input error:\n  %s%s"
+      msg (string_of_backtrace_list btl)
+  | _ -> ""
+
 let item_pos = function
   | Section (pos,_) | Variable (pos,_,_) -> pos
 
@@ -397,8 +417,7 @@ let parse_dep_flag = function
   | Ident (_, s) -> Depflag_Unknown s
   | x ->
     bad_format ~pos:(value_pos x)
-      "Unknown dependency flag %S. Expected one of \"build\", \"test\" or \
-       a version constraint operator"
+      "Invalid dependency flag %s, must be an ident"
       (string_of_value x)
 
 let make_dep_flag = function
@@ -694,17 +713,18 @@ let make_flag = function
   | Pkgflag_LightUninstall -> make_ident "light-uninstall"
   | Pkgflag_AllSwitches -> make_ident "all-switches"
   | Pkgflag_Verbose -> make_ident "verbose"
+  | Pkgflag_Plugin -> make_ident "plugin"
   | Pkgflag_Unknown s -> make_ident s
 
 let parse_flag = function
   | Ident (_,"light-uninstall") -> Pkgflag_LightUninstall
   | Ident (_,"all-switches") -> Pkgflag_AllSwitches
   | Ident (_,"verbose") -> Pkgflag_Verbose
+  | Ident (_,"plugin") -> Pkgflag_Plugin
   | Ident (_,s) -> Pkgflag_Unknown s
   | x ->
     bad_format ~pos:(value_pos x)
-      "Unknown package flag %S. Expected one of \"light-uninstall\", \
-       \"all-switches\" or \"verbose\""
+      "Invalid package flag %s, must be an ident"
       (string_of_value x)
 
 (* TAGS *)

src/format/opamFormat.mli

@@ -45,6 +45,8 @@ exception Bad_format of pos option * string list * string
 (** Raise [Bad_format]. *)
 val bad_format: ?pos:pos -> ('a, unit, string, 'b) format4 -> 'a
 
+val string_of_bad_format: ?file:filename -> exn -> string
+
 (** Adds a position to a Bad_format exception if it doesn't have one yet *)
 val add_pos: pos -> exn -> exn
 

src/format/opamTypes.mli

@@ -120,6 +120,9 @@ type package_flag =
   | Pkgflag_LightUninstall (** The package doesn't require downloading to uninstall *)
   | Pkgflag_AllSwitches (** The package is pervasive on all switches *)
   | Pkgflag_Verbose (** The package's scripts output is to be displayed to the user *)
+  | Pkgflag_Plugin (** The package is an opam plugin that will install a
+                       [opam-<name>] exec, and may be auto-installed when doing
+                       [opam <name>] *)
   | Pkgflag_Unknown of string (** Used for error reporting, otherwise ignored *)
 
 (** Flags on dependencies *)

src_ext/Makefile

@@ -36,8 +36,8 @@ lib-ext: clone build copy
 	@
 
 build: clone
-	$(MAKE) -f $(OCAMLMAKEFILE) subprojs SUBTARGET=bcl
-	$(MAKE) -f $(OCAMLMAKEFILE) subprojs SUBTARGET=ncl
+	[ $(OCAMLC) = no ] || $(MAKE) -f $(OCAMLMAKEFILE) subprojs SUBTARGET=bcl
+	[ $(OCAMLOPT) = no ] || $(MAKE) -f $(OCAMLMAKEFILE) subprojs SUBTARGET=ncl
 
 all: bcl ncl