Add registry v2

.gitignore

@@ -1,5 +1,6 @@
 /_build
 /deps
 /tmp
+/src/safe_erl_term.erl
 erl_crash.dump
 *.ez

lib/hex.ex

@@ -14,6 +14,7 @@ defmodule Hex do
 
   def start(_, _) do
     import Supervisor.Spec
+    dev_setup()
 
     Mix.SCM.append(Hex.SCM)
     Mix.RemoteConverger.register(Hex.RemoteConverger)
@@ -23,7 +24,7 @@ defmodule Hex do
 
     children = [
       worker(Hex.State, []),
-      worker(Hex.Registry.ETS, []),
+      worker(Hex.Registry.Server, []),
       worker(Hex.Parallel, [:hex_fetcher, [max_parallel: 64]]),
     ]
 
@@ -52,4 +53,12 @@ defmodule Hex do
   else
     def string_to_charlist(string), do: String.to_charlist(string)
   end
+
+  if Mix.env in [:dev, :test] do
+    defp dev_setup do
+      :erlang.system_flag(:backtrace_depth, 30)
+    end
+  else
+    defp dev_setup, do: :ok
+  end
 end

lib/hex/api/package.ex

@@ -5,6 +5,10 @@ defmodule Hex.API.Package do
     API.request(:get, API.api_url("packages/#{name}"), [])
   end
 
+  def search(search) do
+    API.request(:get, API.api_url("packages?search=#{search}&sort=downloads"), [])
+  end
+
   defmodule Owner do
     def add(package, owner, auth) do
       owner = URI.encode_www_form(owner)

lib/hex/api/registry.ex

@@ -1,16 +1,49 @@
 defmodule Hex.API.Registry do
   alias Hex.API
+  alias Hex.Crypto.PublicKey
 
-  def get(opts \\ []) do
+  @default_repo "https://repo.hex.pm"
+  @public_keys_html "https://hex.pm/docs/public_keys"
+
+  def get_package(name, opts \\ []) do
     headers =
       if etag = opts[:etag] do
-        %{'if-none-match' => '"' ++ etag ++ '"'}
+        %{'if-none-match' => etag}
       end
 
-    API.request(:get, API.repo_url("registry.ets.gz"), headers || [])
+    API.request(:get, API.repo_url("packages/#{name}"), headers || [])
+  end
+
+  def verify(body) do
+    %{signature: signature, payload: payload} = :hex_pb_signed.decode_msg(body, :Signed)
+    if Hex.State.fetch!(:check_registry?) do
+      do_verify(payload, signature)
+    end
+    payload
+  end
+
+  defp do_verify(payload, signature) do
+    repo = Hex.State.fetch!(:repo) || @default_repo
+    key = PublicKey.public_key(repo)
+
+    unless key do
+      Mix.raise "No public key stored for #{repo}. Either install a public " <>
+                "key with `mix hex.public_keys` or disable the registry " <>
+                "verification check by setting `HEX_UNSAFE_REGISTRY=1`."
+    end
+
+    unless Hex.Crypto.PublicKey.verify(payload, :sha512, signature, [key]) do
+      Mix.raise "Could not verify authenticity of fetched registry file. " <>
+                "This may happen because a proxy or some entity is " <>
+                "interfering with the download or because you don't have a " <>
+                "public key to verify the registry.\n\nYou may try again " <>
+                "later or check if a new public key has been released on " <>
+                "our public keys page: #{@public_keys_html}"
+    end
   end
 
-  def get_signature() do
-    API.request(:get, API.repo_url("registry.ets.gz.signed"), [])
+  def decode(body) do
+    %{releases: releases} = :hex_pb_package.decode_msg(body, :Package)
+    releases
   end
 end

lib/hex/public_key.ex → lib/hex/crypto/public_key.ex

@@ -1,4 +1,4 @@
-defmodule Hex.PublicKey do
+defmodule Hex.Crypto.PublicKey do
   @doc """
   Returns the filesystem path for public keys.
   """
@@ -17,11 +17,11 @@ defmodule Hex.PublicKey do
         []
     end
 
-    keys ++ [{"hex.pm", Hex.State.fetch!(:hexpm_pk)}]
+    keys ++ [{"https://repo.hex.pm", Hex.State.fetch!(:hexpm_pk)}]
   end
 
-  def public_keys(domain) do
-    Enum.find(public_keys(), fn {domain2, _key} -> domain == domain2 end)
+  def public_key(url) do
+    Enum.find(public_keys(), fn {url2, _key} -> url == url2 end)
   end
 
   @doc """
@@ -46,7 +46,7 @@ defmodule Hex.PublicKey do
   """
   def verify(binary, hash, signature, keys) do
     Enum.any?(keys, fn {id, key} ->
-      :public_key.verify binary, hash, signature, decode!(id, key)
+      :public_key.verify(binary, hash, signature, decode!(id, key))
     end)
   end
 end

lib/hex/mix.ex

@@ -178,13 +178,23 @@ defmodule Hex.Mix do
   @spec to_lock(%{}) :: %{}
   def to_lock(result) do
     Enum.into(result, %{}, fn {name, app, version} ->
-      checksum = Hex.Registry.get_checksum(name, version) |> String.downcase
-      managers = Hex.Registry.get_build_tools(name, version) |> Enum.map(&String.to_atom/1)
-      deps = Hex.Registry.get_deps(name, version) |> Enum.map(&registry_dep_to_def/1)
-      {String.to_atom(app), {:hex, String.to_atom(name), version, checksum, managers, deps}}
+      checksum = Hex.Registry.checksum(name, version) |> Base.encode16(case: :lower)
+      deps = Hex.Registry.deps(name, version) |> Enum.map(&registry_dep_to_def/1)
+      {String.to_atom(app), {:hex, String.to_atom(name), version, checksum, [], deps}}
     end)
   end
 
   defp registry_dep_to_def({name, app, req, optional}),
     do: {String.to_atom(app), req, hex: String.to_atom(name), optional: optional}
+
+  def packages_from_lock(lock) do
+    Enum.flat_map(lock, fn {_app, info} ->
+      case Hex.Utils.lock(info) do
+        [:hex, name, _version, _checksum, _managers, _deps] ->
+          [Atom.to_string(name)]
+        _ ->
+          []
+      end
+    end)
+  end
 end

lib/hex/registry.ex

@@ -1,28 +1,23 @@
 defmodule Hex.Registry do
   @pdict_id :"$hex_registry"
 
-  @callback open(Keyword.t)                           :: {:ok, term} | {:error, String.t}
-  @callback close(term)                               :: boolean
-  @callback version(term)                             :: String.t
-  @callback installs(term)                            :: [{String.t, String.t}]
-  @callback stat(term)                                :: {non_neg_integer, non_neg_integer}
-  @callback search(term, String.t)                    :: [String.t]
-  @callback all_packages(term)                        :: [String.t]
-  @callback get_versions(term, String.t)              :: [String.t]
-  @callback get_deps(term, String.t, String.t)        :: [{String.t, String.t, String.t, boolean}]
-  @callback get_checksum(term, String.t, String.t)    :: binary
-  @callback get_build_tools(term, String.t, String.t) :: [String.t]
+  @type name :: term
+  @type package :: String.t
+  @type version :: String.t
+  @callback open(Keyword.t) :: {:ok, name} | {:already_open, name} | {:error, String.t}
+  @callback close(name) :: boolean
+  @callback prefetch(name, package) :: :ok
+  @callback versions(name, package) :: [version]
+  @callback deps(name, package, version) :: [{String.t, String.t, String.t, boolean}]
+  @callback checksum(name, package, version) :: binary
 
   options = quote do [
-    version(),
-    installs(),
-    stat(),
-    search(term),
-    all_packages(),
-    get_versions(package),
-    get_deps(package, version),
-    get_checksum(package, version),
-    get_build_tools(package, version)]
+    prefetch(packages),
+    versions(package),
+    deps(package, version),
+    checksum(package, version),
+    tarball_etag(package, version),
+    tarball_etag(package, version, etag)]
   end
 
   Enum.each(options, fn {function, _, args} ->
@@ -66,60 +61,4 @@ defmodule Hex.Registry do
         false
     end
   end
-
-  def info_installs do
-    installs = installs()
-    if version = latest_version(installs) do
-      Hex.Shell.warn "A new Hex version is available (#{version}), " <>
-                     "please update with `mix local.hex`"
-    else
-      check_elixir_version(installs)
-    end
-  end
-
-  defp latest_version(versions) do
-    current_elixir = System.version
-    current_hex    = Hex.version
-
-    versions
-    |> Enum.filter(fn {hex, _} -> Hex.Version.compare(hex, current_hex) == :gt end)
-    |> Enum.filter(fn {_, elixirs} -> Hex.Version.compare(hd(elixirs), current_elixir) != :gt end)
-    |> Enum.map(&elem(&1, 0))
-    |> Enum.sort(&(Hex.Version.compare(&1, &2) == :gt))
-    |> List.first
-  end
-
-  defp check_elixir_version(versions) do
-    {:ok, built}   = Hex.Version.parse(Hex.elixir_version())
-    {:ok, current} = Hex.Version.parse(System.version)
-
-    unless match_minor?(current, built) do
-      case :lists.keyfind(Hex.version, 1, versions) do
-        {_, elixirs} ->
-          if match_elixir_version?(elixirs, current) do
-            Hex.Shell.warn "Hex was built against Elixir #{Hex.elixir_version} " <>
-              "and you are running #{System.version}, please run `mix local.hex` " <>
-              "to update to a matching version"
-          end
-
-        false ->
-          :ok
-      end
-    end
-  end
-
-  defp match_elixir_version?(elixirs, current) do
-    Enum.any?(elixirs, fn elixir ->
-      {:ok, elixir} = Hex.Version.parse(elixir)
-      elixir.major == current.major and elixir.minor == current.minor
-    end)
-  end
-
-  defp match_minor?(current, %Version{major: major, minor: minor}) do
-    lower = %Version{major: major, minor: minor,     patch: 0, pre: [],  build: nil}
-    upper = %Version{major: major, minor: minor + 1, patch: 0, pre: [0], build: nil}
-
-    Hex.Version.compare(current, lower) in [:gt, :eq] and
-      Hex.Version.compare(current, upper) == :lt
-  end
 end