[vcpkg] Implement VersionedPortfileProvider and BaselineProvider (mic…

…rosoft#14123) * WIP: Get versions from database files * Fix formatting * Provider inherits ResourceBase * Correct versions JSON file location * Fix formatting * Fix formatting * Fix include in versions.h * Fetch port versions using git tree object * Undo changes to x-history * Remove unnecesary moves Co-authored-by: nicole mazzuca <[email protected]> * Extract Git manipulation code * [WIP] Review comments * [WIP] Review comments pt. 2 * [WIP] Review comments / fix formatting * Generate baseline.json * Extract deserializers from registries source file * BaselineProvider initial implementation * Modify gitignore * Update .gitignore again * Use JSON deserializer for versions db * Lazy load baseline file * Fetch baseline.json from baseline commit * More git abstractions * Clean up code * Path helpers * Formatting * Move data into impl object * Use implementation object for VersionedPortfileProvider * Reuse cloned instance for checkouts * Code cleanup and formatting * Fix returning dangling reference * Prepare to remove files in port_versions/ * Remove files in port_versions/ * Update .gitignore * Some PR review comments * Use StringView * More StringView conversions * More refactoring * Make some implementation members private * Functions for parsing baseline and version files * Hide deserializers implementation * Check for `versions` feature flag in registries. Co-authored-by: Robert Schumacher <[email protected]> Co-authored-by: nicole mazzuca <[email protected]>
Thelta · Nov 27, 2020 · 6c9cda1 · 6c9cda1
1 parent 62fe6ff
commit 6c9cda1
Show file tree

Hide file tree

Showing 13 changed files with 832 additions and 87 deletions.
diff --git a/.gitignore b/.gitignore
@@ -295,6 +295,8 @@ __pycache__/
 /toolsrc/windows-bootstrap/msbuild.x86.release/
 /toolsrc/windows-bootstrap/msbuild.x64.debug/
 /toolsrc/windows-bootstrap/msbuild.x64.release/
+#ignore db
+/port_versions/
 #ignore custom triplets
 /triplets/*
 #add vcpkg-designed triplets back in

diff --git a/scripts/generatePortVersionsDb.py b/scripts/generatePortVersionsDb.py
@@ -4,6 +4,7 @@
 import subprocess
 import json
 import time
+import shutil
 
 from subprocess import CalledProcessError
 from json.decoder import JSONDecodeError
@@ -14,9 +15,9 @@
 
 
 def get_current_git_ref():
-    output = subprocess.run(['git.exe', '-C', SCRIPT_DIRECTORY, 'rev-parse', '--verify', 'HEAD'], 
-        capture_output=True, 
-        encoding='utf-8')
+    output = subprocess.run(['git', '-C', SCRIPT_DIRECTORY, 'rev-parse', '--verify', 'HEAD'],
+                            capture_output=True,
+                            encoding='utf-8')
     if output.returncode == 0:
         return output.stdout.strip()
     print(f"Failed to get git ref:", output.stderr.strip(), file=sys.stderr)
@@ -25,47 +26,89 @@ def get_current_git_ref():
 
 def generate_port_versions_db(ports_path, db_path, revision):
     start_time = time.time()
-    port_names = [item for item in os.listdir(ports_path) if os.path.isdir(os.path.join(ports_path, item))]
+
+    # Assume each directory in ${VCPKG_ROOT}/ports is a different port
+    port_names = [item for item in os.listdir(
+        ports_path) if os.path.isdir(os.path.join(ports_path, item))]
+    port_names.sort()
     total_count = len(port_names)
+
+    # Dictionary to collect the latest version of each port as baseline
+    baseline_objects = {}
+    baseline_objects['default'] = {}
+
     for counter, port_name in enumerate(port_names):
         containing_dir = os.path.join(db_path, f'{port_name[0]}-')
         os.makedirs(containing_dir, exist_ok=True)
+
         output_filepath = os.path.join(containing_dir, f'{port_name}.json')
-        if not os.path.exists(output_filepath):            
+        if not os.path.exists(output_filepath):
             output = subprocess.run(
-                [os.path.join(SCRIPT_DIRECTORY, '../vcpkg'), 'x-history', port_name, '--x-json'], 
+                [os.path.join(SCRIPT_DIRECTORY, '../vcpkg'),
+                 'x-history', port_name, '--x-json'],
                 capture_output=True, encoding='utf-8')
+
             if output.returncode == 0:
                 try:
                     versions_object = json.loads(output.stdout)
+
+                    # Put latest version in baseline dictionary
+                    latest_version = versions_object["versions"][0]
+                    baseline_objects['default'][port_name] = {
+                        "version-string": latest_version["version-string"],
+                        "port-version": latest_version["port-version"]
+                    }
                     with open(output_filepath, 'w') as output_file:
                         json.dump(versions_object, output_file)
                 except JSONDecodeError:
-                    print(f'Maformed JSON from vcpkg x-history {port_name}: ', output.stdout.strip(), file=sys.stderr)
+                    print(
+                        f'Malformed JSON from vcpkg x-history {port_name}: ', output.stdout.strip(), file=sys.stderr)
             else:
-                print(f'x-history {port_name} failed: ', output.stdout.strip(), file=sys.stderr)
+                print(f'x-history {port_name} failed: ',
+                      output.stdout.strip(), file=sys.stderr)
+
         # This should be replaced by a progress bar
         if counter > 0 and counter % 100 == 0:
             elapsed_time = time.time() - start_time
-            print(f'Processed {counter} out of {total_count}. Elapsed time: {elapsed_time:.2f} seconds')
+            print(
+                f'Processed {counter} out of {total_count}. Elapsed time: {elapsed_time:.2f} seconds')
+
+    # Generate baseline.json
+    baseline_file_path = os.path.join(db_path, 'baseline.json')
+    with open(baseline_file_path, 'w') as baseline_output_file:
+        json.dump(baseline_objects, baseline_output_file)
+
+    # Generate timestamp
     rev_file = os.path.join(db_path, revision)
     Path(rev_file).touch()
+
     elapsed_time = time.time() - start_time
-    print(f'Processed {total_count} total ports. Elapsed time: {elapsed_time:.2f} seconds')
-
+    print(
+        f'Processed {total_count} total ports. Elapsed time: {elapsed_time:.2f} seconds')
+
 
 def main(ports_path, db_path):
     revision = get_current_git_ref()
     if not revision:
         print('Couldn\'t fetch current Git revision', file=sys.stderr)
         sys.exit(1)
+
     rev_file = os.path.join(db_path, revision)
     if os.path.exists(rev_file):
         print(f'Database files already exist for commit {revision}')
         sys.exit(0)
-    generate_port_versions_db(ports_path=ports_path, db_path=db_path, revision=revision)
+
+    if (os.path.exists(db_path)):
+        try:
+            shutil.rmtree(db_path)
+        except OSError as e:
+            print(f'Could not delete folder: {db_path}.\nError: {e.strerror}')
+
+    generate_port_versions_db(ports_path=ports_path,
+                              db_path=db_path,
+                              revision=revision)
 
 
 if __name__ == "__main__":
-    main(ports_path=os.path.join(SCRIPT_DIRECTORY, '../ports'), 
-        db_path=os.path.join(SCRIPT_DIRECTORY, '../port_versions'))
+    main(ports_path=os.path.join(SCRIPT_DIRECTORY, '../ports'),
+         db_path=os.path.join(SCRIPT_DIRECTORY, '../port_versions'))
diff --git a/toolsrc/include/vcpkg/base/jsonreader.h b/toolsrc/include/vcpkg/base/jsonreader.h
@@ -23,7 +23,7 @@ namespace vcpkg::Json
         Optional<Type> visit(Reader&, const Value&);
         Optional<Type> visit(Reader&, const Object&);
 
-    protected:
+    public:
         virtual Optional<Type> visit_null(Reader&);
         virtual Optional<Type> visit_boolean(Reader&, bool);
         virtual Optional<Type> visit_integer(Reader& r, int64_t i);
@@ -33,6 +33,7 @@ namespace vcpkg::Json
         virtual Optional<Type> visit_object(Reader&, const Object&);
         virtual View<StringView> valid_fields() const;
 
+    protected:
         IDeserializer() = default;
         IDeserializer(const IDeserializer&) = default;
         IDeserializer& operator=(const IDeserializer&) = default;

diff --git a/toolsrc/include/vcpkg/portfileprovider.h b/toolsrc/include/vcpkg/portfileprovider.h
@@ -6,6 +6,7 @@
 #include <vcpkg/base/util.h>
 
 #include <vcpkg/sourceparagraph.h>
+#include <vcpkg/versions.h>
 
 namespace vcpkg::PortFileProvider
 {
@@ -36,4 +37,48 @@ namespace vcpkg::PortFileProvider
         std::vector<fs::path> overlay_ports;
         mutable std::unordered_map<std::string, SourceControlFileLocation> cache;
     };
+
+    struct IVersionedPortfileProvider
+    {
+        virtual const std::vector<vcpkg::Versions::VersionSpec>& get_port_versions(StringView port_name) const = 0;
+
+        virtual ExpectedS<const SourceControlFileLocation&> get_control_file(
+            const vcpkg::Versions::VersionSpec& version_spec) const = 0;
+    };
+
+    struct IBaselineProvider
+    {
+        virtual Optional<VersionT> get_baseline_version(StringView port_name) const = 0;
+    };
+
+    namespace details
+    {
+        struct BaselineProviderImpl;
+        struct VersionedPortfileProviderImpl;
+    }
+
+    struct VersionedPortfileProvider : IVersionedPortfileProvider, Util::ResourceBase
+    {
+        explicit VersionedPortfileProvider(const vcpkg::VcpkgPaths& paths);
+        ~VersionedPortfileProvider();
+
+        const std::vector<vcpkg::Versions::VersionSpec>& get_port_versions(StringView port_name) const override;
+
+        ExpectedS<const SourceControlFileLocation&> get_control_file(
+            const vcpkg::Versions::VersionSpec& version_spec) const override;
+
+    private:
+        std::unique_ptr<details::VersionedPortfileProviderImpl> m_impl;
+    };
+
+    struct BaselineProvider : IBaselineProvider, Util::ResourceBase
+    {
+        explicit BaselineProvider(const vcpkg::VcpkgPaths& paths, const std::string& baseline);
+        ~BaselineProvider();
+
+        Optional<VersionT> get_baseline_version(StringView port_name) const override;
+
+    private:
+        std::unique_ptr<details::BaselineProviderImpl> m_impl;
+    };
 }
diff --git a/toolsrc/include/vcpkg/vcpkgpaths.h b/toolsrc/include/vcpkg/vcpkgpaths.h
@@ -102,11 +102,23 @@ namespace vcpkg
         fs::path vcpkg_dir_info;
         fs::path vcpkg_dir_updates;
 
+        fs::path baselines_dot_git_dir;
+        fs::path baselines_work_tree;
+        fs::path baselines_output;
+
+        fs::path versions_dot_git_dir;
+        fs::path versions_work_tree;
+        fs::path versions_output;
+
         fs::path ports_cmake;
 
         const fs::path& get_tool_exe(const std::string& tool) const;
         const std::string& get_tool_version(const std::string& tool) const;
 
+        // Git manipulation
+        fs::path git_checkout_baseline(Files::Filesystem& filesystem, StringView commit_sha) const;
+        fs::path git_checkout_port(Files::Filesystem& filesystem, StringView port_name, StringView git_tree) const;
+
         Optional<const Json::Object&> get_manifest() const;
         Optional<const fs::path&> get_manifest_path() const;
         const Configuration& get_configuration() const;
@@ -133,5 +145,20 @@ namespace vcpkg
 
     private:
         std::unique_ptr<details::VcpkgPathsImpl> m_pimpl;
+
+        static void git_checkout_subpath(const VcpkgPaths& paths,
+                                         StringView commit_sha,
+                                         const fs::path& subpath,
+                                         const fs::path& local_repo,
+                                         const fs::path& destination,
+                                         const fs::path& dot_git_dir,
+                                         const fs::path& work_tree);
+
+        static void git_checkout_object(const VcpkgPaths& paths,
+                                        StringView git_object,
+                                        const fs::path& local_repo,
+                                        const fs::path& destination,
+                                        const fs::path& dot_git_dir,
+                                        const fs::path& work_tree);
     };
 }
diff --git a/toolsrc/include/vcpkg/versiondeserializers.h b/toolsrc/include/vcpkg/versiondeserializers.h
@@ -0,0 +1,37 @@
+#pragma once
+
+#include <vcpkg/base/fwd/stringview.h>
+
+#include <vcpkg/base/jsonreader.h>
+#include <vcpkg/base/stringliteral.h>
+
+#include <vcpkg/versions.h>
+#include <vcpkg/versiont.h>
+
+namespace vcpkg
+{
+    struct VersionDbEntry
+    {
+        VersionT version;
+        Versions::Scheme scheme;
+        std::string git_tree;
+
+        VersionDbEntry(const std::string& version_string,
+                       int port_version,
+                       Versions::Scheme scheme,
+                       const std::string& git_tree)
+            : version(VersionT(version_string, port_version)), scheme(scheme), git_tree(git_tree)
+        {
+        }
+    };
+
+    Json::IDeserializer<VersionT>& get_versiont_deserializer_instance();
+
+    ExpectedS<std::map<std::string, VersionT, std::less<>>> parse_baseline_file(Files::Filesystem& fs,
+                                                                                StringView baseline_name,
+                                                                                const fs::path& baseline_file_path);
+
+    ExpectedS<std::vector<VersionDbEntry>> parse_versions_file(Files::Filesystem& fs,
+                                                               StringView port_name,
+                                                               const fs::path& versions_file_path);
+}
diff --git a/toolsrc/include/vcpkg/versions.h b/toolsrc/include/vcpkg/versions.h
@@ -1,5 +1,7 @@
 #pragma once
 
+#include <vcpkg/versiont.h>
+
 namespace vcpkg::Versions
 {
     enum class Scheme
@@ -10,6 +12,25 @@ namespace vcpkg::Versions
         String
     };
 
+    struct VersionSpec
+    {
+        std::string port_name;
+        VersionT version;
+        Scheme scheme;
+
+        VersionSpec(const std::string& port_name, const VersionT& version, Scheme scheme);
+
+        VersionSpec(const std::string& port_name, const std::string& version_string, int port_version, Scheme scheme);
+
+        friend bool operator==(const VersionSpec& lhs, const VersionSpec& rhs);
+        friend bool operator!=(const VersionSpec& lhs, const VersionSpec& rhs);
+    };
+
+    struct VersionSpecHasher
+    {
+        std::size_t operator()(const VersionSpec& key) const;
+    };
+
     struct Constraint
     {
         enum class Type