Add BaGet.Protocol project; improve read-through caching (loic-shar…

…ma#124)

.azure/pipelines/ci-official.yml

@@ -16,11 +16,11 @@ steps:
     custom: tool
     arguments: install --tool-path . nbgv
   displayName: Install NBGV tool
-  condition: ne(variables['system.pullrequest.isfork'], true)
+  condition: eq(variables['build.sourcebranch'], 'refs/heads/master')
 
 - script: nbgv cloud
   displayName: Set Version
-  condition: ne(variables['system.pullrequest.isfork'], true)
+  condition: eq(variables['build.sourcebranch'], 'refs/heads/master')
 
 - script: dotnet build --configuration $(BuildConfiguration)
   displayName: Build
@@ -34,7 +34,7 @@ steps:
 
 - script: dotnet pack --configuration $(BuildConfiguration) --output $(Build.ArtifactStagingDirectory)
   displayName: Pack
-  condition: ne(variables['system.pullrequest.isfork'], true)
+  condition: eq(variables['build.sourcebranch'], 'refs/heads/master')
 
 - task: DotNetCoreCLI@2
   displayName: Publish
@@ -43,9 +43,10 @@ steps:
     publishWebProjects: True
     arguments: '--configuration $(BuildConfiguration) --output $(Build.ArtifactStagingDirectory)'
     zipAfterPublish: True
+  condition: eq(variables['build.sourcebranch'], 'refs/heads/master')
 
 - task: PublishBuildArtifacts@1
   displayName: 'Publish Artifacts'
   inputs:
     PathtoPublish: '$(Build.ArtifactStagingDirectory)'
-  condition: ne(variables['system.pullrequest.isfork'], true)
+  condition: eq(variables['build.sourcebranch'], 'refs/heads/master')

BaGet.sln

@@ -19,6 +19,10 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{26A0B557-53F
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tests", "tests", "{C237857D-AD8E-4C52-974F-6A8155BB0C18}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BaGet.Protocol", "src\BaGet.Protocol\BaGet.Protocol.csproj", "{A2D23427-9278-4D52-B31F-759212252832}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BaGet.Protocol.Tests", "tests\BaGet.Protocol.Tests\BaGet.Protocol.Tests.csproj", "{AC764A9A-9EAF-422B-9223-D3290C3CFD79}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -49,6 +53,14 @@ Global
 		{892A7A82-4283-4315-B7E5-6D5B70543000}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{892A7A82-4283-4315-B7E5-6D5B70543000}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{892A7A82-4283-4315-B7E5-6D5B70543000}.Release|Any CPU.Build.0 = Release|Any CPU
+		{A2D23427-9278-4D52-B31F-759212252832}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{A2D23427-9278-4D52-B31F-759212252832}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{A2D23427-9278-4D52-B31F-759212252832}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{A2D23427-9278-4D52-B31F-759212252832}.Release|Any CPU.Build.0 = Release|Any CPU
+		{AC764A9A-9EAF-422B-9223-D3290C3CFD79}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{AC764A9A-9EAF-422B-9223-D3290C3CFD79}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{AC764A9A-9EAF-422B-9223-D3290C3CFD79}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{AC764A9A-9EAF-422B-9223-D3290C3CFD79}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -60,6 +72,8 @@ Global
 		{B232DAFE-5CE8-441F-ACC7-2BB54BCD094F} = {26A0B557-53FB-4B9A-94C4-BCCF1BDCB0CC}
 		{89AB1AE2-6CAA-4809-8B74-D78CBE00B049} = {C237857D-AD8E-4C52-974F-6A8155BB0C18}
 		{892A7A82-4283-4315-B7E5-6D5B70543000} = {C237857D-AD8E-4C52-974F-6A8155BB0C18}
+		{A2D23427-9278-4D52-B31F-759212252832} = {26A0B557-53FB-4B9A-94C4-BCCF1BDCB0CC}
+		{AC764A9A-9EAF-422B-9223-D3290C3CFD79} = {C237857D-AD8E-4C52-974F-6A8155BB0C18}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {1423C027-2C90-417F-8629-2A4CF107C055}

docs/configuration.md

@@ -40,16 +40,15 @@ The following `Mirror` settings configures BaGet to index packages from [nuget.o
 
     "Mirror": {
         "Enabled":  true,
-        "PackageSource": "https://api.nuget.org/v3-flatcontainer/"
+        "PackageSource": "https://api.nuget.org/v3/index.json/"
     },
 
     ...
 }
 ```
 
 !!! info
-    `PackageSource` is the value of the [`PackageBaseAddress`](https://docs.microsoft.com/en-us/nuget/api/overview#resources-and-schema) resource
-    on a [NuGet service index](https://docs.microsoft.com/en-us/nuget/api/service-index).
+    `PackageSource` is the value of the [NuGet service index](https://docs.microsoft.com/en-us/nuget/api/service-index).
 
 ## Enabling Package Hard Deletions
 

src/BaGet.Azure/Search/AzureSearchService.cs

@@ -58,7 +58,7 @@ public async Task<IReadOnlyList<SearchResult>> SearchAsync(string query, int ski
                     Id = document.Id,
                     Version = NuGetVersion.Parse(document.Version),
                     Description = document.Description,
-                    Authors = string.Join(",", document.Authors),
+                    Authors = document.Authors,
                     IconUrl = document.IconUrl,
                     LicenseUrl = document.LicenseUrl,
                     Summary = document.Summary,

src/BaGet.Core/BaGet.Core.csproj

@@ -11,4 +11,8 @@
     <PackageReference Include="NuGet.Packaging" Version="4.7.0" />
   </ItemGroup>
 
+  <ItemGroup>
+    <ProjectReference Include="..\BaGet.Protocol\BaGet.Protocol.csproj" />
+  </ItemGroup>
+
 </Project>

src/BaGet.Core/Mirror/FakeMirrorService.cs

@@ -9,10 +9,7 @@ namespace BaGet.Core.Mirror
     /// </summary>
     public class FakeMirrorService : IMirrorService
     {
-        public Task MirrorAsync(
-            string id,
-            NuGetVersion version,
-            CancellationToken cancellationToken)
+        public Task MirrorAsync(string id, CancellationToken cancellationToken)
         {
             return Task.CompletedTask;
         }

src/BaGet.Core/Mirror/IMirrorService.cs

@@ -1,6 +1,5 @@
 using System.Threading;
 using System.Threading.Tasks;
-using NuGet.Versioning;
 
 namespace BaGet.Core.Mirror
 {
@@ -13,9 +12,8 @@ public interface IMirrorService
         /// If the package is unknown, attempt to index it from an upstream source.
         /// </summary>
         /// <param name="id">The package's id</param>
-        /// <param name="version">The package's version</param>
         /// <param name="cancellationToken">The token to cancel the mirroring</param>
         /// <returns>A task that completes when the package has been mirrored.</returns>
-        Task MirrorAsync(string id, NuGetVersion version, CancellationToken cancellationToken);
+        Task MirrorAsync(string id, CancellationToken cancellationToken);
     }
 }

src/BaGet.Core/Mirror/MirrorService.cs

@@ -1,85 +1,98 @@
 using System;
+using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
 using BaGet.Core.Services;
+using BaGet.Protocol;
 using Microsoft.Extensions.Logging;
 using NuGet.Versioning;
 
 namespace BaGet.Core.Mirror
 {
     public class MirrorService : IMirrorService
     {
-        private readonly Uri _packageBaseAddress;
         private readonly IPackageService _localPackages;
+        private readonly IPackageMetadataService _upstreamFeed;
         private readonly IPackageDownloader _downloader;
         private readonly IIndexingService _indexer;
         private readonly ILogger<MirrorService> _logger;
 
         public MirrorService(
-            Uri packageBaseAddress,
             IPackageService localPackages,
+            IPackageMetadataService upstreamFeed,
             IPackageDownloader downloader,
             IIndexingService indexer,
             ILogger<MirrorService> logger)
         {
-            _packageBaseAddress = packageBaseAddress ?? throw new ArgumentNullException(nameof(packageBaseAddress));
             _localPackages = localPackages ?? throw new ArgumentNullException(nameof(localPackages));
+            _upstreamFeed = upstreamFeed ?? throw new ArgumentNullException(nameof(upstreamFeed));
             _downloader = downloader ?? throw new ArgumentNullException(nameof(downloader));
             _indexer = indexer ?? throw new ArgumentNullException(nameof(indexer));
             _logger = logger ?? throw new ArgumentNullException(nameof(logger));
         }
 
-        public async Task MirrorAsync(string id, NuGetVersion version, CancellationToken cancellationToken)
+        public async Task MirrorAsync(string id, CancellationToken cancellationToken)
         {
-            if (await _localPackages.ExistsAsync(id, version))
+            if (await _localPackages.ExistsAsync(id))
             {
                 return;
             }
 
-            var idString = id.ToLowerInvariant();
-            var versionString = version.ToNormalizedString().ToLowerInvariant();
+            _logger.LogInformation("Package {PackageId} does not exist locally. Mirroring...", id);
 
-            await IndexFromSourceAsync(idString, versionString, cancellationToken);
+            var versions = await _upstreamFeed.GetAllVersionsAsync(id, includeUnlisted: true);
+
+            _logger.LogInformation(
+                "Found {VersionsCount} versions for package {PackageId} on upstream feed. Indexing...",
+                versions.Count,
+                id);
+
+            // TODO: This will synchronously index packages one-by-one. This won't perform well
+            // for packages with many versions. Instead, this should index a few packages and
+            // let a background queue index the rest.
+            // See: https://docs.microsoft.com/en-us/aspnet/core/fundamentals/host/hosted-services?view=aspnetcore-2.1#queued-background-tasks
+            foreach (var version in versions)
+            {
+                var packageUri = await _upstreamFeed.GetPackageContentUriAsync(id, version);
+
+                await IndexFromSourceAsync(packageUri, cancellationToken);
+            }
+
+            _logger.LogInformation("Finished indexing {PackageId} from the upstream feed", id);
         }
 
-        private async Task IndexFromSourceAsync(string id, string version, CancellationToken cancellationToken)
+        private async Task IndexFromSourceAsync(Uri packageUri, CancellationToken cancellationToken)
         {
             cancellationToken.ThrowIfCancellationRequested();
 
-            _logger.LogInformation("Attempting to mirror package {Id} {Version}...", id, version);
+            _logger.LogInformation("Attempting to mirror package {PackageUri}...", packageUri);
 
             try
             {
-                // See https://github.com/NuGet/NuGet.Client/blob/4eed67e7e159796ae486d2cca406b283e23b6ac8/src/NuGet.Core/NuGet.Protocol/Resources/DownloadResourceV3.cs#L82
-                var packageUri = new Uri(_packageBaseAddress, $"{id}/{version}/{id}.{version}.nupkg");
-
                 using (var stream = await _downloader.DownloadOrNullAsync(packageUri, cancellationToken))
                 {
                     if (stream == null)
                     {
                         _logger.LogWarning(
-                            "Failed to download package {Id} {Version} at {PackageUri}",
-                            id,
-                            version,
+                            "Failed to download package at {PackageUri}",
                             packageUri);
 
                         return;
                     }
 
-                    _logger.LogInformation("Downloaded package {Id} {Version}, indexing...", id, version);
+                    _logger.LogInformation("Downloaded package at {PackageUri}, indexing...", packageUri);
 
                     var result = await _indexer.IndexAsync(stream, cancellationToken);
 
                     _logger.LogInformation(
-                        "Finished indexing package {Id} {Version} with result {Result}",
-                        id,
-                        version,
+                        "Finished indexing package at {PackageUri} with result {Result}",
+                        packageUri,
                         result);
                 }
             }
             catch (Exception e)
             {
-                _logger.LogError(e, "Failed to mirror package {Id} {Version}", id, version);
+                _logger.LogError(e, "Failed to mirror package at {PackageUri}", packageUri);
             }
         }
     }

src/BaGet.Core/Services/DatabaseSearchService.cs

@@ -53,7 +53,7 @@ public async Task<IReadOnlyList<SearchResult>> SearchAsync(string query, int ski
                     Id = latest.Id,
                     Version = latest.Version,
                     Description = latest.Description,
-                    Authors = string.Join(", ", latest.Authors),
+                    Authors = latest.Authors,
                     IconUrl = latest.IconUrlString,
                     LicenseUrl = latest.LicenseUrlString,
                     ProjectUrl = latest.ProjectUrlString,

src/BaGet.Core/Services/IPackageService.cs

@@ -58,7 +58,7 @@ public interface IPackageService
         /// <param name="id">The package id to search.</param>
         /// <param name="version">The package version to search.</param>
         /// <returns>Whether the package exists in the database.</returns>
-        Task<bool> ExistsAsync(string id, NuGetVersion version);
+        Task<bool> ExistsAsync(string id, NuGetVersion version = null);
 
         /// <summary>
         /// Unlist a package, making it undiscoverable.

src/BaGet.Core/Services/ISearchService.cs

@@ -22,7 +22,7 @@ public class SearchResult
         public NuGetVersion Version { get; set; }
 
         public string Description { get; set; }
-        public string Authors { get; set; }
+        public IReadOnlyList<string> Authors { get; set; }
         public string IconUrl { get; set; }
         public string LicenseUrl { get; set; }
         public string ProjectUrl { get; set; }

src/BaGet.Core/Services/PackageService.cs

@@ -34,11 +34,17 @@ public async Task<PackageAddResult> AddAsync(Package package)
             }
         }
 
-        public Task<bool> ExistsAsync(string id, NuGetVersion version)
-            => _context.Packages
-                .Where(p => p.Id == id)
-                .Where(p => p.VersionString == version.ToNormalizedString())
-                .AnyAsync();
+        public Task<bool> ExistsAsync(string id, NuGetVersion version = null)
+        {
+            var query = _context.Packages.Where(p => p.Id == id);
+
+            if (version != null)
+            {
+                query = query.Where(p => p.VersionString == version.ToNormalizedString());
+            }
+
+            return query.AnyAsync();
+        }
 
         public async Task<IReadOnlyList<Package>> FindAsync(string id, bool includeUnlisted = false)
         {

src/BaGet.Protocol/BaGet.Protocol.csproj

@@ -0,0 +1,14 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>netstandard2.0</TargetFramework>
+
+    <Details>Libraries to interact with NuGet server APIs</Details>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Newtonsoft.Json" Version="11.0.2" />
+    <PackageReference Include="NuGet.Versioning" Version="4.8.0" />
+  </ItemGroup>
+
+</Project>

src/BaGet.Protocol/Converters/NuGetVersionConverter.cs

@@ -0,0 +1,25 @@
+using System;
+using Newtonsoft.Json;
+using NuGet.Versioning;
+
+namespace BaGet.Protocol.Converters
+{
+    public class NuGetVersionConverter : JsonConverter
+    {
+        public override bool CanConvert(Type objectType)
+        {
+            return objectType == typeof(NuGetVersion);
+        }
+
+        public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer)
+        {
+            var version = (NuGetVersion)value;
+            serializer.Serialize(writer, version.ToNormalizedString());
+        }
+
+        public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)
+        {
+            return reader.TokenType != JsonToken.Null ? NuGetVersion.Parse(serializer.Deserialize<string>(reader)) : null;
+        }
+    }
+}

src/BaGet.Protocol/Converters/NuGetVersionListConverter.cs

@@ -0,0 +1,30 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using Newtonsoft.Json;
+using NuGet.Versioning;
+
+namespace BaGet.Protocol.Converters
+{
+    public class NuGetVersionListConverter : JsonConverter
+    {
+        public override bool CanConvert(Type objectType)
+        {
+            return objectType == typeof(IReadOnlyList<NuGetVersion>);
+        }
+
+        public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer)
+        {
+            var versions = ((IReadOnlyList<NuGetVersion>)value);
+
+            serializer.Serialize(writer, versions.Select(v => v.ToString()));
+        }
+
+        public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)
+        {
+            return serializer.Deserialize<IReadOnlyList<string>>(reader)
+                .Select(NuGetVersion.Parse)
+                .ToList();
+        }
+    }
+}