From 43c63639ea20cc357e79e1c6c963021bf9e75d14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Sharma?= Date: Sun, 11 Nov 2018 23:59:01 -0800 Subject: [PATCH] Add `BaGet.Protocol` project; improve read-through caching (#124) --- .azure/pipelines/ci-official.yml | 9 +- BaGet.sln | 14 + docs/configuration.md | 5 +- src/BaGet.Azure/Search/AzureSearchService.cs | 2 +- src/BaGet.Core/BaGet.Core.csproj | 4 + src/BaGet.Core/Mirror/FakeMirrorService.cs | 5 +- src/BaGet.Core/Mirror/IMirrorService.cs | 4 +- src/BaGet.Core/Mirror/MirrorService.cs | 55 ++-- .../Services/DatabaseSearchService.cs | 2 +- src/BaGet.Core/Services/IPackageService.cs | 2 +- src/BaGet.Core/Services/ISearchService.cs | 2 +- src/BaGet.Core/Services/PackageService.cs | 16 +- src/BaGet.Protocol/BaGet.Protocol.csproj | 14 + .../Converters/NuGetVersionConverter.cs | 25 ++ .../Converters/NuGetVersionListConverter.cs | 30 +++ .../Converters/SingleOrListConverter.cs | 31 +++ src/BaGet.Protocol/HttpContentExtensions.cs | 29 ++ .../PackageContent/IPackageContentClient.cs | 14 + .../PackageContent/PackageContentClient.cs | 54 ++++ .../PackageContent/PackageVersions.cs | 24 ++ .../Registration/IRegistrationClient.cs | 13 + .../Registration/RegistrationClient.cs | 43 +++ .../Registration/RegistrationIndex.cs | 35 +++ .../Registration/RegistrationIndexPage.cs | 51 ++++ .../Registration/RegistrationIndexPageItem.cs | 153 +++++++++++ .../Registration/RegistrationLeaf.cs | 43 +++ .../Search/AutocompleteResult.cs | 28 ++ src/BaGet.Protocol/Search/ISearchClient.cs | 11 + src/BaGet.Protocol/Search/SearchClient.cs | 34 +++ src/BaGet.Protocol/Search/SearchResponse.cs | 28 ++ src/BaGet.Protocol/Search/SearchResult.cs | 76 ++++++ .../Search/SearchResultVersion.cs | 37 +++ .../ServiceIndex/IServiceIndexClient.cs | 9 + .../ServiceIndex/ServiceIndex.cs | 33 +++ .../ServiceIndex/ServiceIndexClient.cs | 28 ++ .../ServiceIndex/ServiceIndexResource.cs | 36 +++ .../Services/IPackageMetadataService.cs | 34 +++ .../Services/IServiceIndexService.cs | 13 + .../Services/PackageMetadataService.cs | 104 ++++++++ .../Services/ServiceIndexService.cs | 65 +++++ src/BaGet/BaGet.csproj | 1 + src/BaGet/Controllers/IndexController.cs | 50 +--- src/BaGet/Controllers/PackageController.cs | 14 +- .../RegistrationIndexController.cs | 249 +++++------------- .../RegistrationLeafController.cs | 44 +--- src/BaGet/Controllers/SearchController.cs | 97 +++---- .../Extensions/ServiceCollectionExtensions.cs | 34 ++- src/BaGet/appsettings.json | 2 +- src/readme.md | 3 +- .../BaGet.Protocol.Tests.csproj | 22 ++ .../PackageContentClientTests.cs | 35 +++ .../PackageMetadataServiceIntegrationTests.cs | 45 ++++ .../RegistrationClientTests.cs | 85 ++++++ .../BaGet.Protocol.Tests/SearchClientTests.cs | 45 ++++ .../ServiceIndexClientTests.cs | 23 ++ 55 files changed, 1575 insertions(+), 389 deletions(-) create mode 100644 src/BaGet.Protocol/BaGet.Protocol.csproj create mode 100644 src/BaGet.Protocol/Converters/NuGetVersionConverter.cs create mode 100644 src/BaGet.Protocol/Converters/NuGetVersionListConverter.cs create mode 100644 src/BaGet.Protocol/Converters/SingleOrListConverter.cs create mode 100644 src/BaGet.Protocol/HttpContentExtensions.cs create mode 100644 src/BaGet.Protocol/PackageContent/IPackageContentClient.cs create mode 100644 src/BaGet.Protocol/PackageContent/PackageContentClient.cs create mode 100644 src/BaGet.Protocol/PackageContent/PackageVersions.cs create mode 100644 src/BaGet.Protocol/Registration/IRegistrationClient.cs create mode 100644 src/BaGet.Protocol/Registration/RegistrationClient.cs create mode 100644 src/BaGet.Protocol/Registration/RegistrationIndex.cs create mode 100644 src/BaGet.Protocol/Registration/RegistrationIndexPage.cs create mode 100644 src/BaGet.Protocol/Registration/RegistrationIndexPageItem.cs create mode 100644 src/BaGet.Protocol/Registration/RegistrationLeaf.cs create mode 100644 src/BaGet.Protocol/Search/AutocompleteResult.cs create mode 100644 src/BaGet.Protocol/Search/ISearchClient.cs create mode 100644 src/BaGet.Protocol/Search/SearchClient.cs create mode 100644 src/BaGet.Protocol/Search/SearchResponse.cs create mode 100644 src/BaGet.Protocol/Search/SearchResult.cs create mode 100644 src/BaGet.Protocol/Search/SearchResultVersion.cs create mode 100644 src/BaGet.Protocol/ServiceIndex/IServiceIndexClient.cs create mode 100644 src/BaGet.Protocol/ServiceIndex/ServiceIndex.cs create mode 100644 src/BaGet.Protocol/ServiceIndex/ServiceIndexClient.cs create mode 100644 src/BaGet.Protocol/ServiceIndex/ServiceIndexResource.cs create mode 100644 src/BaGet.Protocol/Services/IPackageMetadataService.cs create mode 100644 src/BaGet.Protocol/Services/IServiceIndexService.cs create mode 100644 src/BaGet.Protocol/Services/PackageMetadataService.cs create mode 100644 src/BaGet.Protocol/Services/ServiceIndexService.cs create mode 100644 tests/BaGet.Protocol.Tests/BaGet.Protocol.Tests.csproj create mode 100644 tests/BaGet.Protocol.Tests/PackageContentClientTests.cs create mode 100644 tests/BaGet.Protocol.Tests/PackageMetadataServiceIntegrationTests.cs create mode 100644 tests/BaGet.Protocol.Tests/RegistrationClientTests.cs create mode 100644 tests/BaGet.Protocol.Tests/SearchClientTests.cs create mode 100644 tests/BaGet.Protocol.Tests/ServiceIndexClientTests.cs diff --git a/.azure/pipelines/ci-official.yml b/.azure/pipelines/ci-official.yml index de72fafbc..3a3bcb233 100644 --- a/.azure/pipelines/ci-official.yml +++ b/.azure/pipelines/ci-official.yml @@ -16,11 +16,11 @@ steps: custom: tool arguments: install --tool-path . nbgv displayName: Install NBGV tool - condition: ne(variables['system.pullrequest.isfork'], true) + condition: eq(variables['build.sourcebranch'], 'refs/heads/master') - script: nbgv cloud displayName: Set Version - condition: ne(variables['system.pullrequest.isfork'], true) + condition: eq(variables['build.sourcebranch'], 'refs/heads/master') - script: dotnet build --configuration $(BuildConfiguration) displayName: Build @@ -34,7 +34,7 @@ steps: - script: dotnet pack --configuration $(BuildConfiguration) --output $(Build.ArtifactStagingDirectory) displayName: Pack - condition: ne(variables['system.pullrequest.isfork'], true) + condition: eq(variables['build.sourcebranch'], 'refs/heads/master') - task: DotNetCoreCLI@2 displayName: Publish @@ -43,9 +43,10 @@ steps: publishWebProjects: True arguments: '--configuration $(BuildConfiguration) --output $(Build.ArtifactStagingDirectory)' zipAfterPublish: True + condition: eq(variables['build.sourcebranch'], 'refs/heads/master') - task: PublishBuildArtifacts@1 displayName: 'Publish Artifacts' inputs: PathtoPublish: '$(Build.ArtifactStagingDirectory)' - condition: ne(variables['system.pullrequest.isfork'], true) + condition: eq(variables['build.sourcebranch'], 'refs/heads/master') diff --git a/BaGet.sln b/BaGet.sln index 0abe86d02..f904327ea 100644 --- a/BaGet.sln +++ b/BaGet.sln @@ -19,6 +19,10 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{26A0B557-53F EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tests", "tests", "{C237857D-AD8E-4C52-974F-6A8155BB0C18}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BaGet.Protocol", "src\BaGet.Protocol\BaGet.Protocol.csproj", "{A2D23427-9278-4D52-B31F-759212252832}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BaGet.Protocol.Tests", "tests\BaGet.Protocol.Tests\BaGet.Protocol.Tests.csproj", "{AC764A9A-9EAF-422B-9223-D3290C3CFD79}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -49,6 +53,14 @@ Global {892A7A82-4283-4315-B7E5-6D5B70543000}.Debug|Any CPU.Build.0 = Debug|Any CPU {892A7A82-4283-4315-B7E5-6D5B70543000}.Release|Any CPU.ActiveCfg = Release|Any CPU {892A7A82-4283-4315-B7E5-6D5B70543000}.Release|Any CPU.Build.0 = Release|Any CPU + {A2D23427-9278-4D52-B31F-759212252832}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {A2D23427-9278-4D52-B31F-759212252832}.Debug|Any CPU.Build.0 = Debug|Any CPU + {A2D23427-9278-4D52-B31F-759212252832}.Release|Any CPU.ActiveCfg = Release|Any CPU + {A2D23427-9278-4D52-B31F-759212252832}.Release|Any CPU.Build.0 = Release|Any CPU + {AC764A9A-9EAF-422B-9223-D3290C3CFD79}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {AC764A9A-9EAF-422B-9223-D3290C3CFD79}.Debug|Any CPU.Build.0 = Debug|Any CPU + {AC764A9A-9EAF-422B-9223-D3290C3CFD79}.Release|Any CPU.ActiveCfg = Release|Any CPU + {AC764A9A-9EAF-422B-9223-D3290C3CFD79}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -60,6 +72,8 @@ Global {B232DAFE-5CE8-441F-ACC7-2BB54BCD094F} = {26A0B557-53FB-4B9A-94C4-BCCF1BDCB0CC} {89AB1AE2-6CAA-4809-8B74-D78CBE00B049} = {C237857D-AD8E-4C52-974F-6A8155BB0C18} {892A7A82-4283-4315-B7E5-6D5B70543000} = {C237857D-AD8E-4C52-974F-6A8155BB0C18} + {A2D23427-9278-4D52-B31F-759212252832} = {26A0B557-53FB-4B9A-94C4-BCCF1BDCB0CC} + {AC764A9A-9EAF-422B-9223-D3290C3CFD79} = {C237857D-AD8E-4C52-974F-6A8155BB0C18} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {1423C027-2C90-417F-8629-2A4CF107C055} diff --git a/docs/configuration.md b/docs/configuration.md index 2b7a9dc4b..6e1602ecd 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -40,7 +40,7 @@ The following `Mirror` settings configures BaGet to index packages from [nuget.o "Mirror": { "Enabled": true, - "PackageSource": "https://api.nuget.org/v3-flatcontainer/" + "PackageSource": "https://api.nuget.org/v3/index.json/" }, ... @@ -48,8 +48,7 @@ The following `Mirror` settings configures BaGet to index packages from [nuget.o ``` !!! info - `PackageSource` is the value of the [`PackageBaseAddress`](https://docs.microsoft.com/en-us/nuget/api/overview#resources-and-schema) resource - on a [NuGet service index](https://docs.microsoft.com/en-us/nuget/api/service-index). + `PackageSource` is the value of the [NuGet service index](https://docs.microsoft.com/en-us/nuget/api/service-index). ## Enabling Package Hard Deletions diff --git a/src/BaGet.Azure/Search/AzureSearchService.cs b/src/BaGet.Azure/Search/AzureSearchService.cs index 75798f1b5..408101409 100644 --- a/src/BaGet.Azure/Search/AzureSearchService.cs +++ b/src/BaGet.Azure/Search/AzureSearchService.cs @@ -58,7 +58,7 @@ public async Task> SearchAsync(string query, int ski Id = document.Id, Version = NuGetVersion.Parse(document.Version), Description = document.Description, - Authors = string.Join(",", document.Authors), + Authors = document.Authors, IconUrl = document.IconUrl, LicenseUrl = document.LicenseUrl, Summary = document.Summary, diff --git a/src/BaGet.Core/BaGet.Core.csproj b/src/BaGet.Core/BaGet.Core.csproj index 281491f10..a970f485e 100644 --- a/src/BaGet.Core/BaGet.Core.csproj +++ b/src/BaGet.Core/BaGet.Core.csproj @@ -11,4 +11,8 @@ + + + + diff --git a/src/BaGet.Core/Mirror/FakeMirrorService.cs b/src/BaGet.Core/Mirror/FakeMirrorService.cs index 806161435..ee95a72a1 100644 --- a/src/BaGet.Core/Mirror/FakeMirrorService.cs +++ b/src/BaGet.Core/Mirror/FakeMirrorService.cs @@ -9,10 +9,7 @@ namespace BaGet.Core.Mirror /// public class FakeMirrorService : IMirrorService { - public Task MirrorAsync( - string id, - NuGetVersion version, - CancellationToken cancellationToken) + public Task MirrorAsync(string id, CancellationToken cancellationToken) { return Task.CompletedTask; } diff --git a/src/BaGet.Core/Mirror/IMirrorService.cs b/src/BaGet.Core/Mirror/IMirrorService.cs index 2c24eaf1e..8385c16c9 100644 --- a/src/BaGet.Core/Mirror/IMirrorService.cs +++ b/src/BaGet.Core/Mirror/IMirrorService.cs @@ -1,6 +1,5 @@ using System.Threading; using System.Threading.Tasks; -using NuGet.Versioning; namespace BaGet.Core.Mirror { @@ -13,9 +12,8 @@ public interface IMirrorService /// If the package is unknown, attempt to index it from an upstream source. /// /// The package's id - /// The package's version /// The token to cancel the mirroring /// A task that completes when the package has been mirrored. - Task MirrorAsync(string id, NuGetVersion version, CancellationToken cancellationToken); + Task MirrorAsync(string id, CancellationToken cancellationToken); } } diff --git a/src/BaGet.Core/Mirror/MirrorService.cs b/src/BaGet.Core/Mirror/MirrorService.cs index 1459b80f5..c4c500831 100644 --- a/src/BaGet.Core/Mirror/MirrorService.cs +++ b/src/BaGet.Core/Mirror/MirrorService.cs @@ -1,7 +1,9 @@ using System; +using System.Linq; using System.Threading; using System.Threading.Tasks; using BaGet.Core.Services; +using BaGet.Protocol; using Microsoft.Extensions.Logging; using NuGet.Versioning; @@ -9,77 +11,88 @@ namespace BaGet.Core.Mirror { public class MirrorService : IMirrorService { - private readonly Uri _packageBaseAddress; private readonly IPackageService _localPackages; + private readonly IPackageMetadataService _upstreamFeed; private readonly IPackageDownloader _downloader; private readonly IIndexingService _indexer; private readonly ILogger _logger; public MirrorService( - Uri packageBaseAddress, IPackageService localPackages, + IPackageMetadataService upstreamFeed, IPackageDownloader downloader, IIndexingService indexer, ILogger logger) { - _packageBaseAddress = packageBaseAddress ?? throw new ArgumentNullException(nameof(packageBaseAddress)); _localPackages = localPackages ?? throw new ArgumentNullException(nameof(localPackages)); + _upstreamFeed = upstreamFeed ?? throw new ArgumentNullException(nameof(upstreamFeed)); _downloader = downloader ?? throw new ArgumentNullException(nameof(downloader)); _indexer = indexer ?? throw new ArgumentNullException(nameof(indexer)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); } - public async Task MirrorAsync(string id, NuGetVersion version, CancellationToken cancellationToken) + public async Task MirrorAsync(string id, CancellationToken cancellationToken) { - if (await _localPackages.ExistsAsync(id, version)) + if (await _localPackages.ExistsAsync(id)) { return; } - var idString = id.ToLowerInvariant(); - var versionString = version.ToNormalizedString().ToLowerInvariant(); + _logger.LogInformation("Package {PackageId} does not exist locally. Mirroring...", id); - await IndexFromSourceAsync(idString, versionString, cancellationToken); + var versions = await _upstreamFeed.GetAllVersionsAsync(id, includeUnlisted: true); + + _logger.LogInformation( + "Found {VersionsCount} versions for package {PackageId} on upstream feed. Indexing...", + versions.Count, + id); + + // TODO: This will synchronously index packages one-by-one. This won't perform well + // for packages with many versions. Instead, this should index a few packages and + // let a background queue index the rest. + // See: https://docs.microsoft.com/en-us/aspnet/core/fundamentals/host/hosted-services?view=aspnetcore-2.1#queued-background-tasks + foreach (var version in versions) + { + var packageUri = await _upstreamFeed.GetPackageContentUriAsync(id, version); + + await IndexFromSourceAsync(packageUri, cancellationToken); + } + + _logger.LogInformation("Finished indexing {PackageId} from the upstream feed", id); } - private async Task IndexFromSourceAsync(string id, string version, CancellationToken cancellationToken) + private async Task IndexFromSourceAsync(Uri packageUri, CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); - _logger.LogInformation("Attempting to mirror package {Id} {Version}...", id, version); + _logger.LogInformation("Attempting to mirror package {PackageUri}...", packageUri); try { - // See https://github.com/NuGet/NuGet.Client/blob/4eed67e7e159796ae486d2cca406b283e23b6ac8/src/NuGet.Core/NuGet.Protocol/Resources/DownloadResourceV3.cs#L82 - var packageUri = new Uri(_packageBaseAddress, $"{id}/{version}/{id}.{version}.nupkg"); - using (var stream = await _downloader.DownloadOrNullAsync(packageUri, cancellationToken)) { if (stream == null) { _logger.LogWarning( - "Failed to download package {Id} {Version} at {PackageUri}", - id, - version, + "Failed to download package at {PackageUri}", packageUri); return; } - _logger.LogInformation("Downloaded package {Id} {Version}, indexing...", id, version); + _logger.LogInformation("Downloaded package at {PackageUri}, indexing...", packageUri); var result = await _indexer.IndexAsync(stream, cancellationToken); _logger.LogInformation( - "Finished indexing package {Id} {Version} with result {Result}", - id, - version, + "Finished indexing package at {PackageUri} with result {Result}", + packageUri, result); } } catch (Exception e) { - _logger.LogError(e, "Failed to mirror package {Id} {Version}", id, version); + _logger.LogError(e, "Failed to mirror package at {PackageUri}", packageUri); } } } diff --git a/src/BaGet.Core/Services/DatabaseSearchService.cs b/src/BaGet.Core/Services/DatabaseSearchService.cs index 8dcf0ecf3..7e375759b 100644 --- a/src/BaGet.Core/Services/DatabaseSearchService.cs +++ b/src/BaGet.Core/Services/DatabaseSearchService.cs @@ -53,7 +53,7 @@ public async Task> SearchAsync(string query, int ski Id = latest.Id, Version = latest.Version, Description = latest.Description, - Authors = string.Join(", ", latest.Authors), + Authors = latest.Authors, IconUrl = latest.IconUrlString, LicenseUrl = latest.LicenseUrlString, ProjectUrl = latest.ProjectUrlString, diff --git a/src/BaGet.Core/Services/IPackageService.cs b/src/BaGet.Core/Services/IPackageService.cs index 5a42b61ef..58736a7b4 100644 --- a/src/BaGet.Core/Services/IPackageService.cs +++ b/src/BaGet.Core/Services/IPackageService.cs @@ -58,7 +58,7 @@ public interface IPackageService /// The package id to search. /// The package version to search. /// Whether the package exists in the database. - Task ExistsAsync(string id, NuGetVersion version); + Task ExistsAsync(string id, NuGetVersion version = null); /// /// Unlist a package, making it undiscoverable. diff --git a/src/BaGet.Core/Services/ISearchService.cs b/src/BaGet.Core/Services/ISearchService.cs index 4a2a1b742..f57455954 100644 --- a/src/BaGet.Core/Services/ISearchService.cs +++ b/src/BaGet.Core/Services/ISearchService.cs @@ -22,7 +22,7 @@ public class SearchResult public NuGetVersion Version { get; set; } public string Description { get; set; } - public string Authors { get; set; } + public IReadOnlyList Authors { get; set; } public string IconUrl { get; set; } public string LicenseUrl { get; set; } public string ProjectUrl { get; set; } diff --git a/src/BaGet.Core/Services/PackageService.cs b/src/BaGet.Core/Services/PackageService.cs index b4c697499..39b7a7038 100644 --- a/src/BaGet.Core/Services/PackageService.cs +++ b/src/BaGet.Core/Services/PackageService.cs @@ -34,11 +34,17 @@ public async Task AddAsync(Package package) } } - public Task ExistsAsync(string id, NuGetVersion version) - => _context.Packages - .Where(p => p.Id == id) - .Where(p => p.VersionString == version.ToNormalizedString()) - .AnyAsync(); + public Task ExistsAsync(string id, NuGetVersion version = null) + { + var query = _context.Packages.Where(p => p.Id == id); + + if (version != null) + { + query = query.Where(p => p.VersionString == version.ToNormalizedString()); + } + + return query.AnyAsync(); + } public async Task> FindAsync(string id, bool includeUnlisted = false) { diff --git a/src/BaGet.Protocol/BaGet.Protocol.csproj b/src/BaGet.Protocol/BaGet.Protocol.csproj new file mode 100644 index 000000000..3e46a406d --- /dev/null +++ b/src/BaGet.Protocol/BaGet.Protocol.csproj @@ -0,0 +1,14 @@ + + + + netstandard2.0 + +
Libraries to interact with NuGet server APIs
+ + + + + + + + diff --git a/src/BaGet.Protocol/Converters/NuGetVersionConverter.cs b/src/BaGet.Protocol/Converters/NuGetVersionConverter.cs new file mode 100644 index 000000000..aa787a32d --- /dev/null +++ b/src/BaGet.Protocol/Converters/NuGetVersionConverter.cs @@ -0,0 +1,25 @@ +using System; +using Newtonsoft.Json; +using NuGet.Versioning; + +namespace BaGet.Protocol.Converters +{ + public class NuGetVersionConverter : JsonConverter + { + public override bool CanConvert(Type objectType) + { + return objectType == typeof(NuGetVersion); + } + + public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer) + { + var version = (NuGetVersion)value; + serializer.Serialize(writer, version.ToNormalizedString()); + } + + public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer) + { + return reader.TokenType != JsonToken.Null ? NuGetVersion.Parse(serializer.Deserialize(reader)) : null; + } + } +} diff --git a/src/BaGet.Protocol/Converters/NuGetVersionListConverter.cs b/src/BaGet.Protocol/Converters/NuGetVersionListConverter.cs new file mode 100644 index 000000000..d577bd16b --- /dev/null +++ b/src/BaGet.Protocol/Converters/NuGetVersionListConverter.cs @@ -0,0 +1,30 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Newtonsoft.Json; +using NuGet.Versioning; + +namespace BaGet.Protocol.Converters +{ + public class NuGetVersionListConverter : JsonConverter + { + public override bool CanConvert(Type objectType) + { + return objectType == typeof(IReadOnlyList); + } + + public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer) + { + var versions = ((IReadOnlyList)value); + + serializer.Serialize(writer, versions.Select(v => v.ToString())); + } + + public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer) + { + return serializer.Deserialize>(reader) + .Select(NuGetVersion.Parse) + .ToList(); + } + } +} diff --git a/src/BaGet.Protocol/Converters/SingleOrListConverter.cs b/src/BaGet.Protocol/Converters/SingleOrListConverter.cs new file mode 100644 index 000000000..36e5f5bbb --- /dev/null +++ b/src/BaGet.Protocol/Converters/SingleOrListConverter.cs @@ -0,0 +1,31 @@ +using System; +using System.Collections.Generic; +using Newtonsoft.Json; + +namespace BaGet.Protocol.Converters +{ + public class SingleOrListConverter : JsonConverter + { + public override bool CanConvert(Type objectType) + { + return objectType == typeof(IReadOnlyList); + } + + public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer) + { + if (reader.TokenType == JsonToken.StartArray) + { + return serializer.Deserialize>(reader); + } + else + { + return serializer.Deserialize(reader); + } + } + + public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer) + { + serializer.Serialize(writer, value); + } + } +} diff --git a/src/BaGet.Protocol/HttpContentExtensions.cs b/src/BaGet.Protocol/HttpContentExtensions.cs new file mode 100644 index 000000000..5ab6e9fbb --- /dev/null +++ b/src/BaGet.Protocol/HttpContentExtensions.cs @@ -0,0 +1,29 @@ +using System.IO; +using System.Net.Http; +using System.Threading.Tasks; +using Newtonsoft.Json; + +namespace BaGet.Protocol +{ + internal static class HttpContentExtensions + { + public static JsonSerializer Serializer => JsonSerializer.Create(Settings); + + public static JsonSerializerSettings Settings => new JsonSerializerSettings + { + DateTimeZoneHandling = DateTimeZoneHandling.Utc, + DateParseHandling = DateParseHandling.DateTimeOffset, + NullValueHandling = NullValueHandling.Ignore, + }; + + public static async Task ReadAsAsync(this HttpContent content) + { + using (var stream = await content.ReadAsStreamAsync()) + using (var textReader = new StreamReader(stream)) + using (var jsonReader = new JsonTextReader(textReader)) + { + return Serializer.Deserialize(jsonReader); + } + } + } +} diff --git a/src/BaGet.Protocol/PackageContent/IPackageContentClient.cs b/src/BaGet.Protocol/PackageContent/IPackageContentClient.cs new file mode 100644 index 000000000..4e216e9ab --- /dev/null +++ b/src/BaGet.Protocol/PackageContent/IPackageContentClient.cs @@ -0,0 +1,14 @@ +using System.IO; +using System.Threading.Tasks; + +namespace BaGet.Protocol +{ + public interface IPackageContentClient + { + Task GetPackageVersionsOrNullAsync(string url); + + Task GetPackageContentStreamAsync(string url); + + Task GetPackageManifestStreamAsync(string url); + } +} diff --git a/src/BaGet.Protocol/PackageContent/PackageContentClient.cs b/src/BaGet.Protocol/PackageContent/PackageContentClient.cs new file mode 100644 index 000000000..cd5b461ee --- /dev/null +++ b/src/BaGet.Protocol/PackageContent/PackageContentClient.cs @@ -0,0 +1,54 @@ +using System; +using System.IO; +using System.Net; +using System.Net.Http; +using System.Threading.Tasks; + +namespace BaGet.Protocol +{ + public class PackageContentClient : IPackageContentClient + { + private readonly HttpClient _httpClient; + + public PackageContentClient(HttpClient httpClient) + { + _httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient)); + } + + /// + /// Get a package's versions, or null if the package does not exist. + /// + /// The URL to fetch the package versions from. + /// The package's versions, or null if the package does not exist + public async Task GetPackageVersionsOrNullAsync(string url) + { + var response = await _httpClient.GetAsync(url); + if (response.StatusCode == HttpStatusCode.NotFound) + { + return null; + } + + response.EnsureSuccessStatusCode(); + + return await response.Content.ReadAsAsync(); + } + + public async Task GetPackageContentStreamAsync(string url) + { + var response = await _httpClient.GetAsync(url); + + response.EnsureSuccessStatusCode(); + + return await response.Content.ReadAsStreamAsync(); + } + + public async Task GetPackageManifestStreamAsync(string url) + { + var response = await _httpClient.GetAsync(url); + + response.EnsureSuccessStatusCode(); + + return await response.Content.ReadAsStreamAsync(); + } + } +} diff --git a/src/BaGet.Protocol/PackageContent/PackageVersions.cs b/src/BaGet.Protocol/PackageContent/PackageVersions.cs new file mode 100644 index 000000000..fb31b5b5c --- /dev/null +++ b/src/BaGet.Protocol/PackageContent/PackageVersions.cs @@ -0,0 +1,24 @@ +using System; +using System.Collections.Generic; +using BaGet.Protocol.Converters; +using Newtonsoft.Json; +using NuGet.Versioning; + +namespace BaGet.Protocol +{ + /// + /// The full list of versions for a package. + /// See: https://docs.microsoft.com/en-us/nuget/api/package-base-address-resource#enumerate-package-versions + /// Example: https://api.nuget.org/v3-flatcontainer/newtonsoft.json/index.json + /// + public class PackageVersions + { + public PackageVersions(IReadOnlyList versions) + { + Versions = versions ?? throw new ArgumentNullException(nameof(versions)); + } + + [JsonConverter(typeof(NuGetVersionListConverter))] + public IReadOnlyList Versions { get; } + } +} diff --git a/src/BaGet.Protocol/Registration/IRegistrationClient.cs b/src/BaGet.Protocol/Registration/IRegistrationClient.cs new file mode 100644 index 000000000..4d7952d0b --- /dev/null +++ b/src/BaGet.Protocol/Registration/IRegistrationClient.cs @@ -0,0 +1,13 @@ +using System.Threading.Tasks; + +namespace BaGet.Protocol +{ + public interface IRegistrationClient + { + Task GetRegistrationIndexAsync(string indexUrl); + + Task GetRegistrationIndexPageAsync(string pageUrl); + + Task GetRegistrationLeafAsync(string leafUrl); + } +} diff --git a/src/BaGet.Protocol/Registration/RegistrationClient.cs b/src/BaGet.Protocol/Registration/RegistrationClient.cs new file mode 100644 index 000000000..1076895a2 --- /dev/null +++ b/src/BaGet.Protocol/Registration/RegistrationClient.cs @@ -0,0 +1,43 @@ +using System; +using System.Net.Http; +using System.Threading.Tasks; + +namespace BaGet.Protocol +{ + public class RegistrationClient : IRegistrationClient + { + private readonly HttpClient _httpClient; + + public RegistrationClient(HttpClient httpClient) + { + _httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient)); + } + + public async Task GetRegistrationIndexAsync(string indexUrl) + { + var response = await _httpClient.GetAsync(indexUrl); + + response.EnsureSuccessStatusCode(); + + return await response.Content.ReadAsAsync(); + } + + public async Task GetRegistrationIndexPageAsync(string pageUrl) + { + var response = await _httpClient.GetAsync(pageUrl); + + response.EnsureSuccessStatusCode(); + + return await response.Content.ReadAsAsync(); + } + + public async Task GetRegistrationLeafAsync(string leafUrl) + { + var response = await _httpClient.GetAsync(leafUrl); + + response.EnsureSuccessStatusCode(); + + return await response.Content.ReadAsAsync(); + } + } +} diff --git a/src/BaGet.Protocol/Registration/RegistrationIndex.cs b/src/BaGet.Protocol/Registration/RegistrationIndex.cs new file mode 100644 index 000000000..58ac613ee --- /dev/null +++ b/src/BaGet.Protocol/Registration/RegistrationIndex.cs @@ -0,0 +1,35 @@ +using System; +using System.Collections.Generic; +using Newtonsoft.Json; + +namespace BaGet.Protocol +{ + /// + /// The metadata for a package and all of its versions. + /// + public class RegistrationIndex + { + public RegistrationIndex(int count, long totalDownloads, IReadOnlyList pages) + { + Count = count; + Pages = pages ?? throw new ArgumentNullException(nameof(pages)); + } + + /// + /// The number of registration pages. See . + /// + public int Count { get; } + + /// + /// How many times all versions of this package have been downloaded. + /// + public long TotalDownloads { get; } + + /// + /// The pages that contain all of the versions of the package, ordered + /// by the package's version. + /// + [JsonProperty(PropertyName = "items")] + public IReadOnlyList Pages { get; } + } +} diff --git a/src/BaGet.Protocol/Registration/RegistrationIndexPage.cs b/src/BaGet.Protocol/Registration/RegistrationIndexPage.cs new file mode 100644 index 000000000..1f3b1a825 --- /dev/null +++ b/src/BaGet.Protocol/Registration/RegistrationIndexPage.cs @@ -0,0 +1,51 @@ +using System; +using System.Collections.Generic; +using BaGet.Protocol.Converters; +using Newtonsoft.Json; +using NuGet.Versioning; + +namespace BaGet.Protocol +{ + public class RegistrationIndexPage + { + public RegistrationIndexPage( + string pageUrl, + int count, + IReadOnlyList itemsOrNull, + NuGetVersion lower, + NuGetVersion upper) + { + if (string.IsNullOrEmpty(pageUrl)) throw new ArgumentNullException(nameof(pageUrl)); + + PageUrl = pageUrl; + Count = count; + ItemsOrNull = itemsOrNull; + Lower = lower ?? throw new ArgumentNullException(nameof(lower)); + Upper = upper ?? throw new ArgumentNullException(nameof(upper)); + } + + [JsonProperty(PropertyName = "@id")] + public string PageUrl { get; } + + public int Count { get; } + + /// + /// Null if this package's registration is paged. The items can be found + /// by following the page's . + /// + [JsonProperty(PropertyName = "items")] + public IReadOnlyList ItemsOrNull { get; } + + /// + /// This page's lowest package version. + /// + [JsonConverter(typeof(NuGetVersionConverter))] + public NuGetVersion Lower { get; } + + /// + /// This page's highest package version. + /// + [JsonConverter(typeof(NuGetVersionConverter))] + public NuGetVersion Upper { get; } + } +} \ No newline at end of file diff --git a/src/BaGet.Protocol/Registration/RegistrationIndexPageItem.cs b/src/BaGet.Protocol/Registration/RegistrationIndexPageItem.cs new file mode 100644 index 000000000..03905cd18 --- /dev/null +++ b/src/BaGet.Protocol/Registration/RegistrationIndexPageItem.cs @@ -0,0 +1,153 @@ +using System; +using System.Collections.Generic; +using BaGet.Protocol.Converters; +using Newtonsoft.Json; +using NuGet.Versioning; + +namespace BaGet.Protocol +{ + public class RegistrationIndexPageItem + { + public RegistrationIndexPageItem(string leafUrl, CatalogEntry catalogEntry, string packageContent) + { + if (string.IsNullOrEmpty(leafUrl)) throw new ArgumentNullException(nameof(leafUrl)); + + LeafUrl = leafUrl; + CatalogEntry = catalogEntry ?? throw new ArgumentNullException(nameof(catalogEntry)); + PackageContent = packageContent ?? throw new ArgumentNullException(nameof(packageContent)); + } + + [JsonProperty(PropertyName = "@id")] + public string LeafUrl { get; } + + public CatalogEntry CatalogEntry { get; } + + public string PackageContent { get; } + } + + public class CatalogEntry + { + public CatalogEntry( + string catalogUri, + string packageId, + NuGetVersion version, + string authors, + string description, + long downloads, + bool hasReadme, + string iconUrl, + string language, + string licenseUrl, + bool listed, + string minClientVersion, + string packageContent, + string projectUrl, + string repositoryUrl, + string repositoryType, + DateTime published, + bool requireLicenseAcceptance, + string summary, + IReadOnlyList tags, + string title, + IReadOnlyList dependencyGroups) + { + CatalogUri = catalogUri ?? throw new ArgumentNullException(nameof(catalogUri)); + + PackageId = packageId; + Version = version; + Authors = authors; + Description = description; + Downloads = downloads; + HasReadme = hasReadme; + IconUrl = iconUrl; + Language = language; + LicenseUrl = licenseUrl; + Listed = listed; + MinClientVersion = minClientVersion; + PackageContent = packageContent; + ProjectUrl = projectUrl; + RepositoryUrl = repositoryUrl; + RepositoryType = repositoryType; + Published = published; + RequireLicenseAcceptance = requireLicenseAcceptance; + Summary = summary; + Tags = tags; + Title = title; + DependencyGroups = dependencyGroups; + } + + [JsonProperty(PropertyName = "@id")] + public string CatalogUri { get; } + + [JsonProperty(PropertyName = "id")] + public string PackageId { get; } + + [JsonConverter(typeof(NuGetVersionConverter))] + public NuGetVersion Version { get; } + + public string Authors { get; } + public string Description { get; } + public long Downloads { get; } + public bool HasReadme { get; } + public string IconUrl { get; } + public string Language { get; } + public string LicenseUrl { get; } + public bool Listed { get; } + public string MinClientVersion { get; } + public string PackageContent { get; } + public string ProjectUrl { get; } + public string RepositoryUrl { get; } + public string RepositoryType { get; } + public DateTime Published { get; } + public bool RequireLicenseAcceptance { get; } + public string Summary { get; } + public IReadOnlyList Tags { get; } + public string Title { get; } + public IReadOnlyList DependencyGroups { get; } + } + + public class DependencyGroupItem + { + public DependencyGroupItem( + string id, + string targetFramework, + IReadOnlyList dependencyItems) + { + Id = id; + Type = "PackageDependencyGroup"; + TargetFramework = targetFramework; + Dependencies = (dependencyItems?.Count > 0) ? dependencyItems : null; + } + + [JsonProperty(PropertyName = "@id")] + public string Id { get; } + + [JsonProperty(PropertyName = "@type")] + public string Type { get; } + + public string TargetFramework { get; } + + [JsonProperty(DefaultValueHandling = DefaultValueHandling.Ignore)] + public IReadOnlyList Dependencies { get; } + } + + public class DependencyItem + { + [JsonProperty(PropertyName = "@id")] + public string DepId { get; } + + [JsonProperty(PropertyName = "@type")] + public string Type { get; } + + public string Id { get; } + public string Range { get; } + + public DependencyItem(string groupId, string depId, string versionRange) + { + DepId = groupId + "/" + depId; + Type = "PackageDependency"; + Id = depId; + Range = versionRange; + } + } +} \ No newline at end of file diff --git a/src/BaGet.Protocol/Registration/RegistrationLeaf.cs b/src/BaGet.Protocol/Registration/RegistrationLeaf.cs new file mode 100644 index 000000000..c063a21c3 --- /dev/null +++ b/src/BaGet.Protocol/Registration/RegistrationLeaf.cs @@ -0,0 +1,43 @@ +using System; +using Newtonsoft.Json; + +namespace BaGet.Protocol +{ + /// + /// The metadata for a single version of a package. + /// Documentation: https://docs.microsoft.com/en-us/nuget/api/registration-base-url-resource + /// + public class RegistrationLeaf + { + public RegistrationLeaf( + string registrationUri, + bool listed, + long downloads, + string packageContentUrl, + DateTimeOffset published, + string registrationIndexUrl) + { + RegistrationUri = registrationUri ?? throw new ArgumentNullException(nameof(registrationIndexUrl)); + Listed = listed; + Published = published; + Downloads = downloads; + PackageContentUrl = packageContentUrl ?? throw new ArgumentNullException(nameof(packageContentUrl)); + RegistrationIndexUrl = registrationIndexUrl ?? throw new ArgumentNullException(nameof(registrationIndexUrl)); + } + + [JsonProperty(PropertyName = "@id")] + public string RegistrationUri { get; } + + public bool Listed { get; } + + public long Downloads { get; } + + [JsonProperty(PropertyName = "packageContent")] + public string PackageContentUrl { get; } + + public DateTimeOffset Published { get; } + + [JsonProperty(PropertyName = "registration")] + public string RegistrationIndexUrl { get; } + } +} diff --git a/src/BaGet.Protocol/Search/AutocompleteResult.cs b/src/BaGet.Protocol/Search/AutocompleteResult.cs new file mode 100644 index 000000000..775b02cd7 --- /dev/null +++ b/src/BaGet.Protocol/Search/AutocompleteResult.cs @@ -0,0 +1,28 @@ +using System; +using System.Collections.Generic; + +namespace BaGet.Protocol +{ + /// + /// The package ids that matched the autocomplete query. + /// Documentation: https://docs.microsoft.com/en-us/nuget/api/search-autocomplete-service-resource#search-for-package-ids + /// + public class AutocompleteResult + { + public AutocompleteResult(int totalHits, IReadOnlyList data) + { + TotalHits = totalHits; + Data = data ?? throw new ArgumentNullException(nameof(data)); + } + + /// + /// The total number of matches, disregarding skip and take. + /// + public int TotalHits; + + /// + /// The package IDs matched by the autocomplete query. + /// + public IReadOnlyList Data; + } +} diff --git a/src/BaGet.Protocol/Search/ISearchClient.cs b/src/BaGet.Protocol/Search/ISearchClient.cs new file mode 100644 index 000000000..74c38ceaa --- /dev/null +++ b/src/BaGet.Protocol/Search/ISearchClient.cs @@ -0,0 +1,11 @@ +using System.Threading.Tasks; + +namespace BaGet.Protocol +{ + public interface ISearchClient + { + Task GetSearchResultsAsync(string searchUrl); + + Task GetAutocompleteResultsAsync(string searchUrl); + } +} diff --git a/src/BaGet.Protocol/Search/SearchClient.cs b/src/BaGet.Protocol/Search/SearchClient.cs new file mode 100644 index 000000000..88de4ed48 --- /dev/null +++ b/src/BaGet.Protocol/Search/SearchClient.cs @@ -0,0 +1,34 @@ +using System; +using System.Net.Http; +using System.Threading.Tasks; + +namespace BaGet.Protocol +{ + public class SearchClient : ISearchClient + { + private readonly HttpClient _httpClient; + + public SearchClient(HttpClient httpClient) + { + _httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient)); + } + + public async Task GetSearchResultsAsync(string searchUrl) + { + var response = await _httpClient.GetAsync(searchUrl); + + response.EnsureSuccessStatusCode(); + + return await response.Content.ReadAsAsync(); + } + + public async Task GetAutocompleteResultsAsync(string searchUrl) + { + var response = await _httpClient.GetAsync(searchUrl); + + response.EnsureSuccessStatusCode(); + + return await response.Content.ReadAsAsync(); + } + } +} diff --git a/src/BaGet.Protocol/Search/SearchResponse.cs b/src/BaGet.Protocol/Search/SearchResponse.cs new file mode 100644 index 000000000..bf6efbbb4 --- /dev/null +++ b/src/BaGet.Protocol/Search/SearchResponse.cs @@ -0,0 +1,28 @@ +using System; +using System.Collections.Generic; + +namespace BaGet.Protocol +{ + /// + /// The response to a search query. + /// Documentation: https://docs.microsoft.com/en-us/nuget/api/search-query-service-resource#response + /// + public class SearchResponse + { + public SearchResponse(int totalHits, IReadOnlyList data) + { + TotalHits = totalHits; + Data = data ?? throw new ArgumentNullException(nameof(data)); + } + + /// + /// The total number of matches, disregarding skip and take. + /// + public int TotalHits; + + /// + /// The packages that matched the search query. + /// + public IReadOnlyList Data; + } +} diff --git a/src/BaGet.Protocol/Search/SearchResult.cs b/src/BaGet.Protocol/Search/SearchResult.cs new file mode 100644 index 000000000..88901ab32 --- /dev/null +++ b/src/BaGet.Protocol/Search/SearchResult.cs @@ -0,0 +1,76 @@ +using System; +using System.Collections.Generic; +using BaGet.Protocol.Converters; +using Newtonsoft.Json; +using NuGet.Versioning; + +namespace BaGet.Protocol +{ + /// + /// A package that matched a search query. + /// Documentation: https://docs.microsoft.com/en-us/nuget/api/search-query-service-resource#search-result + /// + public class SearchResult + { + public SearchResult( + string id, + NuGetVersion version, + string description, + IReadOnlyList authors, + string iconUrl, + string licenseUrl, + string projectUrl, + string registrationUrl, + string summary, + IReadOnlyList tags, + string title, + long totalDownloads, + IReadOnlyList versions) + { + if (string.IsNullOrEmpty(id)) + { + throw new ArgumentException(nameof(id)); + } + + version = version ?? throw new ArgumentNullException(nameof(version)); + versions = versions ?? throw new ArgumentNullException(nameof(versions)); + + Id = id; + Version = version; + Description = description; + Authors = authors; + IconUrl = iconUrl; + LicenseUrl = licenseUrl; + ProjectUrl = projectUrl; + RegistrationUrl = registrationUrl; + Summary = summary; + Tags = tags; + Title = title; + TotalDownloads = totalDownloads; + + Versions = versions; + } + + public string Id { get; } + + [JsonConverter(typeof(NuGetVersionConverter))] + public NuGetVersion Version { get; } + + public string Description { get; } + + [JsonConverter(typeof(SingleOrListConverter))] + public IReadOnlyList Authors { get; } + public string IconUrl { get; } + public string LicenseUrl { get; } + public string ProjectUrl { get; } + + [JsonProperty(PropertyName = "registration")] + public string RegistrationUrl { get; } + public string Summary { get; } + public IReadOnlyList Tags { get; } + public string Title { get; } + public long TotalDownloads { get; } + + public IReadOnlyList Versions { get; } + } +} diff --git a/src/BaGet.Protocol/Search/SearchResultVersion.cs b/src/BaGet.Protocol/Search/SearchResultVersion.cs new file mode 100644 index 000000000..1bb91fcef --- /dev/null +++ b/src/BaGet.Protocol/Search/SearchResultVersion.cs @@ -0,0 +1,37 @@ +using System; +using BaGet.Protocol.Converters; +using Newtonsoft.Json; +using NuGet.Versioning; + +namespace BaGet.Protocol +{ + /// + /// The version of a package that matched a search query. + /// See: . + /// Documentation: https://docs.microsoft.com/en-us/nuget/api/search-query-service-resource#search-result + /// + public class SearchResultVersion + { + public SearchResultVersion( + string registrationLeafUrl, + NuGetVersion version, + long downloads) + { + if (string.IsNullOrEmpty(registrationLeafUrl)) throw new ArgumentNullException(nameof(registrationLeafUrl)); + + version = version ?? throw new ArgumentNullException(nameof(version)); + + RegistrationLeafUrl = registrationLeafUrl; + Version = version; + Downloads = downloads; + } + + [JsonProperty(PropertyName = "@id")] + public string RegistrationLeafUrl { get; } + + [JsonConverter(typeof(NuGetVersionConverter))] + public NuGetVersion Version { get; } + + public long Downloads { get; } + } +} \ No newline at end of file diff --git a/src/BaGet.Protocol/ServiceIndex/IServiceIndexClient.cs b/src/BaGet.Protocol/ServiceIndex/IServiceIndexClient.cs new file mode 100644 index 000000000..fc009b8b2 --- /dev/null +++ b/src/BaGet.Protocol/ServiceIndex/IServiceIndexClient.cs @@ -0,0 +1,9 @@ +using System.Threading.Tasks; + +namespace BaGet.Protocol +{ + public interface IServiceIndexClient + { + Task GetServiceIndexAsync(string indexUrl); + } +} diff --git a/src/BaGet.Protocol/ServiceIndex/ServiceIndex.cs b/src/BaGet.Protocol/ServiceIndex/ServiceIndex.cs new file mode 100644 index 000000000..b9950dab4 --- /dev/null +++ b/src/BaGet.Protocol/ServiceIndex/ServiceIndex.cs @@ -0,0 +1,33 @@ +using System; +using System.Collections.Generic; +using BaGet.Protocol.Converters; +using Newtonsoft.Json; +using NuGet.Versioning; + +namespace BaGet.Protocol +{ + /// + /// The entry point for a NuGet package source used by the client to find APIs. + /// Documentation: https://docs.microsoft.com/en-us/nuget/api/overview + /// NuGet.org: https://api.nuget.org/v3-index/index.json + /// + public class ServiceIndex + { + public ServiceIndex(NuGetVersion version, IReadOnlyList resources) + { + Version = version ?? throw new ArgumentNullException(nameof(version)); + Resources = resources ?? throw new ArgumentNullException(nameof(resources)); + } + + /// + /// The service index's version. + /// + [JsonConverter(typeof(NuGetVersionConverter))] + public NuGetVersion Version { get; } + + /// + /// The resource contained by this service index. + /// + public IReadOnlyList Resources { get; } + } +} diff --git a/src/BaGet.Protocol/ServiceIndex/ServiceIndexClient.cs b/src/BaGet.Protocol/ServiceIndex/ServiceIndexClient.cs new file mode 100644 index 000000000..57db884ff --- /dev/null +++ b/src/BaGet.Protocol/ServiceIndex/ServiceIndexClient.cs @@ -0,0 +1,28 @@ +using System; +using System.Net.Http; +using System.Threading.Tasks; + +namespace BaGet.Protocol +{ + /// + /// See: https://docs.microsoft.com/en-us/nuget/api/service-index + /// + public class ServiceIndexClient : IServiceIndexClient + { + private readonly HttpClient _httpClient; + + public ServiceIndexClient(HttpClient httpClient) + { + _httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient)); + } + + public async Task GetServiceIndexAsync(string indexUrl) + { + var response = await _httpClient.GetAsync(indexUrl); + + response.EnsureSuccessStatusCode(); + + return await response.Content.ReadAsAsync(); + } + } +} diff --git a/src/BaGet.Protocol/ServiceIndex/ServiceIndexResource.cs b/src/BaGet.Protocol/ServiceIndex/ServiceIndexResource.cs new file mode 100644 index 000000000..a0632196c --- /dev/null +++ b/src/BaGet.Protocol/ServiceIndex/ServiceIndexResource.cs @@ -0,0 +1,36 @@ +using System; +using Newtonsoft.Json; + +namespace BaGet.Protocol +{ + /// + /// A resource in the . + /// See: https://docs.microsoft.com/en-us/nuget/api/service-index#resources + /// + public class ServiceIndexResource + { + public ServiceIndexResource(string type, string url, string comment = null) + { + Url = url ?? throw new ArgumentNullException(nameof(url)); + Type = type ?? throw new ArgumentNullException(nameof(type)); + Comment = comment ?? string.Empty; + } + + /// + /// The resource's base URL. + /// + [JsonProperty(PropertyName = "@id")] + public string Url { get; } + + /// + /// The resource's type. + /// + [JsonProperty(PropertyName = "@type")] + public string Type { get; } + + /// + /// Human readable comments about the resource. + /// + public string Comment { get; } + } +} \ No newline at end of file diff --git a/src/BaGet.Protocol/Services/IPackageMetadataService.cs b/src/BaGet.Protocol/Services/IPackageMetadataService.cs new file mode 100644 index 000000000..a4d25c155 --- /dev/null +++ b/src/BaGet.Protocol/Services/IPackageMetadataService.cs @@ -0,0 +1,34 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using NuGet.Versioning; + +namespace BaGet.Protocol +{ + /// + /// Gets metadata about a package from a remote feed. + /// + public interface IPackageMetadataService + { + /// + /// Get all versions of a package from a remote NuGet feed. + /// + /// The package whose versions should be fetched. + /// Whether results should include unlisted versions. + /// A token to cancel the operation. + /// All versions for the package. + Task> GetAllVersionsAsync( + string packageId, + bool includeUnlisted = false, + CancellationToken cancellationToken = default); + + /// + /// Get the URI to download a package's content from a remote NuGet feed. + /// + /// The package whose URL should be fetched. + /// The package's version whose uRL should be fetched. + /// A URI that can be used to download the package. + Task GetPackageContentUriAsync(string packageId, NuGetVersion version); + } +} diff --git a/src/BaGet.Protocol/Services/IServiceIndexService.cs b/src/BaGet.Protocol/Services/IServiceIndexService.cs new file mode 100644 index 000000000..f6bc0eab3 --- /dev/null +++ b/src/BaGet.Protocol/Services/IServiceIndexService.cs @@ -0,0 +1,13 @@ +using System.Threading.Tasks; + +namespace BaGet.Protocol +{ + public interface IServiceIndexService + { + Task GetPackageContentUrlAsync(); + + Task GetRegistrationUrlAsync(); + + Task GetSearchUrlAsync(); + } +} diff --git a/src/BaGet.Protocol/Services/PackageMetadataService.cs b/src/BaGet.Protocol/Services/PackageMetadataService.cs new file mode 100644 index 000000000..98fb32719 --- /dev/null +++ b/src/BaGet.Protocol/Services/PackageMetadataService.cs @@ -0,0 +1,104 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using NuGet.Versioning; + +namespace BaGet.Protocol +{ + public class PackageMetadataService : IPackageMetadataService + { + private readonly IServiceIndexService _serviceIndexService; + private readonly IRegistrationClient _registrationClient; + private readonly IPackageContentClient _packageContentClient; + + public PackageMetadataService( + IServiceIndexService serviceIndexService, + IRegistrationClient registrationClient, + IPackageContentClient packageContentClient) + { + _serviceIndexService = serviceIndexService ?? throw new ArgumentNullException(nameof(serviceIndexService)); + _registrationClient = registrationClient ?? throw new ArgumentNullException(nameof(registrationClient)); + _packageContentClient = packageContentClient ?? throw new ArgumentNullException(nameof(packageContentClient)); + } + + public async Task> GetAllVersionsAsync( + string packageId, + bool includeUnlisted = false, + CancellationToken cancellationToken = default) + { + if (!includeUnlisted) + { + return await GetAllListedVersionsFromRegistrationResourceAsync(packageId, cancellationToken); + } + { + return await GetAllVersionsFromPackageContentResourceAsync(packageId, cancellationToken); + } + } + + public async Task GetPackageContentUriAsync(string id, NuGetVersion version) + { + var packageContentUrl = await _serviceIndexService.GetPackageContentUrlAsync(); + var packageId = id.ToLowerInvariant(); + var packageVersion = version.ToNormalizedString().ToLowerInvariant(); + + return new Uri($"{packageContentUrl}/{packageId}/{packageVersion}/{packageId}.{packageVersion}.nupkg"); + } + + private async Task> GetAllListedVersionsFromRegistrationResourceAsync(string packageId, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + + var registrationUrl = await _serviceIndexService.GetRegistrationUrlAsync(); + var requestUrl = $"{registrationUrl}/{packageId.ToLowerInvariant()}/index.json"; + + var packageIndex = await _registrationClient.GetRegistrationIndexAsync(requestUrl); + var result = new List(); + + foreach (var page in packageIndex.Pages) + { + cancellationToken.ThrowIfCancellationRequested(); + + // If the package's registration index is too big, its pages' items will be + // stored at different URLs. We will need to fetch each page's items individually. + // We can detect this case as the index's pages will have "null" items. + var items = page.ItemsOrNull; + + if (items == null) + { + var externalPage = await _registrationClient.GetRegistrationIndexPageAsync(page.PageUrl); + + if (externalPage.ItemsOrNull == null) + { + // This should never happen... + continue; + } + + items = externalPage.ItemsOrNull; + } + + result.AddRange( + items.Where(i => i.CatalogEntry.Listed).Select(i => i.CatalogEntry.Version)); + } + + return result; + } + + private async Task> GetAllVersionsFromPackageContentResourceAsync(string packageId, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + + var packageContentUrl = await _serviceIndexService.GetPackageContentUrlAsync(); + var requestUrl = $"{packageContentUrl}/{packageId.ToLowerInvariant()}/index.json"; + + var result = await _packageContentClient.GetPackageVersionsOrNullAsync(requestUrl); + if (result == null) + { + return new NuGetVersion[0]; + } + + return result.Versions; + } + } +} diff --git a/src/BaGet.Protocol/Services/ServiceIndexService.cs b/src/BaGet.Protocol/Services/ServiceIndexService.cs new file mode 100644 index 000000000..fadbde2f5 --- /dev/null +++ b/src/BaGet.Protocol/Services/ServiceIndexService.cs @@ -0,0 +1,65 @@ +using System; +using System.Linq; +using System.Threading.Tasks; + +namespace BaGet.Protocol +{ + public class ServiceIndexService : IServiceIndexService + { + // See: https://github.com/NuGet/NuGet.Client/blob/dev/src/NuGet.Core/NuGet.Protocol/Constants.cs + public static readonly string Version200 = "/2.0.0"; + public static readonly string Version300beta = "/3.0.0-beta"; + public static readonly string Version300 = "/3.0.0"; + public static readonly string Version340 = "/3.4.0"; + public static readonly string Versioned = "/Versioned"; + public static readonly string Version470 = "/4.7.0"; + public static readonly string Version490 = "/4.9.0"; + + public static readonly string[] SearchQueryService = { "SearchQueryService" + Versioned, "SearchQueryService" + Version340, "SearchQueryService" + Version300beta }; + public static readonly string[] RegistrationsBaseUrl = { "RegistrationsBaseUrl" + Versioned, "RegistrationsBaseUrl" + Version340, "RegistrationsBaseUrl" + Version300beta }; + public static readonly string[] SearchAutocompleteService = { "SearchAutocompleteService" + Versioned, "SearchAutocompleteService" + Version300beta }; + public static readonly string[] ReportAbuse = { "ReportAbuseUriTemplate" + Versioned, "ReportAbuseUriTemplate" + Version300 }; + public static readonly string[] LegacyGallery = { "LegacyGallery" + Versioned, "LegacyGallery" + Version200 }; + public static readonly string[] PackagePublish = { "PackagePublish" + Versioned, "PackagePublish" + Version200 }; + public static readonly string[] PackageBaseAddress = { "PackageBaseAddress" + Versioned, "PackageBaseAddress" + Version300 }; + public static readonly string[] RepositorySignatures = { "RepositorySignatures" + Version490, "RepositorySignatures" + Version470 }; + public static readonly string[] SymbolPackagePublish = { "SymbolPackagePublish" + Version490 }; + + private readonly IServiceIndexClient _client; + private readonly Lazy> _serviceIndexTask; + + public ServiceIndexService( + string serviceIndexUrl, + IServiceIndexClient client) + { + _client = client ?? throw new ArgumentNullException(nameof(client)); + _serviceIndexTask = new Lazy>(() => + { + return client.GetServiceIndexAsync(serviceIndexUrl); + }); + } + + public async Task GetPackageContentUrlAsync() + { + return await GetUrlForResourceType(PackageBaseAddress); + } + + public async Task GetRegistrationUrlAsync() + { + return await GetUrlForResourceType(RegistrationsBaseUrl); + } + + public async Task GetSearchUrlAsync() + { + return await GetUrlForResourceType(SearchQueryService); + } + + private async Task GetUrlForResourceType(string[] types) + { + var serviceIndex = await _serviceIndexTask.Value; + var resource = serviceIndex.Resources.First(r => types.Contains(r.Type)); + + return resource.Url.Trim('/'); + } + } +} diff --git a/src/BaGet/BaGet.csproj b/src/BaGet/BaGet.csproj index d02f08fec..cc372144f 100644 --- a/src/BaGet/BaGet.csproj +++ b/src/BaGet/BaGet.csproj @@ -21,6 +21,7 @@ + diff --git a/src/BaGet/Controllers/IndexController.cs b/src/BaGet/Controllers/IndexController.cs index c6813f9a1..3fd1f6efd 100644 --- a/src/BaGet/Controllers/IndexController.cs +++ b/src/BaGet/Controllers/IndexController.cs @@ -1,9 +1,8 @@ -using System; using System.Collections.Generic; -using System.Linq; using BaGet.Extensions; +using BaGet.Protocol; using Microsoft.AspNetCore.Mvc; -using Newtonsoft.Json; +using NuGet.Versioning; namespace BaGet.Controllers { @@ -12,50 +11,29 @@ namespace BaGet.Controllers /// public class IndexController : Controller { - private IEnumerable ServiceWithAliases(string name, string url, params string[] versions) + private IEnumerable BuildResource(string name, string url, params string[] versions) { foreach (var version in versions) { - string fullname = string.IsNullOrEmpty(version) ? name : name + "/" + version; - yield return new ServiceResource(fullname, url); + var type = string.IsNullOrEmpty(version) ? name : $"{name}/{version}"; + + yield return new ServiceIndexResource(type, url); } } // GET v3/index [HttpGet] - public object Get() - { - // Documentation: https://docs.microsoft.com/en-us/nuget/api/overview - // NuGet.org: https://api.nuget.org/v3-index/index.json - return new - { - Version = "3.0.0", - Resources = - ServiceWithAliases("PackagePublish", Url.PackagePublish(), "2.0.0") // api.nuget.org returns this too. - .Concat(ServiceWithAliases("SearchQueryService", Url.PackageSearch(), "", "3.0.0-beta", "3.0.0-rc")) // each version is an alias of others - .Concat(ServiceWithAliases("RegistrationsBaseUrl", Url.RegistrationsBase(), "", "3.0.0-rc", "3.0.0-beta")) - .Concat(ServiceWithAliases("PackageBaseAddress", Url.PackageBase(), "3.0.0")) - .Concat(ServiceWithAliases("SearchAutocompleteService", Url.PackageAutocomplete(), "", "3.0.0-rc", "3.0.0-beta")) - .ToList() - }; - } - - private class ServiceResource + public ServiceIndex Get() { - public ServiceResource(string type, string id, string comment = null) - { - Id = id ?? throw new ArgumentNullException(nameof(id)); - Type = type ?? throw new ArgumentNullException(nameof(type)); - Comment = comment ?? string.Empty; - } - - [JsonProperty(PropertyName = "@id")] - public string Id { get; } + var resources = new List(); - [JsonProperty(PropertyName = "@type")] - public string Type { get; } + resources.AddRange(BuildResource("PackagePublish", Url.PackagePublish(), "2.0.0")); + resources.AddRange(BuildResource("SearchQueryService", Url.PackageSearch(), "", "3.0.0-beta", "3.0.0-rc")); + resources.AddRange(BuildResource("RegistrationsBaseUrl", Url.RegistrationsBase(), "", "3.0.0-rc", "3.0.0-beta")); + resources.AddRange(BuildResource("PackageBaseAddress", Url.PackageBase(), "3.0.0")); + resources.AddRange(BuildResource("SearchAutocompleteService", Url.PackageAutocomplete(), "", "3.0.0-rc", "3.0.0-beta")); - public string Comment { get; } + return new ServiceIndex(new NuGetVersion("3.0.0"), resources); } } } \ No newline at end of file diff --git a/src/BaGet/Controllers/PackageController.cs b/src/BaGet/Controllers/PackageController.cs index eb5b359eb..a7b3642df 100644 --- a/src/BaGet/Controllers/PackageController.cs +++ b/src/BaGet/Controllers/PackageController.cs @@ -4,6 +4,7 @@ using System.Threading.Tasks; using BaGet.Core.Mirror; using BaGet.Core.Services; +using BaGet.Protocol; using Microsoft.AspNetCore.Mvc; using NuGet.Packaging.Core; using NuGet.Versioning; @@ -32,10 +33,9 @@ public async Task Versions(string id) return NotFound(); } - return Json(new - { - Versions = packages.Select(p => p.VersionString).ToList() - }); + var versions = packages.Select(p => p.Version).ToList(); + + return Json(new PackageVersions(versions)); } public async Task DownloadPackage(string id, string version, CancellationToken cancellationToken) @@ -46,7 +46,7 @@ public async Task DownloadPackage(string id, string version, Canc } // Allow read-through caching if it is configured. - await _mirror.MirrorAsync(id, nugetVersion, cancellationToken); + await _mirror.MirrorAsync(id, cancellationToken); if (!await _packages.AddDownloadAsync(id, nugetVersion)) { @@ -66,7 +66,7 @@ public async Task DownloadNuspec(string id, string version, Cance } // Allow read-through caching if it is configured. - await _mirror.MirrorAsync(id, nugetVersion, cancellationToken); + await _mirror.MirrorAsync(id, cancellationToken); if (!await _packages.ExistsAsync(id, nugetVersion)) { @@ -86,7 +86,7 @@ public async Task DownloadReadme(string id, string version, Cance } // Allow read-through caching if it is configured. - await _mirror.MirrorAsync(id, nugetVersion, cancellationToken); + await _mirror.MirrorAsync(id, cancellationToken); var package = await _packages.FindOrNullAsync(id, nugetVersion); diff --git a/src/BaGet/Controllers/Registration/RegistrationIndexController.cs b/src/BaGet/Controllers/Registration/RegistrationIndexController.cs index c79f4dfd4..90d4c35a7 100644 --- a/src/BaGet/Controllers/Registration/RegistrationIndexController.cs +++ b/src/BaGet/Controllers/Registration/RegistrationIndexController.cs @@ -1,12 +1,14 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Threading; using System.Threading.Tasks; using BaGet.Core.Entities; +using BaGet.Core.Mirror; using BaGet.Core.Services; using BaGet.Extensions; +using BaGet.Protocol; using Microsoft.AspNetCore.Mvc; -using Newtonsoft.Json; namespace BaGet.Controllers.Registration { @@ -15,18 +17,22 @@ namespace BaGet.Controllers.Registration /// public class RegistrationIndexController : Controller { + private readonly IMirrorService _mirror; private readonly IPackageService _packages; - public RegistrationIndexController(IPackageService packages) + public RegistrationIndexController(IMirrorService mirror, IPackageService packages) { + _mirror = mirror ?? throw new ArgumentNullException(nameof(mirror)); _packages = packages ?? throw new ArgumentNullException(nameof(packages)); } // GET v3/registration/{id}.json [HttpGet] - public async Task Get(string id) + public async Task Get(string id, CancellationToken cancellationToken) { - // Documentation: https://docs.microsoft.com/en-us/nuget/api/registration-base-url-resource + // Allow read-through caching to happen if it is configured. + await _mirror.MirrorAsync(id, cancellationToken); + var packages = await _packages.FindAsync(id); var versions = packages.Select(p => p.Version).ToList(); @@ -38,204 +44,69 @@ public async Task Get(string id) // TODO: Paging of registration items. // "Un-paged" example: https://api.nuget.org/v3/registration3/newtonsoft.json/index.json // Paged example: https://api.nuget.org/v3/registration3/fake/index.json - return Json(new - { - Count = packages.Count, - TotalDownloads = packages.Sum(p => p.Downloads), - Items = new[] + return Json(new RegistrationIndex( + count: packages.Count, + totalDownloads: packages.Sum(p => p.Downloads), + pages: new[] { - new RegistrationIndexItem( - packageId: id, - items: packages.Select(ToRegistrationIndexLeaf).ToList(), - lower: versions.Min().ToNormalizedString(), - upper: versions.Max().ToNormalizedString() - ), - } - }); + new RegistrationIndexPage( + Url.PackageRegistration(packages.First().Id), + count: packages.Count(), + itemsOrNull: packages.Select(ToRegistrationIndexPageItem).ToList(), + lower: versions.Min(), + upper: versions.Max()) + })); } - private RegistrationIndexLeaf ToRegistrationIndexLeaf(Package package) => - new RegistrationIndexLeaf( - packageId: package.Id, + private RegistrationIndexPageItem ToRegistrationIndexPageItem(Package package) => + new RegistrationIndexPageItem( + leafUrl: Url.PackageRegistration(package.Id, package.Version), catalogEntry: new CatalogEntry( - package: package, catalogUri: $"https://api.nuget.org/v3/catalog0/data/2015.02.01.06.24.15/{package.Id}.{package.Version}.json", - packageContent: Url.PackageDownload(package.Id, package.Version)), + packageId: package.Id, + version: package.Version, + authors: string.Join(", ", package.Authors), + description: package.Description, + downloads: package.Downloads, + hasReadme: package.HasReadme, + iconUrl: package.IconUrlString, + language: package.Language, + licenseUrl: package.LicenseUrlString, + listed: package.Listed, + minClientVersion: package.MinClientVersion, + packageContent: Url.PackageDownload(package.Id, package.Version), + projectUrl: package.ProjectUrlString, + repositoryUrl: package.RepositoryUrlString, + repositoryType: package.RepositoryType, + published: package.Published, + requireLicenseAcceptance: package.RequireLicenseAcceptance, + summary: package.Summary, + tags: package.Tags, + title: package.Title, + dependencyGroups: ToDependencyGroups(package)), packageContent: Url.PackageDownload(package.Id, package.Version)); - private class RegistrationIndexItem - { - public RegistrationIndexItem( - string packageId, - IReadOnlyList items, - string lower, - string upper) - { - if (string.IsNullOrEmpty(packageId)) throw new ArgumentNullException(nameof(packageId)); - if (string.IsNullOrEmpty(lower)) throw new ArgumentNullException(nameof(lower)); - if (string.IsNullOrEmpty(upper)) throw new ArgumentNullException(nameof(upper)); - - PackageId = packageId; - Items = items ?? throw new ArgumentNullException(nameof(items)); - Lower = lower; - Upper = upper; - } - - [JsonProperty(PropertyName = "id")] - public string PackageId { get; } - - public int Count => Items.Count; - - public IReadOnlyList Items { get; } - - public string Lower { get; } - public string Upper { get; } - } - - private class RegistrationIndexLeaf + private IReadOnlyList ToDependencyGroups(Package package) { - public RegistrationIndexLeaf(string packageId, CatalogEntry catalogEntry, string packageContent) - { - if (string.IsNullOrEmpty(packageId)) throw new ArgumentNullException(nameof(packageId)); - - PackageId = packageId; - CatalogEntry = catalogEntry ?? throw new ArgumentNullException(nameof(catalogEntry)); - PackageContent = packageContent ?? throw new ArgumentNullException(nameof(packageContent)); - } - - [JsonProperty(PropertyName = "id")] - public string PackageId { get; } - - public CatalogEntry CatalogEntry { get; } - - public string PackageContent { get; } - } - - private class CatalogEntry - { - public CatalogEntry(Package package, string catalogUri, string packageContent) - { - if (package == null) throw new ArgumentNullException(nameof(package)); - - CatalogUri = catalogUri ?? throw new ArgumentNullException(nameof(catalogUri)); - - PackageId = package.Id; - Version = package.VersionString; - Authors = string.Join(", ", package.Authors); - Description = package.Description; - Downloads = package.Downloads; - HasReadme = package.HasReadme; - IconUrl = package.IconUrlString; - Language = package.Language; - LicenseUrl = package.LicenseUrlString; - Listed = package.Listed; - MinClientVersion = package.MinClientVersion; - PackageContent = packageContent; - ProjectUrl = package.ProjectUrlString; - RepositoryUrl = package.RepositoryUrlString; - RepositoryType = package.RepositoryType; - Published = package.Published; - RequireLicenseAcceptance = package.RequireLicenseAcceptance; - Summary = package.Summary; - Tags = package.Tags; - Title = package.Title; - DependencyGroups = ToDependencyItems(package); - } - - private IReadOnlyList ToDependencyItems(Package package) - { - var groups = new List(); - - var targetFrameworks = package.Dependencies.Select(d => d.TargetFramework).Distinct(); - - foreach(var target in targetFrameworks) - { - // A package may have no dependencies for a target framework. This is represented - // by a single dependency item with a null "Id" and "VersionRange". - var groupId = $"https://api.nuget.org/v3/catalog0/data/2015.02.01.06.24.15/{package.Id}.{package.Version}.json#dependencygroup/{target}"; - var dependencyItems = package.Dependencies - .Where(d => d.TargetFramework == target) - .Where(d => d.Id != null && d.VersionRange != null) - .Select(d => new DependencyItem(groupId, d.Id, d.VersionRange)) - .ToList(); + var groups = new List(); - groups.Add(new DependencyGroupItem(groupId, target, dependencyItems)); - } - - return groups; - } - - [JsonProperty(PropertyName = "@id")] - public string CatalogUri { get; } - - [JsonProperty(PropertyName = "id")] - public string PackageId { get; } - - public string Version { get; } - public string Authors { get; } - public string Description { get; } - public long Downloads { get; } - public bool HasReadme { get; } - public string IconUrl { get; } - public string Language { get; } - public string LicenseUrl { get; } - public bool Listed { get; } - public string MinClientVersion { get; } - public string PackageContent { get; } - public string ProjectUrl { get; } - public string RepositoryUrl { get; } - public string RepositoryType { get; } - public DateTime Published { get; } - public bool RequireLicenseAcceptance { get; } - public string Summary { get; } - public string[] Tags { get; } - public string Title { get; } - public IReadOnlyList DependencyGroups { get; } - } + var targetFrameworks = package.Dependencies.Select(d => d.TargetFramework).Distinct(); - private class DependencyGroupItem - { - public DependencyGroupItem( - string id, - string targetFramework, - IReadOnlyList dependencyItems) + foreach (var target in targetFrameworks) { - Id = id; - Type = "PackageDependencyGroup"; - TargetFramework = targetFramework; - Dependencies = (dependencyItems.Count > 0) ? dependencyItems : null; + // A package may have no dependencies for a target framework. This is represented + // by a single dependency item with a null "Id" and "VersionRange". + var groupId = $"https://api.nuget.org/v3/catalog0/data/2015.02.01.06.24.15/{package.Id}.{package.Version}.json#dependencygroup/{target}"; + var dependencyItems = package.Dependencies + .Where(d => d.TargetFramework == target) + .Where(d => d.Id != null && d.VersionRange != null) + .Select(d => new DependencyItem(groupId, d.Id, d.VersionRange)) + .ToList(); + + groups.Add(new DependencyGroupItem(groupId, target, dependencyItems)); } - [JsonProperty(PropertyName = "@id")] - public string Id { get; } - - [JsonProperty(PropertyName = "@type")] - public string Type { get; } - - public string TargetFramework { get; } - - [JsonProperty(DefaultValueHandling=DefaultValueHandling.Ignore)] - public IReadOnlyList Dependencies { get; } - } - - private class DependencyItem - { - [JsonProperty(PropertyName = "@id")] - public string DepId { get; } - - [JsonProperty(PropertyName = "@type")] - public string Type { get; } - - public string Id { get; } - public string Range { get; } - - public DependencyItem(string groupId, string depId, string versionRange) - { - DepId = groupId + "/" + depId; - Type = "PackageDependency"; - Id = depId; - Range = versionRange; - } + return groups; } } } \ No newline at end of file diff --git a/src/BaGet/Controllers/Registration/RegistrationLeafController.cs b/src/BaGet/Controllers/Registration/RegistrationLeafController.cs index 9a416a24c..d5f48de76 100644 --- a/src/BaGet/Controllers/Registration/RegistrationLeafController.cs +++ b/src/BaGet/Controllers/Registration/RegistrationLeafController.cs @@ -4,8 +4,8 @@ using BaGet.Core.Mirror; using BaGet.Core.Services; using BaGet.Extensions; +using BaGet.Protocol; using Microsoft.AspNetCore.Mvc; -using Newtonsoft.Json; using NuGet.Versioning; namespace BaGet.Controllers.Registration @@ -33,8 +33,8 @@ public async Task Get(string id, string version, CancellationToke return NotFound(); } - // Allow read-through caching to happen if it is confiured. - await _mirror.MirrorAsync(id, nugetVersion, cancellationToken); + // Allow read-through caching to happen if it is configured. + await _mirror.MirrorAsync(id, cancellationToken); var package = await _packages.FindOrNullAsync(id, nugetVersion); @@ -43,49 +43,15 @@ public async Task Get(string id, string version, CancellationToke return NotFound(); } - // Documentation: https://docs.microsoft.com/en-us/nuget/api/registration-base-url-resource var result = new RegistrationLeaf( registrationUri: Url.PackageRegistration(id, nugetVersion), listed: package.Listed, downloads: package.Downloads, - packageContentUri: Url.PackageDownload(id, nugetVersion), + packageContentUrl: Url.PackageDownload(id, nugetVersion), published: package.Published, - registrationIndexUri: Url.PackageRegistration(id)); + registrationIndexUrl: Url.PackageRegistration(id)); return Json(result); } - - public class RegistrationLeaf - { - public RegistrationLeaf( - string registrationUri, - bool listed, - long downloads, - string packageContentUri, - DateTimeOffset published, - string registrationIndexUri) - { - RegistrationUri = registrationUri ?? throw new ArgumentNullException(nameof(registrationIndexUri)); - Listed = listed; - Published = published; - Downloads = downloads; - PackageContent = packageContentUri ?? throw new ArgumentNullException(nameof(packageContentUri)); - RegistrationIndexUri = registrationIndexUri ?? throw new ArgumentNullException(nameof(registrationIndexUri)); - } - - [JsonProperty(PropertyName = "@id")] - public string RegistrationUri { get; } - - public bool Listed { get; } - - public long Downloads { get; } - - public string PackageContent { get; } - - public DateTimeOffset Published { get; } - - [JsonProperty(PropertyName = "registration")] - public string RegistrationIndexUri { get; } - } } } \ No newline at end of file diff --git a/src/BaGet/Controllers/SearchController.cs b/src/BaGet/Controllers/SearchController.cs index ebf47c093..9608ec86a 100644 --- a/src/BaGet/Controllers/SearchController.cs +++ b/src/BaGet/Controllers/SearchController.cs @@ -1,14 +1,16 @@ using System; -using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; using BaGet.Core.Services; using BaGet.Extensions; +using BaGet.Protocol; using Microsoft.AspNetCore.Mvc; -using Newtonsoft.Json; namespace BaGet.Controllers { + using ProtocolSearchResult = Protocol.SearchResult; + using QuerySearchResult = Core.Services.SearchResult; + public class SearchController : Controller { private readonly ISearchService _searchService; @@ -18,83 +20,48 @@ public SearchController(ISearchService searchService) _searchService = searchService ?? throw new ArgumentNullException(nameof(searchService)); } - public async Task Get([FromQuery(Name = "q")] string query = null) + public async Task Get([FromQuery(Name = "q")] string query = null) { query = query ?? string.Empty; var results = await _searchService.SearchAsync(query); + var response = new SearchResponse( + totalHits: results.Count, + data: results.Select(ToSearchResult).ToList()); - return new - { - TotalHits = results.Count, - Data = results.Select(p => new SearchResultModel(p, Url)) - }; + return Json(response); } public async Task Autocomplete([FromQuery(Name = "q")] string query = null) { var results = await _searchService.AutocompleteAsync(query); + var response = new AutocompleteResult(results.Count, results); - return Json(new - { - TotalHits = results.Count, - Data = results, - }); + return Json(response); } - private class SearchResultModel + private ProtocolSearchResult ToSearchResult(QuerySearchResult result) { - private readonly SearchResult _result; - private readonly IUrlHelper _url; - - public SearchResultModel(SearchResult result, IUrlHelper url) - { - _result = result ?? throw new ArgumentNullException(nameof(result)); - _url = url ?? throw new ArgumentNullException(nameof(url)); - - var versions = result.Versions.Select( - v => new SearchResultVersionModel( - url.PackageRegistration(result.Id, v.Version), - v.Version.ToNormalizedString(), - v.Downloads)); - - Versions = versions.ToList().AsReadOnly(); - } - - public string Id => _result.Id; - public string Version => _result.Version.ToNormalizedString(); - public string Description => _result.Description; - public string Authors => _result.Authors; - public string IconUrl => _result.IconUrl; - public string LicenseUrl => _result.LicenseUrl; - public string ProjectUrl => _result.ProjectUrl; - public string Registration => _url.PackageRegistration(_result.Id); - public string Summary => _result.Summary; - public string[] Tags => _result.Tags; - public string Title => _result.Title; - public long TotalDownloads => _result.TotalDownloads; - - public IReadOnlyList Versions { get; } - } - - private class SearchResultVersionModel - { - public SearchResultVersionModel(string registrationUrl, string version, long downloads) - { - if (string.IsNullOrEmpty(registrationUrl)) throw new ArgumentNullException(nameof(registrationUrl)); - if (string.IsNullOrEmpty(version)) throw new ArgumentNullException(nameof(version)); - - RegistrationUrl = registrationUrl; - Version = version; - Downloads = downloads; - } - - [JsonProperty(PropertyName = "id")] - public string RegistrationUrl { get; } - - public string Version { get; } - - public long Downloads { get; } + var versions = result.Versions.Select( + v => new Protocol.SearchResultVersion( + registrationLeafUrl: Url.PackageRegistration(result.Id, v.Version), + version: v.Version, + downloads: v.Downloads)); + + return new ProtocolSearchResult( + id: result.Id, + version: result.Version, + description: result.Description, + authors: result.Authors, + iconUrl: result.IconUrl, + licenseUrl: result.LicenseUrl, + projectUrl: result.ProjectUrl, + registrationUrl: Url.PackageRegistration(result.Id), + summary: result.Summary, + tags: result.Tags, + title: result.Title, + totalDownloads: result.TotalDownloads, + versions: versions.ToList()); } } } \ No newline at end of file diff --git a/src/BaGet/Extensions/ServiceCollectionExtensions.cs b/src/BaGet/Extensions/ServiceCollectionExtensions.cs index eceb01ee4..64b18b3de 100644 --- a/src/BaGet/Extensions/ServiceCollectionExtensions.cs +++ b/src/BaGet/Extensions/ServiceCollectionExtensions.cs @@ -12,6 +12,7 @@ using BaGet.Core.Mirror; using BaGet.Core.Services; using BaGet.Entities; +using BaGet.Protocol; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Cors.Infrastructure; using Microsoft.AspNetCore.HttpOverrides; @@ -199,6 +200,9 @@ public static IServiceCollection ConfigureSearchProviders(this IServiceCollectio /// The defined services. public static IServiceCollection AddMirrorServices(this IServiceCollection services) { + services.AddTransient(); + services.AddTransient(); + services.AddTransient(provider => { var mirrorOptions = provider @@ -210,15 +214,31 @@ public static IServiceCollection AddMirrorServices(this IServiceCollection servi if (!mirrorOptions.Enabled) { - return new FakeMirrorService(); + return provider.GetRequiredService(); + } + else + { + return provider.GetRequiredService(); } + }); + + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + + services.AddSingleton(provider => + { + var mirrorOptions = provider + .GetRequiredService>() + .Value + .Mirror; + + mirrorOptions.EnsureValid(); - return new MirrorService( - mirrorOptions.PackageSource, - provider.GetRequiredService(), - provider.GetRequiredService(), - provider.GetRequiredService(), - provider.GetRequiredService>()); + return new ServiceIndexService( + mirrorOptions.PackageSource.ToString(), + provider.GetRequiredService()); }); services.AddTransient(); diff --git a/src/BaGet/appsettings.json b/src/BaGet/appsettings.json index 8378b5000..a8533fe1d 100644 --- a/src/BaGet/appsettings.json +++ b/src/BaGet/appsettings.json @@ -17,7 +17,7 @@ "Mirror": { "Enabled": false, - "PackageSource": "https://api.nuget.org/v3-flatcontainer/" + "PackageSource": "https://api.nuget.org/v3/index.json" }, "Logging": { diff --git a/src/readme.md b/src/readme.md index 4ae18482a..705966a4f 100644 --- a/src/readme.md +++ b/src/readme.md @@ -2,7 +2,8 @@ These folders contain the core components of BaGet: -* `BaGet` - the app's entry point, API controllers, and CLI commands. +* `BaGet` - the app's entry point, API controllers, and CLI commands * `BaGet.Core` - the core logic and services +* `BaGet.Protocol` - Libraries to interact with NuGet server APIs * `BaGet.UI` - BaGet's frontend * `BaGet.Azure` - the Azure implementation of BaGet diff --git a/tests/BaGet.Protocol.Tests/BaGet.Protocol.Tests.csproj b/tests/BaGet.Protocol.Tests/BaGet.Protocol.Tests.csproj new file mode 100644 index 000000000..f4dfe5e09 --- /dev/null +++ b/tests/BaGet.Protocol.Tests/BaGet.Protocol.Tests.csproj @@ -0,0 +1,22 @@ + + + + netcoreapp2.1 + 7.1 + + false + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/tests/BaGet.Protocol.Tests/PackageContentClientTests.cs b/tests/BaGet.Protocol.Tests/PackageContentClientTests.cs new file mode 100644 index 000000000..b3dd06939 --- /dev/null +++ b/tests/BaGet.Protocol.Tests/PackageContentClientTests.cs @@ -0,0 +1,35 @@ +using System; +using System.Net.Http; +using System.Threading.Tasks; +using Xunit; + +namespace BaGet.Protocol.Tests +{ + public class PackageContentTests + { + private readonly PackageContentClient _target; + + public PackageContentTests() + { + var httpClient = new HttpClient(); + _target = new PackageContentClient(httpClient); + } + + [Fact] + public async Task GetsPackageVersions() + { + var result = await _target.GetPackageVersionsOrNullAsync("https://api.nuget.org/v3-flatcontainer/newtonsoft.json/index.json"); + + Assert.NotNull(result); + Assert.NotEmpty(result.Versions); + } + + [Fact] + public async Task ReturnsNullIfPackageDoesNotExist() + { + var url = $"https://api.nuget.org/v3-flatcontainer/{Guid.NewGuid()}/index.json"; + + Assert.Null(await _target.GetPackageVersionsOrNullAsync(url)); + } + } +} diff --git a/tests/BaGet.Protocol.Tests/PackageMetadataServiceIntegrationTests.cs b/tests/BaGet.Protocol.Tests/PackageMetadataServiceIntegrationTests.cs new file mode 100644 index 000000000..0d2d4116d --- /dev/null +++ b/tests/BaGet.Protocol.Tests/PackageMetadataServiceIntegrationTests.cs @@ -0,0 +1,45 @@ +using System.Net.Http; +using System.Threading.Tasks; +using Xunit; + +namespace BaGet.Protocol.Tests +{ + public class PackageMetadataServiceIntegrationTests + { + private readonly PackageMetadataService _target; + + public PackageMetadataServiceIntegrationTests() + { + var httpClient = new HttpClient(); + var serviceIndexClient = new ServiceIndexClient(httpClient); + var registrationClient = new RegistrationClient(httpClient); + var packageContentClient = new PackageContentClient(httpClient); + + var serviceIndex = "https://api.nuget.org/v3/index.json"; + var serviceIndexService = new ServiceIndexService(serviceIndex, serviceIndexClient); + + _target = new PackageMetadataService( + serviceIndexService, + registrationClient, + packageContentClient); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task GetsAllVersionsForNewtonsoftJson(bool includeUnlisted) + { + var result = await _target.GetAllVersionsAsync("Newtonsoft.Json", includeUnlisted); + + Assert.NotEmpty(result); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task GetsAllVersionsForFake(bool includeUnlisted) + { + var result = await _target.GetAllVersionsAsync("Fake", includeUnlisted); + } + } +} diff --git a/tests/BaGet.Protocol.Tests/RegistrationClientTests.cs b/tests/BaGet.Protocol.Tests/RegistrationClientTests.cs new file mode 100644 index 000000000..6b72d8da9 --- /dev/null +++ b/tests/BaGet.Protocol.Tests/RegistrationClientTests.cs @@ -0,0 +1,85 @@ +using System.Net.Http; +using System.Threading.Tasks; +using NuGet.Versioning; +using Xunit; + +namespace BaGet.Protocol.Tests +{ + public class RegistrationClientTests + { + private readonly RegistrationClient _target; + + public RegistrationClientTests() + { + var httpClient = new HttpClient(); + _target = new RegistrationClient(httpClient); + } + + [Fact] + public async Task GetsNewtonsoftJsonRegistrationIndex() + { + var url = "https://api.nuget.org/v3/registration3/newtonsoft.json/index.json"; + + var result = await _target.GetRegistrationIndexAsync(url); + + Assert.NotNull(result); + Assert.Equal(1, result.Pages.Count); + Assert.True(result.Pages[0].Count >= 63); + Assert.True(result.Pages[0].ItemsOrNull.Count >= 63); + Assert.Equal(new NuGetVersion("3.5.8"), result.Pages[0].Lower); + Assert.Equal(new NuGetVersion("12.0.1-beta1"), result.Pages[0].Upper); + } + + [Fact] + public async Task GetsFakeRegistrationIndex() + { + var url = "https://api.nuget.org/v3/registration3/fake/index.json"; + + var result = await _target.GetRegistrationIndexAsync(url); + + Assert.NotNull(result); + Assert.True(result.Pages.Count >= 27); + Assert.Null(result.Pages[0].ItemsOrNull); + Assert.Equal(64, result.Pages[0].Count); + Assert.Equal(new NuGetVersion("1.0.0-alpha-10"), result.Pages[0].Lower); + Assert.Equal(new NuGetVersion("1.66.1"), result.Pages[0].Upper); + } + + [Fact] + public async Task GetsFakeRegistrationPage() + { + var url = "https://api.nuget.org/v3/registration3/fake/index.json"; + + var index = await _target.GetRegistrationIndexAsync(url); + var result = await _target.GetRegistrationIndexPageAsync(index.Pages[0].PageUrl); + + Assert.NotNull(result); + Assert.Equal(64, result.Count); + Assert.Equal(new NuGetVersion("1.0.0-alpha-10"), result.Lower); + Assert.Equal(new NuGetVersion("1.66.1"), result.Upper); + } + + [Fact] + public async Task GetsNewtonsoftRegistrationLeaf() + { + var url = "https://api.nuget.org/v3/registration3/newtonsoft.json/index.json"; + + var index = await _target.GetRegistrationIndexAsync(url); + var leaf = await _target.GetRegistrationLeafAsync(index.Pages[0].ItemsOrNull[0].LeafUrl); + + Assert.Equal(url, leaf.RegistrationIndexUrl); + } + + [Fact] + public async Task GetFakeRegistrationLeaf() + { + var url = "https://api.nuget.org/v3/registration3/fake/index.json"; + + var index = await _target.GetRegistrationIndexAsync(url); + var page = await _target.GetRegistrationIndexPageAsync(index.Pages[0].PageUrl); + var leaf = await _target.GetRegistrationLeafAsync(page.ItemsOrNull[0].LeafUrl); + + Assert.Equal(url, leaf.RegistrationIndexUrl); + } + } +} diff --git a/tests/BaGet.Protocol.Tests/SearchClientTests.cs b/tests/BaGet.Protocol.Tests/SearchClientTests.cs new file mode 100644 index 000000000..27de73337 --- /dev/null +++ b/tests/BaGet.Protocol.Tests/SearchClientTests.cs @@ -0,0 +1,45 @@ +using System.Net.Http; +using System.Threading.Tasks; +using NuGet.Versioning; +using Xunit; + +namespace BaGet.Protocol.Tests +{ + public class SearchClientTests + { + private readonly SearchClient _target; + + public SearchClientTests() + { + var httpClient = new HttpClient(); + _target = new SearchClient(httpClient); + } + + [Fact] + public async Task GetsNewtonsoftJsonSearchResults() + { + var searchQuery = "https://api-v2v3search-0.nuget.org/query?q=newtonsoft"; + var registrationurl = "https://api.nuget.org/v3/registration3/newtonsoft.json/index.json"; + + var result = await _target.GetSearchResultsAsync(searchQuery); + + Assert.True(result.TotalHits > 0); + Assert.True(result.Data.Count > 0); + Assert.Equal(registrationurl, result.Data[0].RegistrationUrl); + Assert.Equal("Newtonsoft.Json", result.Data[0].Id); + Assert.Equal(new NuGetVersion("11.0.2"), result.Data[0].Version); + } + + [Fact] + public async Task GetsNewtonsoftJsonAutocompleteResults() + { + var query = "https://api-v2v3search-0.nuget.org/autocomplete?q=newt"; + + var result = await _target.GetAutocompleteResultsAsync(query); + + Assert.True(result.TotalHits > 0); + Assert.True(result.Data.Count > 0); + Assert.Contains(result.Data, id => id == "Newtonsoft.Json"); + } + } +} diff --git a/tests/BaGet.Protocol.Tests/ServiceIndexClientTests.cs b/tests/BaGet.Protocol.Tests/ServiceIndexClientTests.cs new file mode 100644 index 000000000..b6630aafd --- /dev/null +++ b/tests/BaGet.Protocol.Tests/ServiceIndexClientTests.cs @@ -0,0 +1,23 @@ +using System.Net.Http; +using System.Threading.Tasks; +using Xunit; + +namespace BaGet.Protocol.Tests +{ + public class ServiceIndexClientTests + { + private readonly ServiceIndexClient _target; + + public ServiceIndexClientTests() + { + var httpClient = new HttpClient(); + _target = new ServiceIndexClient(httpClient); + } + + [Fact] + public async Task GetsServiceIndex() + { + var result = await _target.GetServiceIndexAsync("https://api.nuget.org/v3/index.json"); + } + } +}