Skip to content

Commit 4501642

Browse files
loic-sharmaloic-sharma
authored
Refactor package mirroring (loic-sharma#711)
After this change there's two services to access packages' state: * `IPackageDatabase` - This is a "low level" service for packages' state in BaGet's database * `IPackageService` - This is a "high level" service for packages' state, including both the database and the upstream feed if any This allows us to centralize the mirroring logic into the `IPackageService` (before mirroring logic was sprinkled throughout the codebase). Replaces loic-sharma#699
1 parent dec4013 commit 4501642

15 files changed

+312
-208
lines changed

‎.gitignore

+3-1
Original file line numberDiff line numberDiff line change
@@ -268,5 +268,7 @@ __pycache__/
268268
# Cake - Uncomment if you are using it
269269
# tools/
270270

271-
# Ignore database file
271+
# Ignore database files
272272
**/baget.db
273+
**/baget.db-shm
274+
**/baget.db-wal

‎src/BaGet.Azure/Table/TablePackageDatabase.cs

+3-3
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ public async Task<PackageAddResult> AddAsync(Package package, CancellationToken
4848
return PackageAddResult.Success;
4949
}
5050

51-
public async Task<bool> AddDownloadAsync(
51+
public async Task AddDownloadAsync(
5252
string id,
5353
NuGetVersion version,
5454
CancellationToken cancellationToken)
@@ -68,13 +68,13 @@ public async Task<bool> AddDownloadAsync(
6868

6969
if (entity == null)
7070
{
71-
return false;
71+
return;
7272
}
7373

7474
entity.Downloads += 1;
7575

7676
await _table.ExecuteAsync(TableOperation.Merge(entity), cancellationToken);
77-
return true;
77+
return;
7878
}
7979
catch (StorageException e)
8080
when (attempt < MaxPreconditionFailures && e.IsPreconditionFailedException())

‎src/BaGet.Core/Content/DefaultPackageContentService.cs

+10-26
Original file line numberDiff line numberDiff line change
@@ -9,22 +9,17 @@
99
namespace BaGet.Core
1010
{
1111
/// <summary>
12-
/// Implements the NuGet Package Content resource. Supports read-through caching.
13-
/// Tracks state in a database (<see cref="IPackageDatabase"/>) and stores packages
14-
/// using <see cref="IPackageStorageService"/>.
12+
/// Implements the NuGet Package Content resource in NuGet's V3 protocol.
1513
/// </summary>
1614
public class DefaultPackageContentService : IPackageContentService
1715
{
18-
private readonly IMirrorService _mirror;
19-
private readonly IPackageDatabase _packages;
16+
private readonly IPackageService _packages;
2017
private readonly IPackageStorageService _storage;
2118

2219
public DefaultPackageContentService(
23-
IMirrorService mirror,
24-
IPackageDatabase packages,
20+
IPackageService packages,
2521
IPackageStorageService storage)
2622
{
27-
_mirror = mirror ?? throw new ArgumentNullException(nameof(mirror));
2823
_packages = packages ?? throw new ArgumentNullException(nameof(packages));
2924
_storage = storage ?? throw new ArgumentNullException(nameof(storage));
3025
}
@@ -33,7 +28,7 @@ public async Task<PackageVersionsResponse> GetPackageVersionsOrNullAsync(
3328
string id,
3429
CancellationToken cancellationToken = default)
3530
{
36-
var versions = await _mirror.FindPackageVersionsAsync(id, cancellationToken);
31+
var versions = await _packages.FindPackageVersionsAsync(id, cancellationToken);
3732
if (!versions.Any())
3833
{
3934
return null;
@@ -53,22 +48,17 @@ public async Task<Stream> GetPackageContentStreamOrNullAsync(
5348
NuGetVersion version,
5449
CancellationToken cancellationToken = default)
5550
{
56-
// Allow read-through caching if it is configured.
57-
await _mirror.MirrorAsync(id, version, cancellationToken);
58-
59-
if (!await _packages.AddDownloadAsync(id, version, cancellationToken))
51+
if (!await _packages.ExistsAsync(id, version, cancellationToken))
6052
{
6153
return null;
6254
}
6355

56+
await _packages.AddDownloadAsync(id, version, cancellationToken);
6457
return await _storage.GetPackageStreamAsync(id, version, cancellationToken);
6558
}
6659

6760
public async Task<Stream> GetPackageManifestStreamOrNullAsync(string id, NuGetVersion version, CancellationToken cancellationToken = default)
6861
{
69-
// Allow read-through caching if it is configured.
70-
await _mirror.MirrorAsync(id, version, cancellationToken);
71-
7262
if (!await _packages.ExistsAsync(id, version, cancellationToken))
7363
{
7464
return null;
@@ -79,11 +69,8 @@ public async Task<Stream> GetPackageManifestStreamOrNullAsync(string id, NuGetVe
7969

8070
public async Task<Stream> GetPackageReadmeStreamOrNullAsync(string id, NuGetVersion version, CancellationToken cancellationToken = default)
8171
{
82-
// Allow read-through caching if it is configured.
83-
await _mirror.MirrorAsync(id, version, cancellationToken);
84-
85-
var package = await _packages.FindOrNullAsync(id, version, includeUnlisted: true, cancellationToken);
86-
if (!package.HasReadme)
72+
var package = await _packages.FindPackageOrNullAsync(id, version, cancellationToken);
73+
if (package == null || !package.HasReadme)
8774
{
8875
return null;
8976
}
@@ -93,11 +80,8 @@ public async Task<Stream> GetPackageReadmeStreamOrNullAsync(string id, NuGetVers
9380

9481
public async Task<Stream> GetPackageIconStreamOrNullAsync(string id, NuGetVersion version, CancellationToken cancellationToken = default)
9582
{
96-
// Allow read-through caching if it is configured.
97-
await _mirror.MirrorAsync(id, version, cancellationToken);
98-
99-
var package = await _packages.FindOrNullAsync(id, version, includeUnlisted: true, cancellationToken);
100-
if (!package.HasEmbeddedIcon)
83+
var package = await _packages.FindPackageOrNullAsync(id, version, cancellationToken);
84+
if (package == null || !package.HasEmbeddedIcon)
10185
{
10286
return null;
10387
}

‎src/BaGet.Core/Extensions/DependencyInjectionExtensions.cs

+18-13
Original file line numberDiff line numberDiff line change
@@ -92,22 +92,22 @@ private static void AddBaGetServices(this IServiceCollection services)
9292
services.TryAddTransient<IPackageDeletionService, PackageDeletionService>();
9393
services.TryAddTransient<IPackageIndexingService, PackageIndexingService>();
9494
services.TryAddTransient<IPackageMetadataService, DefaultPackageMetadataService>();
95+
services.TryAddTransient<IPackageService, PackageService>();
9596
services.TryAddTransient<IPackageStorageService, PackageStorageService>();
9697
services.TryAddTransient<IServiceIndexService, BaGetServiceIndex>();
9798
services.TryAddTransient<ISymbolIndexingService, SymbolIndexingService>();
9899
services.TryAddTransient<ISymbolStorageService, SymbolStorageService>();
99100

100101
services.TryAddTransient<DatabaseSearchService>();
101102
services.TryAddTransient<FileStorageService>();
102-
services.TryAddTransient<MirrorService>();
103+
services.TryAddTransient<PackageService>();
103104
services.TryAddTransient<V2UpstreamClient>();
104105
services.TryAddTransient<V3UpstreamClient>();
105-
services.TryAddTransient<DisabledMirrorService>();
106+
services.TryAddTransient<DisabledUpstreamClient>();
106107
services.TryAddSingleton<NullStorageService>();
107108
services.TryAddTransient<PackageDatabase>();
108109

109-
services.TryAddTransient(IMirrorServiceFactory);
110-
services.TryAddTransient(IMirrorClientFactory);
110+
services.TryAddTransient(UpstreamClientFactory);
111111
}
112112

113113
private static void AddDefaultProviders(this IServiceCollection services)
@@ -195,20 +195,25 @@ private static NuGetClientFactory NuGetClientFactoryFactory(IServiceProvider pro
195195
options.Value.PackageSource.ToString());
196196
}
197197

198-
private static IMirrorService IMirrorServiceFactory(IServiceProvider provider)
198+
private static IUpstreamClient UpstreamClientFactory(IServiceProvider provider)
199199
{
200200
var options = provider.GetRequiredService<IOptionsSnapshot<MirrorOptions>>();
201-
var service = options.Value.Enabled ? typeof(MirrorService) : typeof(DisabledMirrorService);
202201

203-
return (IMirrorService)provider.GetRequiredService(service);
204-
}
202+
// TODO: Convert to switch expression.
203+
if (!options.Value.Enabled)
204+
{
205+
return provider.GetRequiredService<DisabledUpstreamClient>();
206+
}
205207

206-
private static IUpstreamClient IMirrorClientFactory(IServiceProvider provider)
207-
{
208-
var options = provider.GetRequiredService<IOptionsSnapshot<MirrorOptions>>();
209-
var service = options.Value.Legacy ? typeof(V2UpstreamClient) : typeof(V3UpstreamClient);
208+
else if (options.Value.Legacy)
209+
{
210+
return provider.GetRequiredService<V2UpstreamClient>();
211+
}
210212

211-
return (IUpstreamClient)provider.GetRequiredService(service);
213+
else
214+
{
215+
return provider.GetRequiredService<V3UpstreamClient>();
216+
}
212217
}
213218
}
214219
}

‎src/BaGet.Core/IPackageDatabase.cs

+2-2
Original file line numberDiff line numberDiff line change
@@ -83,8 +83,8 @@ Task<Package> FindOrNullAsync(
8383
/// <param name="id">The id of the package to update.</param>
8484
/// <param name="version">The id of the package to update.</param>
8585
/// <param name="cancellationToken">A token to cancel the task.</param>
86-
/// <returns>False if the package does not exist.</returns>
87-
Task<bool> AddDownloadAsync(string id, NuGetVersion version, CancellationToken cancellationToken);
86+
/// <returns>Task that completes when the package's download has been incremented.</returns>
87+
Task AddDownloadAsync(string id, NuGetVersion version, CancellationToken cancellationToken);
8888

8989
/// <summary>
9090
/// Completely remove the package from the database.

‎src/BaGet.Core/Metadata/DefaultPackageMetadataService.cs

+4-10
Original file line numberDiff line numberDiff line change
@@ -10,16 +10,13 @@ namespace BaGet.Core
1010
/// <inheritdoc />
1111
public class DefaultPackageMetadataService : IPackageMetadataService
1212
{
13-
private readonly IMirrorService _mirror;
14-
private readonly IPackageDatabase _packages;
13+
private readonly IPackageService _packages;
1514
private readonly RegistrationBuilder _builder;
1615

1716
public DefaultPackageMetadataService(
18-
IMirrorService mirror,
19-
IPackageDatabase packages,
17+
IPackageService packages,
2018
RegistrationBuilder builder)
2119
{
22-
_mirror = mirror ?? throw new ArgumentNullException(nameof(mirror));
2320
_packages = packages ?? throw new ArgumentNullException(nameof(packages));
2421
_builder = builder ?? throw new ArgumentNullException(nameof(builder));
2522
}
@@ -28,7 +25,7 @@ public async Task<BaGetRegistrationIndexResponse> GetRegistrationIndexOrNullAsyn
2825
string packageId,
2926
CancellationToken cancellationToken = default)
3027
{
31-
var packages = await _mirror.FindPackagesAsync(packageId, cancellationToken);
28+
var packages = await _packages.FindPackagesAsync(packageId, cancellationToken);
3229
if (!packages.Any())
3330
{
3431
return null;
@@ -45,10 +42,7 @@ public async Task<RegistrationLeafResponse> GetRegistrationLeafOrNullAsync(
4542
NuGetVersion version,
4643
CancellationToken cancellationToken = default)
4744
{
48-
// Allow read-through caching to happen if it is configured.
49-
await _mirror.MirrorAsync(id, version, cancellationToken);
50-
51-
var package = await _packages.FindOrNullAsync(id, version, includeUnlisted: true, cancellationToken);
45+
var package = await _packages.FindPackageOrNullAsync(id, version, cancellationToken);
5246
if (package == null)
5347
{
5448
return null;

‎src/BaGet.Core/Mirror/DisabledMirrorService.cs

-39
This file was deleted.

‎src/BaGet.Core/Mirror/IMirrorService.cs

-46
This file was deleted.

‎src/BaGet.Core/Mirror/IPackageService.cs

+70
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
using System.Collections.Generic;
2+
using System.Threading;
3+
using System.Threading.Tasks;
4+
using NuGet.Versioning;
5+
6+
namespace BaGet.Core
7+
{
8+
/// <summary>
9+
/// The service that combines the state of indexed packages and
10+
/// upstream packages.
11+
/// For upstream packages, see <see cref="IUpstreamClient"/>.
12+
/// For indexed packages, see <see cref="IPackageDatabase"/>.
13+
/// </summary>
14+
public interface IPackageService
15+
{
16+
/// <summary>
17+
/// Attempt to find a package's versions using mirroring. This will merge
18+
/// results from the configured upstream source with the locally indexed packages.
19+
/// </summary>
20+
/// <param name="id">The package's id to lookup</param>
21+
/// <param name="cancellationToken">The token to cancel the lookup</param>
22+
/// <returns>
23+
/// The package's versions, or an empty list if the package cannot be found.
24+
/// This includes unlisted versions.
25+
/// </returns>
26+
Task<IReadOnlyList<NuGetVersion>> FindPackageVersionsAsync(string id, CancellationToken cancellationToken);
27+
28+
/// <summary>
29+
/// Attempt to find a package's metadata using mirroring. This will merge
30+
/// results from the configured upstream source with the locally indexed packages.
31+
/// </summary>
32+
/// <param name="id">The package's id to lookup</param>
33+
/// <param name="cancellationToken">The token to cancel the lookup</param>
34+
/// <returns>
35+
/// The metadata for all versions of a package, including unlisted versions.
36+
/// Returns an empty list if the package cannot be found.
37+
/// </returns>
38+
Task<IReadOnlyList<Package>> FindPackagesAsync(string id, CancellationToken cancellationToken);
39+
40+
/// <summary>
41+
/// Attempt to find a package's metadata using mirroring. This will merge
42+
/// results from the configured upstream source with the locally indexed packages.
43+
/// </summary>
44+
/// <param name="id">The package's id to lookup</param>
45+
/// <param name="version">The package's version to lookup</param>
46+
/// <param name="cancellationToken">The token to cancel the lookup</param>
47+
/// <returns>
48+
/// The metadata for single version of a package.
49+
/// Returns null if the package does not exist.
50+
/// </returns>
51+
Task<Package> FindPackageOrNullAsync(string id, NuGetVersion version, CancellationToken cancellationToken);
52+
53+
/// <summary>
54+
/// Determine whether a package exists locally or on the upstream source.
55+
/// </summary>
56+
/// <param name="id">The package id to search.</param>
57+
/// <param name="version">The package version to search.</param>
58+
/// <param name="cancellationToken">A token to cancel the task.</param>
59+
/// <returns>Whether the package exists in the database.</returns>
60+
Task<bool> ExistsAsync(string id, NuGetVersion version, CancellationToken cancellationToken);
61+
62+
/// <summary>
63+
/// Increment a package's download count.
64+
/// </summary>
65+
/// <param name="packageId">The id of the package to update.</param>
66+
/// <param name="version">The id of the package to update.</param>
67+
/// <param name="cancellationToken">A token to cancel the task.</param>
68+
Task AddDownloadAsync(string packageId, NuGetVersion version, CancellationToken cancellationToken);
69+
}
70+
}

0 commit comments

Comments
 (0)