Skip to content

Commit 0c7de3d

Browse files
loic-sharmaloic-sharma
authored
[Azure] Add Azure Table Storage database (loic-sharma#347)
Adds using Azure Table Storage as a database. The goal is to mirror nuget.org cheaply at a high scale. Replaces loic-sharma#257
1 parent 241512a commit 0c7de3d

19 files changed

+619
-22
lines changed

‎src/BaGet.Azure/BaGet.Azure.csproj

+1
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
</PropertyGroup>
99

1010
<ItemGroup>
11+
<PackageReference Include="Microsoft.Azure.Cosmos.Table" Version="1.0.0" />
1112
<PackageReference Include="Microsoft.Azure.Search" Version="5.0.1" />
1213
<PackageReference Include="Microsoft.Azure.Storage.Blob" Version="9.4.1" />
1314
<PackageReference Include="Microsoft.Extensions.Options.ConfigurationExtensions" Version="$(MicrosoftExtensionsPackageVersion)" />

‎src/BaGet.Azure/BlobStorageService.cs

+2-7
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
using System;
22
using System.IO;
3-
using System.Net;
43
using System.Threading;
54
using System.Threading.Tasks;
5+
using BaGet.Azure.Extensions;
66
using BaGet.Core;
77
using Microsoft.WindowsAzure.Storage;
88
using Microsoft.WindowsAzure.Storage.Blob;
@@ -64,7 +64,7 @@ await blob.UploadFromStreamAsync(
6464

6565
return StoragePutResult.Success;
6666
}
67-
catch (StorageException e) when (IsAlreadyExistsException(e))
67+
catch (StorageException e) when (e.IsAlreadyExistsException())
6868
{
6969
using (var targetStream = await blob.OpenReadAsync(cancellationToken))
7070
{
@@ -82,10 +82,5 @@ await _container
8282
.GetBlockBlobReference(path)
8383
.DeleteIfExistsAsync(cancellationToken);
8484
}
85-
86-
private bool IsAlreadyExistsException(StorageException e)
87-
{
88-
return e?.RequestInformation?.HttpStatusCode == (int?)HttpStatusCode.Conflict;
89-
}
9085
}
9186
}

‎src/BaGet.Azure/Entities/IDownloadCount.cs

+15
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
using BaGet.Core;
2+
3+
namespace BaGet.Azure
4+
{
5+
/// <summary>
6+
/// Allows updating the <see cref="Package.Downloads"/> column.
7+
/// </summary>
8+
public partial class TablePackageService
9+
{
10+
private interface IDownloadCount
11+
{
12+
long Downloads { get; set; }
13+
}
14+
}
15+
}

‎src/BaGet.Azure/Entities/IListed.cs

+15
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
using BaGet.Core;
2+
3+
namespace BaGet.Azure
4+
{
5+
/// <summary>
6+
/// Allows updating the <see cref="Package.Listed"/> column.
7+
/// </summary>
8+
public partial class TablePackageService
9+
{
10+
private interface IListed
11+
{
12+
bool Listed { get; set; }
13+
}
14+
}
15+
}

‎src/BaGet.Azure/Entities/PackageDownloadsEntity.cs

+22
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
using BaGet.Core;
2+
using Microsoft.Azure.Cosmos.Table;
3+
4+
namespace BaGet.Azure
5+
{
6+
/// <summary>
7+
/// The Azure Table Storage entity to update the <see cref="Package.Downloads"/> column.
8+
/// The <see cref="TableEntity.PartitionKey"/> is the <see cref="Package.Id"/> and
9+
/// the <see cref="TableEntity.RowKey"/> is the <see cref="Package.Version"/>.
10+
/// </summary>
11+
public partial class TablePackageService
12+
{
13+
private class PackageDownloadsEntity : TableEntity, IDownloadCount
14+
{
15+
public PackageDownloadsEntity()
16+
{
17+
}
18+
19+
public long Downloads { get; set; }
20+
}
21+
}
22+
}

‎src/BaGet.Azure/Entities/PackageEntity.cs

+197
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,197 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.Linq;
4+
using BaGet.Core;
5+
using Microsoft.Azure.Cosmos.Table;
6+
using Newtonsoft.Json;
7+
8+
namespace BaGet.Azure
9+
{
10+
public partial class TablePackageService
11+
{
12+
/// <summary>
13+
/// The Azure Table Storage entity that maps to a <see cref="Package"/>.
14+
/// The <see cref="TableEntity.PartitionKey"/> is the <see cref="Package.Id"/> and
15+
/// the <see cref="TableEntity.RowKey"/> is the <see cref="Package.Version"/>.
16+
/// </summary>
17+
private class PackageEntity : TableEntity, IDownloadCount, IListed
18+
{
19+
public PackageEntity()
20+
{
21+
}
22+
23+
public static PackageEntity FromPackage(Package package)
24+
{
25+
if (package == null) throw new ArgumentNullException(nameof(package));
26+
27+
var version = package.Version;
28+
var normalizedVersion = version.ToNormalizedString();
29+
30+
return new PackageEntity
31+
{
32+
PartitionKey = package.Id.ToLowerInvariant(),
33+
RowKey = normalizedVersion.ToLowerInvariant(),
34+
35+
Id = package.Id,
36+
NormalizedVersion = normalizedVersion,
37+
OriginalVersion = version.ToFullString(),
38+
Authors = JsonConvert.SerializeObject(package.Authors),
39+
Description = package.Description,
40+
Downloads = package.Downloads,
41+
HasReadme = package.HasReadme,
42+
IsPrerelease = package.IsPrerelease,
43+
Language = package.Language,
44+
Listed = package.Listed,
45+
MinClientVersion = package.MinClientVersion,
46+
Published = package.Published,
47+
RequireLicenseAcceptance = package.RequireLicenseAcceptance,
48+
SemVerLevel = (int)package.SemVerLevel,
49+
Summary = package.Summary,
50+
Title = package.Title,
51+
IconUrl = package.IconUrlString,
52+
LicenseUrl = package.LicenseUrlString,
53+
ProjectUrl = package.ProjectUrlString,
54+
RepositoryUrl = package.RepositoryUrlString,
55+
RepositoryType = package.RepositoryType,
56+
Tags = JsonConvert.SerializeObject(package.Tags),
57+
Dependencies = SerializeList(package.Dependencies, DependencyEntity.FromPackageDependency),
58+
PackageTypes = SerializeList(package.PackageTypes, PackageTypeEntity.FromPackageType),
59+
TargetFrameworks = SerializeList(package.TargetFrameworks, f => f.Moniker)
60+
};
61+
}
62+
63+
private static string SerializeList<TIn, TOut>(IReadOnlyList<TIn> objects, Func<TIn, TOut> map)
64+
{
65+
var data = objects.Select(map).ToList();
66+
67+
return JsonConvert.SerializeObject(data);
68+
}
69+
70+
public string Id { get; set; }
71+
public string NormalizedVersion { get; set; }
72+
public string OriginalVersion { get; set; }
73+
public string Authors { get; set; }
74+
public string Description { get; set; }
75+
public long Downloads { get; set; }
76+
public bool HasReadme { get; set; }
77+
public bool IsPrerelease { get; set; }
78+
public string Language { get; set; }
79+
public bool Listed { get; set; }
80+
public string MinClientVersion { get; set; }
81+
public DateTime Published { get; set; }
82+
public bool RequireLicenseAcceptance { get; set; }
83+
public int SemVerLevel { get; set; }
84+
public string Summary { get; set; }
85+
public string Title { get; set; }
86+
87+
public string IconUrl { get; set; }
88+
public string LicenseUrl { get; set; }
89+
public string ProjectUrl { get; set; }
90+
91+
public string RepositoryUrl { get; set; }
92+
public string RepositoryType { get; set; }
93+
94+
public string Tags { get; set; }
95+
public string Dependencies { get; set; }
96+
public string PackageTypes { get; set; }
97+
public string TargetFrameworks { get; set; }
98+
99+
public Package AsPackage()
100+
{
101+
var targetFrameworks = JsonConvert.DeserializeObject<List<string>>(TargetFrameworks)
102+
.Select(f => new TargetFramework { Moniker = f })
103+
.ToList();
104+
105+
return new Package
106+
{
107+
Id = Id,
108+
NormalizedVersionString = NormalizedVersion,
109+
OriginalVersionString = OriginalVersion,
110+
111+
Authors = JsonConvert.DeserializeObject<string[]>(Authors),
112+
Description = Description,
113+
Downloads = Downloads,
114+
HasReadme = HasReadme,
115+
IsPrerelease = IsPrerelease,
116+
Language = Language,
117+
Listed = Listed,
118+
MinClientVersion = MinClientVersion,
119+
Published = Published,
120+
RequireLicenseAcceptance = RequireLicenseAcceptance,
121+
SemVerLevel = (SemVerLevel)SemVerLevel,
122+
Summary = Summary,
123+
Title = Title,
124+
IconUrl = CreateUriFromString(IconUrl),
125+
LicenseUrl = CreateUriFromString(LicenseUrl),
126+
ProjectUrl = CreateUriFromString(ProjectUrl),
127+
RepositoryUrl = CreateUriFromString(RepositoryUrl),
128+
RepositoryType = RepositoryType,
129+
Tags = JsonConvert.DeserializeObject<string[]>(Tags),
130+
Dependencies = DependencyEntity.Parse(Dependencies),
131+
PackageTypes = PackageTypeEntity.Parse(PackageTypes),
132+
TargetFrameworks = targetFrameworks,
133+
};
134+
}
135+
136+
private Uri CreateUriFromString(string uriText)
137+
{
138+
return string.IsNullOrEmpty(uriText) ? null : new Uri(uriText);
139+
}
140+
141+
private class DependencyEntity
142+
{
143+
public static DependencyEntity FromPackageDependency(PackageDependency dependency)
144+
{
145+
return new DependencyEntity
146+
{
147+
Id = dependency.Id,
148+
VersionRange = dependency.VersionRange,
149+
TargetFramework = dependency.TargetFramework
150+
};
151+
}
152+
153+
public static List<PackageDependency> Parse(string json)
154+
{
155+
return JsonConvert.DeserializeObject<List<DependencyEntity>>(json)
156+
.Select(e => new PackageDependency
157+
{
158+
Id = e.Id,
159+
VersionRange = e.VersionRange,
160+
TargetFramework = e.TargetFramework,
161+
})
162+
.ToList();
163+
}
164+
165+
public string Id { get; set; }
166+
public string VersionRange { get; set; }
167+
public string TargetFramework { get; set; }
168+
}
169+
170+
private class PackageTypeEntity
171+
{
172+
public static PackageTypeEntity FromPackageType(PackageType packageType)
173+
{
174+
return new PackageTypeEntity
175+
{
176+
Name = packageType.Name,
177+
Version = packageType.Version
178+
};
179+
}
180+
181+
public static List<PackageType> Parse(string json)
182+
{
183+
return JsonConvert.DeserializeObject<List<PackageTypeEntity>>(json)
184+
.Select(e => new PackageType
185+
{
186+
Name = e.Name,
187+
Version = e.Version
188+
})
189+
.ToList();
190+
}
191+
192+
public string Name { get; set; }
193+
public string Version { get; set; }
194+
}
195+
}
196+
}
197+
}

‎src/BaGet.Azure/Entities/PackageListingEntity.cs

+22
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
using BaGet.Core;
2+
using Microsoft.Azure.Cosmos.Table;
3+
4+
namespace BaGet.Azure
5+
{
6+
/// <summary>
7+
/// The Azure Table Storage entity to update the <see cref="Package.Listed"/> column.
8+
/// The <see cref="TableEntity.PartitionKey"/> is the <see cref="Package.Id"/> and
9+
/// the <see cref="TableEntity.RowKey"/> is the <see cref="Package.Version"/>.
10+
/// </summary>
11+
public partial class TablePackageService
12+
{
13+
private class PackageListingEntity : TableEntity, IListed
14+
{
15+
public PackageListingEntity()
16+
{
17+
}
18+
19+
public bool Listed { get; set; }
20+
}
21+
}
22+
}

‎src/BaGet.Azure/Extensions/IServiceCollectionExtensions.cs

+29-3
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,42 @@
11
using BaGet.Azure.Configuration;
22
using BaGet.Azure.Search;
3+
using BaGet.Core;
4+
using Microsoft.Azure.Cosmos.Table;
35
using Microsoft.Azure.Search;
46
using Microsoft.Extensions.DependencyInjection;
57
using Microsoft.Extensions.Options;
6-
using Microsoft.WindowsAzure.Storage;
7-
using Microsoft.WindowsAzure.Storage.Auth;
88
using Microsoft.WindowsAzure.Storage.Blob;
99

1010
namespace BaGet.Azure.Extensions
1111
{
12+
using CloudStorageAccount = Microsoft.WindowsAzure.Storage.CloudStorageAccount;
13+
using StorageCredentials = Microsoft.WindowsAzure.Storage.Auth.StorageCredentials;
14+
15+
using TableStorageAccount = Microsoft.Azure.Cosmos.Table.CloudStorageAccount;
16+
1217
public static class IServiceCollectionExtensions
1318
{
19+
public static IServiceCollection AddTableStorageService(this IServiceCollection services)
20+
{
21+
services.AddSingleton(provider =>
22+
{
23+
var options = provider.GetRequiredService<IOptions<DatabaseOptions>>().Value;
24+
25+
return TableStorageAccount.Parse(options.ConnectionString);
26+
});
27+
28+
services.AddTransient(provider =>
29+
{
30+
var account = provider.GetRequiredService<TableStorageAccount>();
31+
32+
return account.CreateCloudTableClient();
33+
});
34+
35+
services.AddTransient<TablePackageService>();
36+
37+
return services;
38+
}
39+
1440
public static IServiceCollection AddBlobStorageService(this IServiceCollection services)
1541
{
1642
services.AddSingleton(provider =>
@@ -26,7 +52,7 @@ public static IServiceCollection AddBlobStorageService(this IServiceCollection s
2652

2753
services.AddTransient(provider =>
2854
{
29-
var options = provider.GetRequiredService<IOptions<BlobStorageOptions>>().Value;
55+
var options = provider.GetRequiredService<IOptionsSnapshot<BlobStorageOptions>>().Value;
3056
var account = provider.GetRequiredService<CloudStorageAccount>();
3157

3258
var client = account.CreateCloudBlobClient();

‎src/BaGet.Azure/Extensions/StorageExceptionExtensions.cs

+30
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
using System.Net;
2+
3+
namespace BaGet.Azure.Extensions
4+
{
5+
using StorageException = Microsoft.WindowsAzure.Storage.StorageException;
6+
using TableStorageException = Microsoft.Azure.Cosmos.Table.StorageException;
7+
8+
internal static class StorageExceptionExtensions
9+
{
10+
public static bool IsAlreadyExistsException(this StorageException e)
11+
{
12+
return e?.RequestInformation?.HttpStatusCode == (int?)HttpStatusCode.Conflict;
13+
}
14+
15+
public static bool IsNotFoundException(this TableStorageException e)
16+
{
17+
return e?.RequestInformation?.HttpStatusCode == (int?)HttpStatusCode.NotFound;
18+
}
19+
20+
public static bool IsAlreadyExistsException(this TableStorageException e)
21+
{
22+
return e?.RequestInformation?.HttpStatusCode == (int?)HttpStatusCode.Conflict;
23+
}
24+
25+
public static bool IsPreconditionFailedException(this TableStorageException e)
26+
{
27+
return e?.RequestInformation?.HttpStatusCode == (int?)HttpStatusCode.PreconditionFailed;
28+
}
29+
}
30+
}

0 commit comments

Comments
 (0)