Skip to content

Commit dfbd66e

Browse files
LordMikeloic-sharma
authored andcommitted
Add S3 storage service (loic-sharma#144)
Fixes loic-sharma#136
1 parent ffeb211 commit dfbd66e

File tree

10 files changed

+337
-2
lines changed

10 files changed

+337
-2
lines changed

BaGet.sln

+7
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BaGet.Protocol", "src\BaGet
2323
EndProject
2424
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BaGet.Protocol.Tests", "tests\BaGet.Protocol.Tests\BaGet.Protocol.Tests.csproj", "{AC764A9A-9EAF-422B-9223-D3290C3CFD79}"
2525
EndProject
26+
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BaGet.AWS", "src\BaGet.AWS\BaGet.AWS.csproj", "{D067D82E-D515-44D1-A832-C79F29418DFC}"
27+
EndProject
2628
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{0B44364D-952B-497A-82E0-C9AAE94E0369}"
2729
ProjectSection(SolutionItems) = preProject
2830
.editorconfig = .editorconfig
@@ -66,6 +68,10 @@ Global
6668
{AC764A9A-9EAF-422B-9223-D3290C3CFD79}.Debug|Any CPU.Build.0 = Debug|Any CPU
6769
{AC764A9A-9EAF-422B-9223-D3290C3CFD79}.Release|Any CPU.ActiveCfg = Release|Any CPU
6870
{AC764A9A-9EAF-422B-9223-D3290C3CFD79}.Release|Any CPU.Build.0 = Release|Any CPU
71+
{D067D82E-D515-44D1-A832-C79F29418DFC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
72+
{D067D82E-D515-44D1-A832-C79F29418DFC}.Debug|Any CPU.Build.0 = Debug|Any CPU
73+
{D067D82E-D515-44D1-A832-C79F29418DFC}.Release|Any CPU.ActiveCfg = Release|Any CPU
74+
{D067D82E-D515-44D1-A832-C79F29418DFC}.Release|Any CPU.Build.0 = Release|Any CPU
6975
EndGlobalSection
7076
GlobalSection(SolutionProperties) = preSolution
7177
HideSolutionNode = FALSE
@@ -79,6 +85,7 @@ Global
7985
{892A7A82-4283-4315-B7E5-6D5B70543000} = {C237857D-AD8E-4C52-974F-6A8155BB0C18}
8086
{A2D23427-9278-4D52-B31F-759212252832} = {26A0B557-53FB-4B9A-94C4-BCCF1BDCB0CC}
8187
{AC764A9A-9EAF-422B-9223-D3290C3CFD79} = {C237857D-AD8E-4C52-974F-6A8155BB0C18}
88+
{D067D82E-D515-44D1-A832-C79F29418DFC} = {26A0B557-53FB-4B9A-94C4-BCCF1BDCB0CC}
8289
EndGlobalSection
8390
GlobalSection(ExtensibilityGlobals) = postSolution
8491
SolutionGuid = {1423C027-2C90-417F-8629-2A4CF107C055}

src/BaGet.AWS/BaGet.AWS.csproj

+18
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
<Project Sdk="Microsoft.NET.Sdk">
2+
3+
<PropertyGroup>
4+
<TargetFrameworks>netstandard2.0;net461</TargetFrameworks>
5+
6+
<Description>The libraries to host BaGet on AWS.</Description>
7+
</PropertyGroup>
8+
9+
<ItemGroup>
10+
<PackageReference Include="AWSSDK.S3" Version="3.3.30" />
11+
<PackageReference Include="Microsoft.Extensions.Options.ConfigurationExtensions" Version="2.1.1" />
12+
</ItemGroup>
13+
14+
<ItemGroup>
15+
<ProjectReference Include="..\BaGet.Core\BaGet.Core.csproj" />
16+
</ItemGroup>
17+
18+
</Project>
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
using System.ComponentModel.DataAnnotations;
2+
using BaGet.Core.Validation;
3+
4+
namespace BaGet.AWS.Configuration
5+
{
6+
public class S3StorageOptions
7+
{
8+
[RequiredIf(nameof(SecretKey), null, IsInverted = true)]
9+
public string AccessKey { get; set; }
10+
11+
[RequiredIf(nameof(AccessKey), null, IsInverted = true)]
12+
public string SecretKey { get; set; }
13+
14+
[Required]
15+
public string Region { get; set; }
16+
17+
[Required]
18+
public string Bucket { get; set; }
19+
20+
public string Prefix { get; set; }
21+
}
22+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
using Amazon;
2+
using Amazon.Runtime;
3+
using Amazon.S3;
4+
using BaGet.AWS.Configuration;
5+
using Microsoft.Extensions.DependencyInjection;
6+
using Microsoft.Extensions.Options;
7+
8+
namespace BaGet.AWS.Extensions
9+
{
10+
public static class ServiceCollectionExtensions
11+
{
12+
public static IServiceCollection AddS3StorageService(this IServiceCollection services)
13+
{
14+
services.AddSingleton(provider =>
15+
{
16+
var options = provider.GetRequiredService<IOptions<S3StorageOptions>>().Value;
17+
18+
AmazonS3Config config = new AmazonS3Config
19+
{
20+
RegionEndpoint = RegionEndpoint.GetBySystemName(options.Region)
21+
};
22+
23+
if (!string.IsNullOrEmpty(options.AccessKey))
24+
return new AmazonS3Client(new BasicAWSCredentials(options.AccessKey, options.SecretKey), config);
25+
26+
return new AmazonS3Client(config);
27+
});
28+
29+
services.AddTransient<S3StorageService>();
30+
31+
return services;
32+
}
33+
}
34+
}

src/BaGet.AWS/S3StorageService.cs

+98
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
using System;
2+
using System.IO;
3+
using System.Threading;
4+
using System.Threading.Tasks;
5+
using Amazon.S3;
6+
using Amazon.S3.Model;
7+
using BaGet.AWS.Configuration;
8+
using BaGet.Core.Services;
9+
using Microsoft.Extensions.Options;
10+
11+
namespace BaGet.AWS
12+
{
13+
public class S3StorageService : IStorageService
14+
{
15+
private const string Separator = "/";
16+
private readonly string _bucket;
17+
private readonly string _prefix;
18+
private readonly AmazonS3Client _client;
19+
20+
public S3StorageService(IOptions<S3StorageOptions> options, AmazonS3Client client)
21+
{
22+
_bucket = options.Value.Bucket;
23+
_prefix = options.Value.Prefix;
24+
_client = client;
25+
26+
if (!string.IsNullOrEmpty(_prefix) && !_prefix.EndsWith(Separator))
27+
_prefix += Separator;
28+
}
29+
30+
private string PrepareKey(string path)
31+
{
32+
return _prefix + path.Replace("\\", Separator);
33+
}
34+
35+
public async Task<Stream> GetAsync(string path, CancellationToken cancellationToken = default)
36+
{
37+
MemoryStream stream = new MemoryStream();
38+
39+
try
40+
{
41+
using (GetObjectResponse res = await _client.GetObjectAsync(_bucket, PrepareKey(path), cancellationToken))
42+
await res.ResponseStream.CopyToAsync(stream);
43+
44+
stream.Seek(0, SeekOrigin.Begin);
45+
}
46+
catch (Exception)
47+
{
48+
stream.Dispose();
49+
50+
// TODO
51+
throw;
52+
}
53+
54+
return stream;
55+
}
56+
57+
public Task<Uri> GetDownloadUriAsync(string path, CancellationToken cancellationToken = default)
58+
{
59+
string res = _client.GetPreSignedURL(new GetPreSignedUrlRequest
60+
{
61+
BucketName = _bucket,
62+
Key = PrepareKey(path)
63+
});
64+
65+
return Task.FromResult(new Uri(res));
66+
}
67+
68+
public async Task<PutResult> PutAsync(string path, Stream content, string contentType, CancellationToken cancellationToken = default)
69+
{
70+
// TODO: Uploads should be idempotent. This should fail if and only if the blob
71+
// already exists but has different content.
72+
73+
using (MemoryStream ms = new MemoryStream())
74+
{
75+
await content.CopyToAsync(ms, 4096, cancellationToken);
76+
77+
ms.Seek(0, SeekOrigin.Begin);
78+
79+
await _client.PutObjectAsync(new PutObjectRequest
80+
{
81+
BucketName = _bucket,
82+
Key = PrepareKey(path),
83+
InputStream = ms,
84+
ContentType = contentType,
85+
AutoResetStreamPosition = false,
86+
AutoCloseStream = false
87+
}, cancellationToken);
88+
}
89+
90+
return PutResult.Success;
91+
}
92+
93+
public async Task DeleteAsync(string path, CancellationToken cancellationToken = default)
94+
{
95+
await _client.DeleteObjectAsync(_bucket, PrepareKey(path), cancellationToken);
96+
}
97+
}
98+
}

src/BaGet.Core/Configuration/StorageOptions.cs

+1
Original file line numberDiff line numberDiff line change
@@ -9,5 +9,6 @@ public enum StorageType
99
{
1010
FileSystem = 0,
1111
AzureBlobStorage = 1,
12+
AwsS3 = 2
1213
}
1314
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
using System;
2+
using System.ComponentModel.DataAnnotations;
3+
using System.Globalization;
4+
using System.Reflection;
5+
6+
namespace BaGet.Core.Validation
7+
{
8+
/// <summary>
9+
/// Provides conditional validation based on related property value.
10+
///
11+
/// Inspiration: https://stackoverflow.com/a/27666044
12+
/// </summary>
13+
[AttributeUsage(AttributeTargets.Property)]
14+
public sealed class RequiredIfAttribute : ValidationAttribute
15+
{
16+
#region Properties
17+
18+
/// <summary>
19+
/// Gets or sets the other property name that will be used during validation.
20+
/// </summary>
21+
/// <value>
22+
/// The other property name.
23+
/// </value>
24+
public string OtherProperty { get; }
25+
26+
/// <summary>
27+
/// Gets or sets the display name of the other property.
28+
/// </summary>
29+
/// <value>
30+
/// The display name of the other property.
31+
/// </value>
32+
public string OtherPropertyDisplayName { get; set; }
33+
34+
/// <summary>
35+
/// Gets or sets the other property value that will be relevant for validation.
36+
/// </summary>
37+
/// <value>
38+
/// The other property value.
39+
/// </value>
40+
public object OtherPropertyValue { get; }
41+
42+
/// <summary>
43+
/// Gets or sets a value indicating whether other property's value should match or differ from provided other property's value (default is <c>false</c>).
44+
/// </summary>
45+
/// <value>
46+
/// <c>true</c> if other property's value validation should be inverted; otherwise, <c>false</c>.
47+
/// </value>
48+
/// <remarks>
49+
/// How this works
50+
/// - true: validated property is required when other property doesn't equal provided value
51+
/// - false: validated property is required when other property matches provided value
52+
/// </remarks>
53+
public bool IsInverted { get; set; }
54+
55+
/// <summary>
56+
/// Gets a value that indicates whether the attribute requires validation context.
57+
/// </summary>
58+
/// <returns><c>true</c> if the attribute requires validation context; otherwise, <c>false</c>.</returns>
59+
public override bool RequiresValidationContext => true;
60+
61+
#endregion
62+
63+
#region Constructor
64+
65+
/// <summary>
66+
/// Initializes a new instance of the <see cref="RequiredIfAttribute"/> class.
67+
/// </summary>
68+
/// <param name="otherProperty">The other property.</param>
69+
/// <param name="otherPropertyValue">The other property value.</param>
70+
public RequiredIfAttribute(string otherProperty, object otherPropertyValue)
71+
: base("'{0}' is required because '{1}' has a value {3}'{2}'.")
72+
{
73+
OtherProperty = otherProperty;
74+
OtherPropertyValue = otherPropertyValue;
75+
IsInverted = false;
76+
}
77+
78+
#endregion
79+
80+
/// <summary>
81+
/// Applies formatting to an error message, based on the data field where the error occurred.
82+
/// </summary>
83+
/// <param name="name">The name to include in the formatted message.</param>
84+
/// <returns>
85+
/// An instance of the formatted error message.
86+
/// </returns>
87+
public override string FormatErrorMessage(string name)
88+
{
89+
return string.Format(
90+
CultureInfo.CurrentCulture,
91+
ErrorMessageString,
92+
name,
93+
OtherPropertyDisplayName ?? OtherProperty,
94+
OtherPropertyValue,
95+
IsInverted ? "other than " : "of ");
96+
}
97+
98+
/// <summary>
99+
/// Validates the specified value with respect to the current validation attribute.
100+
/// </summary>
101+
/// <param name="value">The value to validate.</param>
102+
/// <param name="validationContext">The context information about the validation operation.</param>
103+
/// <returns>
104+
/// An instance of the <see cref="T:System.ComponentModel.DataAnnotations.ValidationResult" /> class.
105+
/// </returns>
106+
protected override ValidationResult IsValid(object value, ValidationContext validationContext)
107+
{
108+
if (validationContext == null)
109+
throw new ArgumentNullException(nameof(validationContext));
110+
111+
PropertyInfo otherProperty = validationContext.ObjectType.GetProperty(OtherProperty);
112+
if (otherProperty == null)
113+
{
114+
return new ValidationResult(
115+
string.Format(CultureInfo.CurrentCulture, "Could not find a property named '{0}'.", OtherProperty));
116+
}
117+
118+
object otherValue = otherProperty.GetValue(validationContext.ObjectInstance);
119+
120+
// check if this value is actually required and validate it
121+
if (!IsInverted && Equals(otherValue, OtherPropertyValue) ||
122+
IsInverted && !Equals(otherValue, OtherPropertyValue))
123+
{
124+
if (value == null)
125+
return new ValidationResult(FormatErrorMessage(validationContext.DisplayName));
126+
127+
// additional check for strings so they're not empty
128+
if (value is string val && val.Trim().Length == 0)
129+
return new ValidationResult(FormatErrorMessage(validationContext.DisplayName));
130+
}
131+
132+
return ValidationResult.Success;
133+
}
134+
}
135+
}

src/BaGet.Tools.AzureSearchImporter/Program.cs

+2
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
using System;
22
using System.Threading.Tasks;
3+
using BaGet.AWS.Extensions;
34
using BaGet.Azure.Extensions;
45
using BaGet.Core.Configuration;
56
using BaGet.Core.Services;
@@ -60,6 +61,7 @@ private static IServiceProvider GetServiceProvider(IConfiguration configuration)
6061

6162
services.Configure<BaGetOptions>(configuration);
6263
services.ConfigureAzure(configuration);
64+
services.ConfigureAws(configuration);
6365

6466
services.AddLogging(logging =>
6567
{

src/BaGet/BaGet.csproj

+1
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
</ItemGroup>
2323

2424
<ItemGroup>
25+
<ProjectReference Include="..\BaGet.AWS\BaGet.AWS.csproj" />
2526
<ProjectReference Include="..\BaGet.Azure\BaGet.Azure.csproj" />
2627
<ProjectReference Include="..\BaGet.Core\BaGet.Core.csproj" />
2728
<ProjectReference Include="..\BaGet.Protocol\BaGet.Protocol.csproj" />

0 commit comments

Comments
 (0)