Skip to content

Commit ece506f

Browse files
loic-sharmaloic-sharma
committed
Start the Azure Search importer
1 parent 9dae62c commit ece506f

11 files changed

+329
-1
lines changed

‎BaGet.sln

+7-1
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,9 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BaGet.Services.Remote", "sr
1111
EndProject
1212
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BaGet.Azure", "src\BaGet.Azure\BaGet.Azure.csproj", "{716C970D-9614-4265-AC92-57E8B227B98E}"
1313
EndProject
14-
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BaGet.Tools.ImportDownloads", "src\BaGet.Tools.ImportDownloads\BaGet.Tools.ImportDownloads.csproj", "{8B7C1C8D-ED4D-4732-86A1-0A7DAD3045DD}"
14+
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BaGet.Tools.ImportDownloads", "src\BaGet.Tools.ImportDownloads\BaGet.Tools.ImportDownloads.csproj", "{8B7C1C8D-ED4D-4732-86A1-0A7DAD3045DD}"
15+
EndProject
16+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BaGet.Tools.AzureSearchIndexer", "src\BaGet.Tools.AzureSearchIndexer\BaGet.Tools.AzureSearchIndexer.csproj", "{B232DAFE-5CE8-441F-ACC7-2BB54BCD094F}"
1517
EndProject
1618
Global
1719
GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -39,6 +41,10 @@ Global
3941
{8B7C1C8D-ED4D-4732-86A1-0A7DAD3045DD}.Debug|Any CPU.Build.0 = Debug|Any CPU
4042
{8B7C1C8D-ED4D-4732-86A1-0A7DAD3045DD}.Release|Any CPU.ActiveCfg = Release|Any CPU
4143
{8B7C1C8D-ED4D-4732-86A1-0A7DAD3045DD}.Release|Any CPU.Build.0 = Release|Any CPU
44+
{B232DAFE-5CE8-441F-ACC7-2BB54BCD094F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
45+
{B232DAFE-5CE8-441F-ACC7-2BB54BCD094F}.Debug|Any CPU.Build.0 = Debug|Any CPU
46+
{B232DAFE-5CE8-441F-ACC7-2BB54BCD094F}.Release|Any CPU.ActiveCfg = Release|Any CPU
47+
{B232DAFE-5CE8-441F-ACC7-2BB54BCD094F}.Release|Any CPU.Build.0 = Release|Any CPU
4248
EndGlobalSection
4349
GlobalSection(SolutionProperties) = preSolution
4450
HideSolutionNode = FALSE

‎src/BaGet.Tools.AzureSearchImporter/BaGet.Tools.AzureSearchImporter.csproj

+29
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
<Project Sdk="Microsoft.NET.Sdk">
2+
3+
<PropertyGroup>
4+
<OutputType>Exe</OutputType>
5+
<TargetFramework>netcoreapp2.0</TargetFramework>
6+
</PropertyGroup>
7+
8+
<ItemGroup>
9+
<ProjectReference Include="..\BaGet\BaGet.csproj" />
10+
</ItemGroup>
11+
12+
<ItemGroup>
13+
<Content Include="appsettings.json" CopyToOutputDirectory="Always" />
14+
</ItemGroup>
15+
16+
<ItemGroup>
17+
<PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="2.0.1" />
18+
<PackageReference Include="morelinq" Version="2.10.0" />
19+
</ItemGroup>
20+
21+
<ItemGroup>
22+
<DotNetCliToolReference Include="Microsoft.EntityFrameworkCore.Tools.DotNet" Version="2.0.1" />
23+
</ItemGroup>
24+
25+
<ItemGroup>
26+
<Folder Include="Migrations\" />
27+
</ItemGroup>
28+
29+
</Project>

‎src/BaGet.Tools.AzureSearchImporter/Entities/IndexerContext.cs

+23
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
using Microsoft.EntityFrameworkCore;
2+
3+
namespace BaGet.Tools.AzureSearchImporter.Entities
4+
{
5+
public class IndexerContext : DbContext
6+
{
7+
public IndexerContext(DbContextOptions<IndexerContext> options)
8+
: base(options)
9+
{}
10+
11+
public DbSet<PackageId> PackageIds { get; set; }
12+
13+
protected override void OnModelCreating(ModelBuilder builder)
14+
{
15+
builder.Entity<PackageId>()
16+
.HasKey(p => p.Key);
17+
18+
builder.Entity<PackageId>()
19+
.Property(p => p.Value)
20+
.HasColumnType("TEXT COLLATE NOCASE");
21+
}
22+
}
23+
}

‎src/BaGet.Tools.AzureSearchImporter/Entities/IndexerContextFactory.cs

+19
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
using Microsoft.EntityFrameworkCore;
2+
using Microsoft.EntityFrameworkCore.Design;
3+
4+
namespace BaGet.Tools.AzureSearchImporter.Entities
5+
{
6+
class IndexerContextFactory : IDesignTimeDbContextFactory<IndexerContext>
7+
{
8+
public const string ConnectionString = "Data Source=indexer.db";
9+
10+
public IndexerContext CreateDbContext(string[] args)
11+
{
12+
var optionsBuilder = new DbContextOptionsBuilder<IndexerContext>();
13+
14+
optionsBuilder.UseSqlite(ConnectionString);
15+
16+
return new IndexerContext(optionsBuilder.Options);
17+
}
18+
}
19+
}

‎src/BaGet.Tools.AzureSearchImporter/Entities/PackageId.cs

+11
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
namespace BaGet.Tools.AzureSearchImporter.Entities
2+
{
3+
public class PackageId
4+
{
5+
public int Key { get; set; }
6+
7+
public string Value { get; set; }
8+
9+
public bool Done { get; set; }
10+
}
11+
}

‎src/BaGet.Tools.AzureSearchImporter/Initializer.cs

+65
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
using System;
2+
using System.Linq;
3+
using System.Threading.Tasks;
4+
using BaGet.Core.Entities;
5+
using BaGet.Tools.AzureSearchImporter.Entities;
6+
using Microsoft.EntityFrameworkCore;
7+
using Microsoft.Extensions.Logging;
8+
using MoreLinq;
9+
10+
namespace BaGet.Tools.AzureSearchImporter
11+
{
12+
public class Initializer
13+
{
14+
public const int InitializationBatchSize = 100;
15+
16+
private readonly IContext _bagetContext;
17+
private readonly IndexerContext _indexerContext;
18+
private readonly ILogger<Initializer> _logger;
19+
20+
public Initializer(IContext bagetContext, IndexerContext indexerContext, ILogger<Initializer> logger)
21+
{
22+
_bagetContext = bagetContext ?? throw new ArgumentNullException(nameof(bagetContext));
23+
_indexerContext = indexerContext ?? throw new ArgumentNullException(nameof(indexerContext));
24+
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
25+
}
26+
27+
public async Task InitializeAsync(bool force = false)
28+
{
29+
if (!force && await _indexerContext.PackageIds.AnyAsync())
30+
{
31+
_logger.LogInformation("Skipping initialization");
32+
return;
33+
}
34+
35+
_logger.LogInformation("Finding packages to initialize...");
36+
37+
var packageIds = await _bagetContext.Packages
38+
.GroupBy(p => p.Id)
39+
.OrderByDescending(g => g.Sum(p => p.Downloads))
40+
.Select(g => g.Key)
41+
.ToListAsync();
42+
43+
_logger.LogInformation("Found {PackageIdCount} package ids to initialize", packageIds.Count);
44+
45+
var batchCount = 1;
46+
47+
foreach (var batch in packageIds.Batch(InitializationBatchSize))
48+
{
49+
foreach (var packageId in batch)
50+
{
51+
_indexerContext.PackageIds.Add(new PackageId
52+
{
53+
Value = packageId,
54+
Done = false,
55+
});
56+
}
57+
58+
_logger.LogInformation("Saving batch {BatchCount}", batchCount);
59+
60+
await _indexerContext.SaveChangesAsync();
61+
batchCount++;
62+
}
63+
}
64+
}
65+
}

‎src/BaGet.Tools.AzureSearchImporter/Migrations/20180413063338_Initial.Designer.cs

+39
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎src/BaGet.Tools.AzureSearchImporter/Migrations/20180413063338_Initial.cs

+32
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
using Microsoft.EntityFrameworkCore.Migrations;
2+
using System;
3+
using System.Collections.Generic;
4+
5+
namespace BaGet.Tools.AzureSearchImporter.Migrations
6+
{
7+
public partial class Initial : Migration
8+
{
9+
protected override void Up(MigrationBuilder migrationBuilder)
10+
{
11+
migrationBuilder.CreateTable(
12+
name: "PackageIds",
13+
columns: table => new
14+
{
15+
Key = table.Column<int>(nullable: false)
16+
.Annotation("Sqlite:Autoincrement", true),
17+
Done = table.Column<bool>(nullable: false),
18+
Value = table.Column<string>(type: "TEXT COLLATE NOCASE", nullable: true)
19+
},
20+
constraints: table =>
21+
{
22+
table.PrimaryKey("PK_PackageIds", x => x.Key);
23+
});
24+
}
25+
26+
protected override void Down(MigrationBuilder migrationBuilder)
27+
{
28+
migrationBuilder.DropTable(
29+
name: "PackageIds");
30+
}
31+
}
32+
}

‎src/BaGet.Tools.AzureSearchImporter/Migrations/IndexerContextModelSnapshot.cs

+38
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
// <auto-generated />
2+
using BaGet.Tools.AzureSearchImporter.Entities;
3+
using Microsoft.EntityFrameworkCore;
4+
using Microsoft.EntityFrameworkCore.Infrastructure;
5+
using Microsoft.EntityFrameworkCore.Metadata;
6+
using Microsoft.EntityFrameworkCore.Migrations;
7+
using Microsoft.EntityFrameworkCore.Storage;
8+
using System;
9+
10+
namespace BaGet.Tools.AzureSearchImporter.Migrations
11+
{
12+
[DbContext(typeof(IndexerContext))]
13+
partial class IndexerContextModelSnapshot : ModelSnapshot
14+
{
15+
protected override void BuildModel(ModelBuilder modelBuilder)
16+
{
17+
#pragma warning disable 612, 618
18+
modelBuilder
19+
.HasAnnotation("ProductVersion", "2.0.1-rtm-125");
20+
21+
modelBuilder.Entity("BaGet.Tools.AzureSearchImporter.Entities.PackageId", b =>
22+
{
23+
b.Property<int>("Key")
24+
.ValueGeneratedOnAdd();
25+
26+
b.Property<bool>("Done");
27+
28+
b.Property<string>("Value")
29+
.HasColumnType("TEXT COLLATE NOCASE");
30+
31+
b.HasKey("Key");
32+
33+
b.ToTable("PackageIds");
34+
});
35+
#pragma warning restore 612, 618
36+
}
37+
}
38+
}

‎src/BaGet.Tools.AzureSearchImporter/Program.cs

+60
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
using System;
2+
using System.Threading.Tasks;
3+
using BaGet.Configuration;
4+
using BaGet.Extensions;
5+
using BaGet.Tools.AzureSearchImporter.Entities;
6+
using Microsoft.EntityFrameworkCore;
7+
using Microsoft.Extensions.Configuration;
8+
using Microsoft.Extensions.DependencyInjection;
9+
using Microsoft.Extensions.Logging;
10+
11+
namespace BaGet.Tools.AzureSearchImporter
12+
{
13+
class Program
14+
{
15+
public static void Main(string[] args)
16+
=> MainAsync(args)
17+
.GetAwaiter()
18+
.GetResult();
19+
20+
private async static Task MainAsync(string[] args)
21+
{
22+
var provider = GetServiceProvider(GetConfiguration());
23+
var scopeFactory = provider.GetRequiredService<IServiceScopeFactory>();
24+
25+
using (var scope = scopeFactory.CreateScope())
26+
{
27+
scope.ServiceProvider
28+
.GetRequiredService<IndexerContext>()
29+
.Database
30+
.Migrate();
31+
}
32+
33+
await provider.GetRequiredService<Initializer>().InitializeAsync();
34+
}
35+
36+
private static IConfiguration GetConfiguration()
37+
=> new ConfigurationBuilder()
38+
.SetBasePath(Environment.CurrentDirectory)
39+
.AddJsonFile("appsettings.json")
40+
.Build();
41+
42+
private static IServiceProvider GetServiceProvider(IConfiguration configuration)
43+
{
44+
var services = new ServiceCollection();
45+
46+
services.Configure<BaGetOptions>(configuration);
47+
services.AddLogging(logging => logging.AddConsole());
48+
49+
services.AddBaGetContext();
50+
services.AddDbContext<IndexerContext>((provider, options) =>
51+
{
52+
options.UseSqlite(IndexerContextFactory.ConnectionString);
53+
});
54+
55+
services.AddTransient<Initializer>();
56+
57+
return services.BuildServiceProvider();
58+
}
59+
}
60+
}

‎src/BaGet.Tools.AzureSearchImporter/appsettings.json

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
{
2+
"Database": {
3+
"Type": "Sqlite",
4+
"ConnectionString": "Data Source=..\\BaGet\\baget.db"
5+
}
6+
}

0 commit comments

Comments
 (0)