Skip to content

Commit

Permalink
Fix microsoft#179 (dynamic NuGet API endpoints) (microsoft#182)
Browse files Browse the repository at this point in the history 
* Remove extra stack trace.

* Update NuGet endpoint (dynamic), Add auto gzip.

* Return RegistrationEndpoint instead.

* Switch cast to 'as'
  • Loading branch information
@scovetta
scovetta authored Dec 9, 2020
1 parent 9f42f0a commit 487817b
Show file tree
Hide file tree
Showing 4 changed files with 188 additions and 37 deletions.
4 changes: 4 additions & 0 deletions src/Shared/CommonInitialization.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,10 @@ public static void Initialize()
};
#pragma warning restore CA2000

if (handler.SupportsAutomaticDecompression)
{
handler.AutomaticDecompression = System.Net.DecompressionMethods.All;
}
WebClient = new HttpClient(handler);
WebClient.DefaultRequestHeaders.UserAgent.ParseAdd(ENV_HTTPCLIENT_USER_AGENT);

Expand Down
219 changes: 183 additions & 36 deletions src/Shared/PackageManagers/NuGetProjectManager.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,16 +7,66 @@
using System.Linq;
using System.Text.Json;
using System.Threading.Tasks;
using System.Xml.Linq;

namespace Microsoft.CST.OpenSource.Shared
{
internal class NuGetProjectManager : BaseProjectManager
{
[System.Diagnostics.CodeAnalysis.SuppressMessage("Style", "IDE0044:Add readonly modifier", Justification = "Modified through reflection.")]
public static string ENV_NUGET_ENDPOINT_API = "https://api.nuget.org";

private readonly string NUGET_DEFAULT_REGISTRATION_ENDPOINT = "https://api.nuget.org/v3/registration5-gz-semver2/";

private string? RegistrationEndpoint { get; set; } = null;

public NuGetProjectManager(string destinationDirectory) : base(destinationDirectory)
{
GetRegistrationEndpointAsync().Wait();
}

/// <summary>
/// Dynamically identifies the registration endpoint.
/// </summary>
/// <returns>NuGet registration endpoint</returns>
private async Task<string> GetRegistrationEndpointAsync()
{
if (RegistrationEndpoint != null)
{
return RegistrationEndpoint;
}

try
{
var doc = await GetJsonCache($"{ENV_NUGET_ENDPOINT_API}/v3/index.json");
var resources = doc.RootElement.GetProperty("resources").EnumerateArray();
foreach (var resource in resources)
{
try
{
var _type = resource.GetProperty("@type").GetString();
if (_type != null && _type.Equals("RegistrationsBaseUrl/Versioned", StringComparison.InvariantCultureIgnoreCase))
{
var _id = resource.GetProperty("@id").GetString();
if (!string.IsNullOrWhiteSpace(_id))
{
RegistrationEndpoint = _id;
return _id;
}
}
}
catch (Exception ex)
{
Logger.Warn(ex, "Error parsing NuGet API endpoint: {0}", ex.Message);
}
}
}
catch(Exception ex)
{
Logger.Warn(ex, "Error parsing NuGet API endpoint: {0}", ex.Message);
}
RegistrationEndpoint = NUGET_DEFAULT_REGISTRATION_ENDPOINT;
return RegistrationEndpoint;
}

/// <summary>
Expand All @@ -40,29 +90,92 @@ public override async Task<IEnumerable<string>> DownloadVersion(PackageURL purl,

try
{
var doc = await GetJsonCache($"{ENV_NUGET_ENDPOINT_API}/v3/registration3/{packageName}/{packageVersion}.json");
var archive = doc.RootElement.GetProperty("packageContent").GetString();
var result = await WebClient.GetAsync(archive);
result.EnsureSuccessStatusCode();
Logger.Debug("Downloading {0}...", purl?.ToString());

var targetName = $"nuget-{packageName}@{packageVersion}";
string extractionPath = Path.Combine(TopLevelExtractionDirectory, targetName);
if (doExtract && Directory.Exists(extractionPath) && cached == true)
{
downloadedPaths.Add(extractionPath);
return downloadedPaths;
}
if (doExtract)
{
downloadedPaths.Add(await ExtractArchive(targetName, await result.Content.ReadAsByteArrayAsync(), cached));
}
else
var doc = await GetJsonCache($"{RegistrationEndpoint}{packageName.ToLowerInvariant()}/index.json");
var versionList = new List<string>();
foreach (var catalogPage in doc.RootElement.GetProperty("items").EnumerateArray())
{
targetName += Path.GetExtension(archive) ?? "";
await File.WriteAllBytesAsync(targetName, await result.Content.ReadAsByteArrayAsync());
downloadedPaths.Add(targetName);
if (catalogPage.TryGetProperty("items", out JsonElement itemElement))
{
foreach (var item in itemElement.EnumerateArray())
{
var catalogEntry = item.GetProperty("catalogEntry");
var version = catalogEntry.GetProperty("version").GetString();
if (version != null && version.Equals(packageVersion, StringComparison.InvariantCultureIgnoreCase))
{
var archive = catalogEntry.GetProperty("packageContent").GetString();
var result = await WebClient.GetAsync(archive);
result.EnsureSuccessStatusCode();
Logger.Debug("Downloading {0}...", purl?.ToString());

var targetName = $"nuget-{packageName}@{packageVersion}";
string extractionPath = Path.Combine(TopLevelExtractionDirectory, targetName);
if (doExtract && Directory.Exists(extractionPath) && cached == true)
{
downloadedPaths.Add(extractionPath);
return downloadedPaths;
}

if (doExtract)
{
downloadedPaths.Add(await ExtractArchive(targetName, await result.Content.ReadAsByteArrayAsync(), cached));
}
else
{
targetName += Path.GetExtension(archive) ?? "";
await File.WriteAllBytesAsync(targetName, await result.Content.ReadAsByteArrayAsync());
downloadedPaths.Add(targetName);
}
return downloadedPaths;
}
}
}
else
{
var subDocUrl = catalogPage.GetProperty("@id").GetString();
if (subDocUrl != null)
{
var subDoc = await GetJsonCache(subDocUrl);
foreach (var subCatalogPage in subDoc.RootElement.GetProperty("items").EnumerateArray())
{
var catalogEntry = subCatalogPage.GetProperty("catalogEntry");
var version = catalogEntry.GetProperty("version").GetString();
Logger.Debug("Identified {0} version {1}.", packageName, version);
if (version != null && version.Equals(packageVersion, StringComparison.InvariantCultureIgnoreCase))
{
var archive = catalogEntry.GetProperty("packageContent").GetString();
var result = await WebClient.GetAsync(archive);
result.EnsureSuccessStatusCode();
Logger.Debug("Downloading {0}...", purl?.ToString());

var targetName = $"nuget-{packageName}@{packageVersion}";
string extractionPath = Path.Combine(TopLevelExtractionDirectory, targetName);
if (doExtract && Directory.Exists(extractionPath) && cached == true)
{
downloadedPaths.Add(extractionPath);
return downloadedPaths;
}

if (doExtract)
{
downloadedPaths.Add(await ExtractArchive(targetName, await result.Content.ReadAsByteArrayAsync(), cached));
}
else
{
targetName += Path.GetExtension(archive) ?? "";
await File.WriteAllBytesAsync(targetName, await result.Content.ReadAsByteArrayAsync());
downloadedPaths.Add(targetName);
}
return downloadedPaths;
}
}
}
else
{
Logger.Debug("Catalog identifier was null.");
}
}
}
Logger.Debug("Unable to find NuGet package.");
}
catch (Exception ex)
{
Expand All @@ -83,7 +196,12 @@ public override async Task<IEnumerable<string>> EnumerateVersions(PackageURL pur
try
{
var packageName = purl.Name;
var doc = await GetJsonCache($"{ENV_NUGET_ENDPOINT_API}/v3/registration3/{packageName}/index.json");
if (packageName == null)
{
return new List<string>();
}

var doc = await GetJsonCache($"{RegistrationEndpoint}{packageName.ToLowerInvariant()}/index.json");
var versionList = new List<string>();
foreach (var catalogPage in doc.RootElement.GetProperty("items").EnumerateArray())
{
Expand All @@ -93,20 +211,42 @@ public override async Task<IEnumerable<string>> EnumerateVersions(PackageURL pur
{
var catalogEntry = item.GetProperty("catalogEntry");
var version = catalogEntry.GetProperty("version").GetString();
Logger.Debug("Identified {0} version {1}.", packageName, version);
versionList.Add(version);
if (version != null)
{
Logger.Debug("Identified {0} version {1}.", packageName, version);
versionList.Add(version);
}
else
{
Logger.Warn("Identified {0} version NULL. This might indicate a parsing error.", packageName);
}
}
}
else
{
var subDocUrl = catalogPage.GetProperty("@id");
var subDoc = await GetJsonCache(subDocUrl.GetString());
foreach (var subCatalogPage in subDoc.RootElement.GetProperty("items").EnumerateArray())
var subDocUrl = catalogPage.GetProperty("@id").GetString();
if (subDocUrl != null)
{
var catalogEntry = subCatalogPage.GetProperty("catalogEntry");
var version = catalogEntry.GetProperty("version").GetString();
Logger.Debug("Identified {0} version {1}.", packageName, version);
versionList.Add(version);
var subDoc = await GetJsonCache(subDocUrl);
foreach (var subCatalogPage in subDoc.RootElement.GetProperty("items").EnumerateArray())
{
var catalogEntry = subCatalogPage.GetProperty("catalogEntry");
var version = catalogEntry.GetProperty("version").GetString();
Logger.Debug("Identified {0} version {1}.", packageName, version);
if (version != null)
{
Logger.Debug("Identified {0} version {1}.", packageName, version);
versionList.Add(version);
}
else
{
Logger.Warn("Identified {0} version NULL. This might indicate a parsing error.", packageName);
}
}
}
else
{
Logger.Debug("Catalog identifier was null.");
}
}
}
Expand All @@ -124,7 +264,11 @@ public override async Task<IEnumerable<string>> EnumerateVersions(PackageURL pur
try
{
var packageName = purl.Name;
var content = await GetHttpStringCache($"{ENV_NUGET_ENDPOINT_API}/v3/registration3/{packageName?.ToLower()}/index.json");
if (packageName == null)
{
return null;
}
var content = await GetHttpStringCache($"{RegistrationEndpoint}{packageName.ToLowerInvariant()}/index.json");
return content;
}
catch (Exception ex)
Expand Down Expand Up @@ -162,16 +306,19 @@ protected async override Task<Dictionary<PackageURL, double>> SearchRepoUrlsInPa
string? repoCandidate = doc.DocumentNode.SelectSingleNode(path)?.GetAttributeValue("href", string.Empty);
if (!string.IsNullOrEmpty(repoCandidate))
{
PackageURL repoPurl = GitHubProjectManager.ExtractGitHubPackageURLs(repoCandidate).ToList().FirstOrDefault();
mapping.Add(repoPurl, 1.0F);
return mapping;
var candidate = ExtractGitHubPackageURLs(repoCandidate).FirstOrDefault();
if (candidate != null)
{
mapping.Add(candidate as PackageURL, 1.0F);

}
}
}
return mapping;
}
catch (Exception ex)
{
Logger.Error(ex, $"Error fetching/parsing NuGet homepage: {ex.Message}");
return mapping;
}

// if nothing worked, return empty
Expand Down
1 change: 0 additions & 1 deletion src/oss-download/DownloadTool.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,6 @@ private async Task RunAsync(Options options)
catch (Exception ex)
{
Logger.Warn(ex, "Error processing {0}: {1}", target, ex.Message);
Logger.Warn(ex.StackTrace);
}
}
}
Expand Down
1 change: 1 addition & 0 deletions src/oss-tests/DownloadTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,7 @@ public async Task NPM_Download_ScopedVersion_Succeeds(string purl, string target
[DataTestMethod]
[DataRow("pkg:nuget/[email protected]", "RandomType.nuspec", 1)]
[DataRow("pkg:nuget/d3.TypeScript.DefinitelyTyped", "d3.TypeScript.DefinitelyTyped.nuspec", 1)]
[DataRow("pkg:nuget/[email protected]", "boxer.nuspec", 1)]
public async Task NuGet_Download_Version_Succeeds(string purl, string targetFilename, int expectedDirectoryCount)
{
await TestDownload(purl, targetFilename, expectedDirectoryCount);
Expand Down

0 comments on commit 487817b

Please sign in to comment.