Skip to content

Commit

Permalink
#1162: url-updater refactoring (#1169)
Browse files Browse the repository at this point in the history
  • Loading branch information
@jan-vcapgemini
jan-vcapgemini authored Jun 2, 2023
1 parent d396136 commit 3bf8654
Show file tree
Hide file tree
Showing 18 changed files with 282 additions and 191 deletions.
2 changes: 1 addition & 1 deletion .lift.toml
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
build = "mvn"

ignoreRules = [ "Var", "Varifier", "MissingSummary", "PATH_TRAVERSAL_IN" ]
ignoreRules = [ "Var", "Varifier", "MissingSummary", "PATH_TRAVERSAL_IN", "NULL_DEREFERENCE" ]
144 changes: 76 additions & 68 deletions url-updater/src/main/java/com/devonfw/tools/ide/url/updater/AbstractUrlUpdater.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,7 @@ protected HttpResponse<InputStream> doGetResponseAsStream(String url) {
/**
* Updates a tool version with the given arguments (OS independent).
*
* @param urlVersion the {@link UrlVersion} instance to update.
* @param urlVersion the {@link UrlVersion} with the {@link UrlVersion#getName() version-number} to process.
* @param downloadUrl the URL of the download for the tool.
* @return true if the version was successfully updated, false otherwise.
*/
Expand All @@ -139,20 +139,35 @@ protected boolean doAddVersion(UrlVersion urlVersion, String downloadUrl) {
/**
* Updates a tool version with the given arguments.
*
* @param urlVersion the {@link UrlVersion} instance to update.
* @param urlVersion the {@link UrlVersion} with the {@link UrlVersion#getName() version-number} to process.
* @param downloadUrl the URL of the download for the tool.
* @param os the {@link OperatingSystem} for the tool (can be null).
* @return true if the version was successfully updated, false otherwise.
*/
protected boolean doAddVersion(UrlVersion urlVersion, String downloadUrl, OperatingSystem os) {

return doAddVersion(urlVersion, downloadUrl, os, null, "");
return doAddVersion(urlVersion, downloadUrl, os, null);
}

/**
* Updates a tool version with the given arguments.
*
* @param urlVersion the UrlVersion instance to update.
* @param urlVersion the {@link UrlVersion} with the {@link UrlVersion#getName() version-number} to process.
* @param downloadUrl the URL of the download for the tool.
* @param os the {@link OperatingSystem} for the tool (can be null).
* @param architecture the optional {@link SystemArchitecture}.
* @return true if the version was successfully updated, false otherwise.
*/
protected boolean doAddVersion(UrlVersion urlVersion, String downloadUrl, OperatingSystem os,
SystemArchitecture architecture) {

return doAddVersion(urlVersion, downloadUrl, os, architecture, "");
}

/**
* Updates a tool version with the given arguments.
*
* @param urlVersion the {@link UrlVersion} with the {@link UrlVersion#getName() version-number} to process.
* @param url the URL of the download for the tool.
* @param os the optional {@link OperatingSystem}.
* @param architecture the optional {@link SystemArchitecture}.
Expand All @@ -172,11 +187,7 @@ protected boolean doAddVersion(UrlVersion urlVersion, String url, OperatingSyste
}
url = url.replace("${edition}", getEdition());

if (checksum.isEmpty()) {
return checkDownloadUrl(url, urlVersion, os, architecture);
} else {
return checkDownloadUrl(url, urlVersion, os, architecture, checksum);
}
return checkDownloadUrl(url, urlVersion, os, architecture, checksum);

}

Expand All @@ -190,58 +201,62 @@ protected boolean isSuccess(HttpResponse<?> response) {
}

/**
* Checks the download URL and takes the provided checksum into account instead of downloading the file and generating
* the checksum from it
* Checks if the download file checksum is still valid
*
* @param url the URL of the download to check.
* @param urlVersion the {@link UrlVersion} where to store the collected information like status and checksum.
* @param url String of the URL to check
* @param urlVersion the {@link UrlVersion} with the {@link UrlVersion#getName() version-number} to process.
* @param os the {@link OperatingSystem}
* @param architecture the {@link SystemArchitecture}
* @param checksum String of the checksum to use
* @return {@code true} if the download was checked successfully, {@code false} otherwise.
* @param checksum String of the new checksum to check
* @param tool String of the tool
* @param version String of the version
* @return {@code true} if update of checksum was successful, {@code false} otherwise.
*/
private boolean checkDownloadUrl(String url, UrlVersion urlVersion, OperatingSystem os,
SystemArchitecture architecture, String checksum) {
private static boolean isChecksumStillValid(String url, UrlVersion urlVersion, OperatingSystem os,
SystemArchitecture architecture, String checksum, String tool, String version) {

HttpResponse<?> response = doCheckDownloadViaHeadRequest(url);
int statusCode = response.statusCode();
boolean success = isSuccess(response);
String tool = getToolWithEdition();
String version = urlVersion.getName();
String contentType = response.headers().firstValue("content-type").orElse("undefined");
if (success && contentType.startsWith("text")) {
logger.error("For tool {} and version {} the download has an invalid content type {} for URL {}", tool, version,
contentType, url);
success = false;
}
if (success) {
UrlDownloadFile urlDownloadFile = urlVersion.getOrCreateUrls(os, architecture);
UrlChecksum urlChecksum = urlVersion.getOrCreateChecksum(urlDownloadFile.getName());
UrlDownloadFile urlDownloadFile = urlVersion.getOrCreateUrls(os, architecture);
UrlChecksum urlChecksum = urlVersion.getOrCreateChecksum(urlDownloadFile.getName());
String oldChecksum = urlChecksum.getChecksum();

if ((oldChecksum != null) && !Objects.equal(oldChecksum, checksum)) {
logger.error("For tool {} and version {} the mirror URL {} points to a different checksum {} but expected {}.",
tool, version, url, checksum, oldChecksum);
return false;
} else {
urlDownloadFile.addUrl(url);
urlChecksum.setChecksum(checksum);
String oldChecksum = urlChecksum.getChecksum();

if ((oldChecksum != null) && !Objects.equal(oldChecksum, checksum)) {
logger.error("For tool {} and version {} the mirror URL {} points to a different checksum {} but expected {}.",
tool, version, url, checksum, oldChecksum);
success = false;
} else {
urlDownloadFile.addUrl(url);
urlChecksum.setChecksum(checksum);
}
}
return true;
}

doUpdateStatusJson(success, statusCode, urlVersion, url, false);
/**
* Checks if the content type is valid (not of type text)
*
* @param url String of the url to check
* @param tool String of the tool name
* @param version String of the version
* @param response the {@link HttpResponse}.
* @return {@code true} if the content type is not of type text, {@code false} otherwise.
*/
private boolean isValidDownload(String url, String tool, String version, HttpResponse<?> response) {

if (success) {
urlVersion.save();
if (isSuccess(response)) {
String contentType = response.headers().firstValue("content-type").orElse("undefined");
if (contentType.startsWith("text")) {
logger.error("For tool {} and version {} the download has an invalid content type {} for URL {}", tool, version,
contentType, url);
return false;
}
} else {
return false;
}

return success;
return true;
}

/**
* Checks the download URL by downloading the file and generating the checksum from it
* Checks the download URL by checksum or by downloading the file and generating the checksum from it
*
* @param url the URL of the download to check.
* @param urlVersion the {@link UrlVersion} where to store the collected information like status and checksum.
Expand All @@ -250,37 +265,30 @@ private boolean checkDownloadUrl(String url, UrlVersion urlVersion, OperatingSys
* @return {@code true} if the download was checked successfully, {@code false} otherwise.
*/
private boolean checkDownloadUrl(String url, UrlVersion urlVersion, OperatingSystem os,
SystemArchitecture architecture) {
SystemArchitecture architecture, String checksum) {

HttpResponse<InputStream> response = doGetResponseAsStream(url);
HttpResponse<?> response = doCheckDownloadViaHeadRequest(url);
int statusCode = response.statusCode();
boolean success = isSuccess(response);
String contentType = response.headers().firstValue("content-type").orElse("undefined");
String tool = getToolWithEdition();
String version = urlVersion.getName();
if (success && contentType.startsWith("text")) {
logger.error("For tool {} and version {} the download has an invalid content type {} for URL {}", tool, version,
contentType, url);
success = false;
}

boolean success = isValidDownload(url, tool, version, response);

if (success) {
String checksum = doGenerateChecksum(response, url, version, contentType);
UrlDownloadFile urlDownloadFile = urlVersion.getOrCreateUrls(os, architecture);
UrlChecksum urlChecksum = urlVersion.getOrCreateChecksum(urlDownloadFile.getName());
String oldChecksum = urlChecksum.getChecksum();
if ((oldChecksum != null) && !Objects.equal(oldChecksum, checksum)) {
logger.error("For tool {} and version {} the mirror URL {} points to a different checksum {} but expected {}.",
tool, version, url, checksum, oldChecksum);
success = false;
} else {
urlDownloadFile.addUrl(url);
urlChecksum.setChecksum(checksum);
if (checksum == null || checksum.isEmpty()) {
String contentType = response.headers().firstValue("content-type").orElse("undefined");
checksum = doGenerateChecksum(doGetResponseAsStream(url), url, version, contentType);
}

success = isChecksumStillValid(url, urlVersion, os, architecture, checksum, tool, version);
}
doUpdateStatusJson(success, statusCode, urlVersion, url, false);

if (success) {
urlVersion.save();
}

doUpdateStatusJson(success, statusCode, urlVersion, url, false);

return success;
}

Expand Down Expand Up @@ -333,7 +341,7 @@ protected HttpResponse<?> doCheckDownloadViaHeadRequest(String url) {

return this.client.send(request, HttpResponse.BodyHandlers.ofString());
} catch (Exception e) {
logger.error("Failed to preform HEAD request of URL {}", url, e);
logger.error("Failed to perform HEAD request of URL {}", url, e);
return null;
}
}
Expand Down
8 changes: 4 additions & 4 deletions ...dater/src/main/java/com/devonfw/tools/ide/url/updater/docker/DockerDesktopUrlUpdater.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,8 @@ protected void addVersion(UrlVersion urlVersion) {
if (!success) {
return;
}
doAddVersion(urlVersion, "https://desktop.docker.com/mac/main/amd64/" + code + "/Docker.dmg", MAC, X64, "");
doAddVersion(urlVersion, "https://desktop.docker.com/mac/main/arm64/" + code + "/Docker.dmg", MAC, ARM64, "");
doAddVersion(urlVersion, "https://desktop.docker.com/mac/main/amd64/" + code + "/Docker.dmg", MAC, X64);
doAddVersion(urlVersion, "https://desktop.docker.com/mac/main/arm64/" + code + "/Docker.dmg", MAC, ARM64);
} else {
// For the latest version, there is no code in the url.
// TODO but that means that the implementation is wrong as the URL will then change later and is therefore
Expand All @@ -46,8 +46,8 @@ protected void addVersion(UrlVersion urlVersion) {
if (!success) {
return;
}
doAddVersion(urlVersion, "https://desktop.docker.com/mac/main/amd64/Docker.dmg", MAC, X64, "");
doAddVersion(urlVersion, "https://desktop.docker.com/mac/main/arm64/Docker.dmg", MAC, ARM64, "");
doAddVersion(urlVersion, "https://desktop.docker.com/mac/main/amd64/Docker.dmg", MAC, X64);
doAddVersion(urlVersion, "https://desktop.docker.com/mac/main/arm64/Docker.dmg", MAC, ARM64);
}

}
Expand Down
2 changes: 1 addition & 1 deletion ...rc/main/java/com/devonfw/tools/ide/url/updater/docker/DockerRancherDesktopUrlUpdater.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ protected void addVersion(UrlVersion urlVersion) {
String baseUrl = "https://github.com/rancher-sandbox/rancher-desktop/releases/download/v${version}/Rancher.Desktop";
doAddVersion(urlVersion, baseUrl + ".Setup.${version}.exe", WINDOWS);
doAddVersion(urlVersion, baseUrl + "-${version}.x86_64.dmg", MAC);
doAddVersion(urlVersion, baseUrl + "-${version}.aarch64.dmg", MAC, ARM64, "");
doAddVersion(urlVersion, baseUrl + "-${version}.aarch64.dmg", MAC, ARM64);

}
}
12 changes: 6 additions & 6 deletions url-updater/src/main/java/com/devonfw/tools/ide/url/updater/dotnet/DotNetUrlUpdater.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,15 +23,15 @@ protected String getVersionPrefixToRemove() {
protected void addVersion(UrlVersion urlVersion) {

String baseUrl = "https://dotnetcli.azureedge.net/dotnet/Sdk/${version}/dotnet-sdk-${version}-";
boolean ok1 = doAddVersion(urlVersion, baseUrl + "win-x64.zip", WINDOWS, X64, "");
boolean ok2 = doAddVersion(urlVersion, baseUrl + "win-arm64.zip", WINDOWS, ARM64, "");
boolean ok1 = doAddVersion(urlVersion, baseUrl + "win-x64.zip", WINDOWS, X64);
boolean ok2 = doAddVersion(urlVersion, baseUrl + "win-arm64.zip", WINDOWS, ARM64);
if (!ok1 && !ok2) {
return;
}
doAddVersion(urlVersion, baseUrl + "linux-x64.tar.gz", LINUX, X64, "");
doAddVersion(urlVersion, baseUrl + "linux-arm64.tar.gz", LINUX, ARM64, "");
doAddVersion(urlVersion, baseUrl + "osx-x64.tar.gz", MAC, X64, "");
doAddVersion(urlVersion, baseUrl + "osx-arm64.tar.gz", MAC, ARM64, "");
doAddVersion(urlVersion, baseUrl + "linux-x64.tar.gz", LINUX, X64);
doAddVersion(urlVersion, baseUrl + "linux-arm64.tar.gz", LINUX, ARM64);
doAddVersion(urlVersion, baseUrl + "osx-x64.tar.gz", MAC, X64);
doAddVersion(urlVersion, baseUrl + "osx-arm64.tar.gz", MAC, ARM64);
}

@Override
Expand Down
10 changes: 5 additions & 5 deletions url-updater/src/main/java/com/devonfw/tools/ide/url/updater/eclipse/EclipseUrlUpdater.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,14 +60,14 @@ protected void addVersion(UrlVersion urlVersion) {
private boolean doUpdateVersions(UrlVersion urlVersion, String baseUrl) {

boolean ok;
ok = doAddVersion(urlVersion, baseUrl + "win32-x86_64.zip", WINDOWS, X64, "");
ok = doAddVersion(urlVersion, baseUrl + "win32-x86_64.zip", WINDOWS, X64);
if (!ok) {
return false;
}
ok = doAddVersion(urlVersion, baseUrl + "linux-gtk-x86_64.tar.gz", LINUX, X64, "");
ok = doAddVersion(urlVersion, baseUrl + "linux-gtk-aarch64.tar.gz", LINUX, ARM64, "");
ok = doAddVersion(urlVersion, baseUrl + "macosx-cocoa-x86_64.tar.gz", MAC, X64, "");
ok = doAddVersion(urlVersion, baseUrl + "macosx-cocoa-aarch64.tar.gz", MAC, ARM64, "");
ok = doAddVersion(urlVersion, baseUrl + "linux-gtk-x86_64.tar.gz", LINUX, X64);
ok = doAddVersion(urlVersion, baseUrl + "linux-gtk-aarch64.tar.gz", LINUX, ARM64);
ok = doAddVersion(urlVersion, baseUrl + "macosx-cocoa-x86_64.tar.gz", MAC, X64);
ok = doAddVersion(urlVersion, baseUrl + "macosx-cocoa-aarch64.tar.gz", MAC, ARM64);
return ok;
}

Expand Down
10 changes: 5 additions & 5 deletions url-updater/src/main/java/com/devonfw/tools/ide/url/updater/gh/GhUrlUpdater.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,11 +23,11 @@ protected String getVersionPrefixToRemove() {
protected void addVersion(UrlVersion urlVersion) {

String baseUrl = "https://github.com/cli/cli/releases/download/v${version}/gh_${version}_";
doAddVersion(urlVersion, baseUrl + "windows_amd64.zip", WINDOWS, X64, "");
doAddVersion(urlVersion, baseUrl + "linux_amd64.tar.gz", LINUX, X64, "");
doAddVersion(urlVersion, baseUrl + "linux_arm64.tar.gz", LINUX, ARM64, "");
doAddVersion(urlVersion, baseUrl + "macOS_amd64.tar.gz", MAC, X64, "");
doAddVersion(urlVersion, baseUrl + "macOS_arm64.tar.gz", MAC, ARM64, "");
doAddVersion(urlVersion, baseUrl + "windows_amd64.zip", WINDOWS, X64);
doAddVersion(urlVersion, baseUrl + "linux_amd64.tar.gz", LINUX, X64);
doAddVersion(urlVersion, baseUrl + "linux_arm64.tar.gz", LINUX, ARM64);
doAddVersion(urlVersion, baseUrl + "macOS_amd64.tar.gz", MAC, X64);
doAddVersion(urlVersion, baseUrl + "macOS_arm64.tar.gz", MAC, ARM64);
}

@Override
Expand Down
8 changes: 4 additions & 4 deletions url-updater/src/main/java/com/devonfw/tools/ide/url/updater/graalvm/GraalVmUrlUpdater.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,10 +23,10 @@ protected String getVersionPrefixToRemove() {
protected void addVersion(UrlVersion urlVersion) {

String baseUrl = "https://github.com/graalvm/graalvm-ce-builds/releases/download/vm-${version}/graalvm-ce-java11-";
doAddVersion(urlVersion, baseUrl + "windows-amd64-${version}.zip", WINDOWS, X64, "");
doAddVersion(urlVersion, baseUrl + "linux-amd64-${version}.tar.gz", LINUX, X64, "");
doAddVersion(urlVersion, baseUrl + "darwin-amd64-${version}.tar.gz", MAC, X64, "");
doAddVersion(urlVersion, baseUrl + "darwin-aarch64-${version}.tar.gz", MAC, ARM64, "");
doAddVersion(urlVersion, baseUrl + "windows-amd64-${version}.zip", WINDOWS, X64);
doAddVersion(urlVersion, baseUrl + "linux-amd64-${version}.tar.gz", LINUX, X64);
doAddVersion(urlVersion, baseUrl + "darwin-amd64-${version}.tar.gz", MAC, X64);
doAddVersion(urlVersion, baseUrl + "darwin-aarch64-${version}.tar.gz", MAC, ARM64);

}

Expand Down
2 changes: 1 addition & 1 deletion url-updater/src/main/java/com/devonfw/tools/ide/url/updater/helm/HelmUrlUpdater.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ protected void addVersion(UrlVersion urlVersion) {
doAddVersion(urlVersion, baseUrl + "windows-amd64.zip", WINDOWS);
doAddVersion(urlVersion, baseUrl + "linux-amd64.tar.gz", LINUX);
doAddVersion(urlVersion, baseUrl + "darwin-amd64.tar.gz", MAC);
doAddVersion(urlVersion, baseUrl + "darwin-arm64.tar.gz", MAC, ARM64, "");
doAddVersion(urlVersion, baseUrl + "darwin-arm64.tar.gz", MAC, ARM64);
}

}
8 changes: 4 additions & 4 deletions ...ater/src/main/java/com/devonfw/tools/ide/url/updater/kotlinc/KotlincNativeUrlUpdater.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,10 @@ protected String getTool() {
protected void addVersion(UrlVersion urlVersion) {

String baseUrl = "https://github.com/JetBrains/kotlin/releases/download/v${version}/kotlin-native-";
doAddVersion(urlVersion, baseUrl + "windows-x86_64-${version}.zip", WINDOWS, X64, "");
doAddVersion(urlVersion, baseUrl + "linux-x86_64-${version}.tar.gz", LINUX, X64, "");
doAddVersion(urlVersion, baseUrl + "macos-x86_64-${version}.tar.gz", MAC, X64, "");
doAddVersion(urlVersion, baseUrl + "macos-aarch64-${version}.tar.gz", MAC, ARM64, "");
doAddVersion(urlVersion, baseUrl + "windows-x86_64-${version}.zip", WINDOWS, X64);
doAddVersion(urlVersion, baseUrl + "linux-x86_64-${version}.tar.gz", LINUX, X64);
doAddVersion(urlVersion, baseUrl + "macos-x86_64-${version}.tar.gz", MAC, X64);
doAddVersion(urlVersion, baseUrl + "macos-aarch64-${version}.tar.gz", MAC, ARM64);
}

@Override
Expand Down
12 changes: 6 additions & 6 deletions ...ater/src/main/java/com/devonfw/tools/ide/url/updater/lazydocker/LazyDockerUrlUpdater.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,12 +35,12 @@ protected String getVersionPrefixToRemove() {
protected void addVersion(UrlVersion urlVersion) {

String baseUrl = "https://github.com/jesseduffield/lazydocker/releases/download/v${version}/lazydocker_${version}_";
doAddVersion(urlVersion, baseUrl + "Windows_x86_64.zip", WINDOWS, X64, "");
doAddVersion(urlVersion, baseUrl + "Windows_arm64.zip", WINDOWS, ARM64, "");
doAddVersion(urlVersion, baseUrl + "Linux_x86_64.tar.gz", LINUX, X64, "");
doAddVersion(urlVersion, baseUrl + "Linux_arm64.tar.gz", LINUX, ARM64, "");
doAddVersion(urlVersion, baseUrl + "Darwin_x86_64.tar.gz", MAC, X64, "");
doAddVersion(urlVersion, baseUrl + "Darwin_arm64.tar.gz", MAC, ARM64, "");
doAddVersion(urlVersion, baseUrl + "Windows_x86_64.zip", WINDOWS, X64);
doAddVersion(urlVersion, baseUrl + "Windows_arm64.zip", WINDOWS, ARM64);
doAddVersion(urlVersion, baseUrl + "Linux_x86_64.tar.gz", LINUX, X64);
doAddVersion(urlVersion, baseUrl + "Linux_arm64.tar.gz", LINUX, ARM64);
doAddVersion(urlVersion, baseUrl + "Darwin_x86_64.tar.gz", MAC, X64);
doAddVersion(urlVersion, baseUrl + "Darwin_arm64.tar.gz", MAC, ARM64);
}

}
6 changes: 3 additions & 3 deletions url-updater/src/main/java/com/devonfw/tools/ide/url/updater/node/NodeUrlUpdater.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,11 +30,11 @@ protected void addVersion(UrlVersion urlVersion) {

String baseUrl = "https://nodejs.org/dist/${version}/node-${version}-";
doAddVersion(urlVersion, baseUrl + "win-x64.zip", WINDOWS);
doAddVersion(urlVersion, baseUrl + "win-aarch64.zip", WINDOWS, ARM64, "");
doAddVersion(urlVersion, baseUrl + "win-aarch64.zip", WINDOWS, ARM64);
doAddVersion(urlVersion, baseUrl + "linux-x64.tar.gz", LINUX);
doAddVersion(urlVersion, baseUrl + "linux-aarch64.tar.gz", LINUX, ARM64, "");
doAddVersion(urlVersion, baseUrl + "linux-aarch64.tar.gz", LINUX, ARM64);
doAddVersion(urlVersion, baseUrl + "darwin-x64.tar.gz", MAC);
doAddVersion(urlVersion, baseUrl + "darwin-aarch64.tar.gz", MAC, ARM64, "");
doAddVersion(urlVersion, baseUrl + "darwin-aarch64.tar.gz", MAC, ARM64);
}

}
Loading

0 comments on commit 3bf8654

Please sign in to comment.