downloading things

build-manifests.js

@@ -1,29 +1,63 @@
 
 var fs = require("fs");
 var resolve = require("path").resolve;
+var download = require("./download");
 
 var tools = fs.readdirSync(resolve(__dirname, "packages"));
 
 tools.forEach(function (toolName) {
-  var fileData = fs.readFileSync(resolve(__dirname, "packages", toolName, "manifest.json"), "utf8");
-  var getUrl = require(resolve(__dirname, "packages", toolName)).getUrl;
+
+  var manifestPath = resolve(__dirname, "packages", toolName, "manifest.json");
+  var fileData = fs.readFileSync(manifestPath);
   var manifest = JSON.parse(fileData);
+
+  var save = function() {
+    console.log("Saving " + manifestPath);
+    var outputStr = JSON.stringify(manifest, null, 2);
+    fs.writeFileSync(manifestPath, outputStr, "utf8");
+  };
+
+  var getUrl = require(resolve(__dirname, "packages", toolName)).getUrl;
   var tags = fs.readFileSync(resolve(__dirname, toolName + ".tags"), "utf8").trim().split("\n");
 
   // Make sure we have an object for each computer/platform/arch combo
   manifest.supported_computers.forEach(function(computer) {
-    console.log(computer);
     var platform = computer.platform;
     var arch = computer.arch;
     tags.forEach(function(version) {
       var verTag = [version, platform, arch].join("-");
       if (manifest.versions[verTag] === undefined) {
-        manifest.versions[verTag] = {
-          url: getUrl(version, platform, arch),
-        };
+        manifest.versions[verTag] = {};
+      }
+      // If null, we've determined this is a bum pair
+      if (manifest.versions[verTag] === null) {
+        return;
       }
+      manifest.versions[verTag].url = getUrl(version, platform, arch);
     });
   });
-  const outputStr = JSON.stringify(manifest, null, 2);
-  fs.writeFileSync(resolve(__dirname, "packages", toolName, "manifest.json"), outputStr, "utf8")
+
+  save();
+
+  const getInfo = function(version) {
+    download(version, function(err, data) {
+      if (err) {
+        console.error(err);
+        process.exit(1);
+      }
+      version.size = data.size;
+      version.sha256 = data.sha256;
+      save();
+    });
+  };
+
+  var versionsToFix = [];
+  Object.keys(manifest.versions).forEach(function(versionTag) {
+    var version = manifest.versions[versionTag];
+    if (!manifest.sha256) {
+      versionsToFix.push(version);
+    }
+  });
+
+  getInfo(manifest.versions[Object.keys(manifest.versions)[0]]);
 });

download.js

@@ -1,8 +1,62 @@
 
 var https = require("https");
+var resolve = require("path").resolve;
+var fs = require("fs");
+var TIMEOUT_DELAY = 20000;
+var PERCENTAGE_INTERVAL = .1;
+var crypto = require("crypto");
 
-module.exports = function(url, cb) {
-  https.get(url).on("data", function(data) {
-    console.log(data.length);
+module.exports = function(version, cb) {
+  if (!version.sha256 && !process.env.DONT_WORRY_I_AM_BUILDING_A_NEW_VERSION) {
+    throw new Error("Unknown SHA256 hash for this version " + JSON.stringify(version));
+  }
+  var filename = version.url.split("/").pop();
+  process.stderr.write("Downloading " + filename + "... ");
+  var outputPath = resolve(__dirname, filename);
+  var outputStream = fs.createWriteStream(outputPath);
+  var shasum = crypto.createHash("sha256");
+
+  var err = function(message) {
+    cb(new Error(message + " while trying to download " + filename + " from " + version.url));
+  };
+
+  var timeout = setTimeout(function() {
+    cb(new Error("HTTP Timeout, waited " + TIMEOUT_DELAY + "ms."));
+  }, TIMEOUT_DELAY);
+
+  var lastUpdated = -PERCENTAGE_INTERVAL;
+
+  https.get(version.url, function(res) {
+    clearTimeout(timeout);
+    if (res.statusCode !== 200) {
+      return err("HTTP " + res.statusCode);
+    }
+    var size = 0;
+    res.on("end", function() {
+      // Make double-sure we're after the last "data"
+      setTimeout(function() {
+        process.stderr.write("Done!\n");
+        var resultHash = shasum.digest("hex");
+        if (version.sha256 && resultHash !== version.sha256) {
+          fs.unlinkSync(outputPath);
+          return err("SHA256 mismatch, expected " + version.sha256 + " got " + resultHash);
+        }
+        cb(null, {
+          size: size,
+          path: outputPath,
+          sha256: resultHash,
+        });
+      }, 0);
+    })
+    .on("data", function(data) {
+      shasum.update(data);
+      size += data.length;
+      if (version.size && ((size / version.size) - lastUpdated > PERCENTAGE_INTERVAL)) {
+        lastUpdated = size / version.size;
+        var percent = Math.round(lastUpdated * 100);
+        process.stderr.write(percent + "% ");
+      }
+    })
+    .pipe(outputStream);
   });
 }

packages/kubectl/manifest.json

@@ -43,7 +43,9 @@
   ],
   "versions": {
     "v0.10.0-darwin-x64": {
-      "url": "https://storage.googleapis.com/kubernetes-release/release/v0.10.0/bin/darwin/amd64/kubectl"
+      "url": "https://storage.googleapis.com/kubernetes-release/release/v0.10.0/bin/darwin/amd64/kubectl",
+      "size": 14071680,
+      "sha256": "05a966053691819c20e07e1afb8debe1e2edd2ab9c866ad6f0324d3cfe7a1c89"
     },
     "v0.10.1-darwin-x64": {
       "url": "https://storage.googleapis.com/kubernetes-release/release/v0.10.1/bin/darwin/amd64/kubectl"
@@ -6313,4 +6315,4 @@
       "url": "https://storage.googleapis.com/kubernetes-release/release/v1.6.0-alpha.2/bin/windows/386/kubectl"
     }
   }
-}
+}