do HTTP caching ourselves

import-map.json

@@ -11,7 +11,6 @@
     "cliffy/": "https://deno.land/x/[email protected]/",
     "s3": "https://deno.land/x/[email protected]/mod.ts",
     "encodeToString": "https://deno.land/[email protected]/encoding/hex.ts",
-    "mxcl/deno-cache": "https://raw.githubusercontent.com/mxcl/deno-cache/0.2.15/mod.ts",
     "outdent": "https://deno.land/x/[email protected]/mod.ts"
   }
 }

src/hooks/useCache.ts

@@ -5,7 +5,7 @@ import * as _ from "utils" // console.verbose
 import useCellar from "hooks/useCellar.ts"
 
 interface DownloadOptions {
-  url: string
+  url: URL
   pkg: Package
   type?: 'src' | 'bottle'
 }
@@ -31,27 +31,21 @@ const bottle = (pkg: Package) => {
 /// download source or bottle
 const download = async ({ url: readURL, pkg, type = 'bottle' }: DownloadOptions) => {
   const filename = (() => {
-    switch(type!) {
-      case 'src': return stem(pkg) + Path.extname(readURL)
+    switch(type) {
+      case 'src': return stem(pkg) + Path.extname(readURL.pathname)
       case 'bottle': return bottle(pkg).string
     }
   })()
   const writeFilename = prefix.join(filename)
-  console.debug(writeFilename)
-  if (writeFilename.isReadableFile()) {
-    console.info({alreadyDownloaded: writeFilename})
-  } else {
-    console.info({downloading: readURL})
-    //FIXME: big hacks
-    const privateRepo = pkg.project === "tea.xyz"
-    await grab({ readURL, writeFilename, privateRepo })
-  }
+  const privateRepo = pkg.project === "tea.xyz" //FIXME: big hacks
+  await grab({ readURL, writeFilename, privateRepo })
   return writeFilename
 }
 
 const download_script = async (url: URL) => {
-  const file = await dl(url)
-  return new Path(file.path)
+  const writeFilename = hash_key(url).join(new Path(url.pathname).basename())
+  await grab({ readURL: url, writeFilename, privateRepo: false })
+  return writeFilename
 }
 
 /// lists all packages with bottles in the cache
@@ -80,40 +74,70 @@ const s3Key = (pkg: Package) => {
   return `${pkg.project}/${platform}/${arch}/v${pkg.version.version}.tar.gz`
 }
 
-async function grab({ readURL, writeFilename, privateRepo = false }: { readURL: string, writeFilename: Path, privateRepo: boolean }) {
+import { readerFromStreamReader, copy } from "deno/streams/conversion.ts"
+
+async function grab({ readURL: url, writeFilename: dst, privateRepo = false }: { readURL: URL, writeFilename: Path, privateRepo: boolean }) {
+
+  if (url.protocol === "file:") throw new Error()
+
   const { verbose } = console
 
-  if (writeFilename.isReadableFile()) return
+  verbose({src: url, dst})
 
-  verbose({downloading: readURL})
-  verbose({destination: writeFilename})
+  const headers: HeadersInit = {}
 
   //TODO: remove; special casing for private tea repos
   if (privateRepo) {
-    const url = new URL(readURL)
     if (url.host != "github.com") { throw new Error("unknown private domain") }
     const token = Deno.env.get("GITHUB_TOKEN")
     if (!token) { throw new Error("private repos require a GITHUB_TOKEN") }
-    const rsp = await fetch(url, { headers: { authorization: `bearer ${token}`} })
-    const file = await Deno.open(writeFilename.string, { create: true, write: true })
-    await rsp.body?.pipeTo(file.writable)
-    return
+    headers["Authorization"] = `bearer ${token}`
   }
-  const file = await dl(readURL)
-  await Deno.link(file.path, writeFilename.string)
-}
 
+  const mtime_entry = hash_key(url).join("mtime")
+  if (mtime_entry.isFile() && dst.isReadableFile()) {
+    headers["If-Modified-Since"] = await mtime_entry.read()
+  }
 
-///////////////////////////////////////////////////////////////////////// HTTP
-import { cache, File, Policy, configure } from "mxcl/deno-cache"
+  const rsp = await fetch(url, { headers })
+
+  switch (rsp.status) {
+  case 200: {
+    const rdr = rsp.body?.getReader()
+    if (!rdr) throw new Error()
+    const r = readerFromStreamReader(rdr)
+    const f = await Deno.open(dst.string, {create: true, write: true})
+    try {
+      await copy(r, f)
+    } finally {
+      f.close()
+    }
+
+    //TODO etags too
+    utils.flatMap(rsp.headers.get("Last-Modified"), text => mtime_entry.write({ text }))
+
+  } break
+  case 304:
+    console.verbose("304: not modified")
+    return  // not modified
+  case 404:
+    throw new Error(`404: ${url}`)
+  default:
+    throw new Error()
+  }
+}
+
+import { createHash } from "deno/hash/mod.ts"
 
-//FIXME lol better
-configure({ directory: prefix.string })
+function hash_key(url: URL): Path {
+  function hash(url: URL) {
+    const formatted = `${url.pathname}${url.search ? "?" + url.search : ""}`;
+    return createHash("sha256").update(formatted).toString();
+  }
 
-function dl(
-  url: string | URL,
-  policy?: Policy,
-  ns?: string,
-): Promise<File> {
-  return cache(url, policy, ns)
+  return prefix
+    .join(url.protocol.slice(0, -1))
+    .join(url.hostname)
+    .join(hash(url))
+    .mkpath()
 }

src/hooks/useGitHubAPI.ts

@@ -1,4 +1,3 @@
-import { RELOAD_POLICY } from "mxcl/deno-cache";
 import { GET, isArray, undent, validateArray, validateString } from "utils"
 
 //TODO pagination

src/hooks/usePantry.ts

@@ -59,16 +59,18 @@ const getRawDistributableURL = (yml: PlainObject) => validateString(
 
 const getDistributable = async (pkg: Package) => {
   const yml = await entry(pkg).yml()
-  let url = getRawDistributableURL(yml)
+  let urlstr = getRawDistributableURL(yml)
   let stripComponents: number | undefined
   if (isPlainObject(yml.distributable)) {
-    url = validateString(yml.distributable.url)
+    urlstr = validateString(yml.distributable.url)
     stripComponents = flatMap(yml.distributable["strip-components"], coerceNumber)
   } else {
-    url = validateString(yml.distributable)
+    urlstr = validateString(yml.distributable)
   }
 
-  url = remapTokens(url, pkg)
+  urlstr = remapTokens(urlstr, pkg)
+
+  const url = new URL(urlstr)
 
   return { url, stripComponents }
 }

src/prefab/install.ts

@@ -22,7 +22,7 @@ export default async function install(pkg: Package): Promise<Installation> {
   const { project, version } = pkg
   const { finalizeInstall } = usePlatform()
   const { s3Key, download } = useCache()
-  const url = `https://dist.tea.xyz/${s3Key(pkg)}`
+  const url = new URL(`https://dist.tea.xyz/${s3Key(pkg)}`)
   const { prefix: dstdir, ...cellar } = useCellar()
   const { verbosity } = useFlags()