Fetch PyPI package info using pypi-to-sqlite, refs simonw#98

requirements.txt

@@ -14,6 +14,8 @@ github-to-sqlite>=2.8.2
 python-graphql-client
 httpx
 feedparser
+pypi-to-sqlite
+pytest
 # These are used in deploy, listing them here so they
 # show up in the GitHub dependency graph:
 datasette-json-html
@@ -25,5 +27,4 @@ datasette-graphql
 datasette-json-html
 datasette-debug-asgi
 html5lib
-beautifulsoup4
-pytest
+beautifulsoup4

scripts/build.sh

@@ -18,6 +18,19 @@ yaml-to-sqlite content.db tool_repos tool_repos.yml --single-column repo
 python build_directory.py content.db --fetch-missing-releases \
    --always-fetch-releases-for-repo simonw/datasette-app
 
+# And fetch data from PyPI via the pypi-datasette-packages cache
+if [ ! -d /tmp/pypi-datasette-packages ]
+then
+  git clone https://github.com/simonw/pypi-datasette-packages /tmp/pypi-datasette-packages
+else
+  (cd /tmp/pypi-datasette-packages && git pull)
+fi
+
+args=$(ls /tmp/pypi-datasette-packages/packages/*.json | awk '{print "-f "$0 " \\"}')
+# Load that into pypi_packages/pypi_versions/pypi_releases
+eval "pypi-to-sqlite content.db $args
+--prefix pypi_"
+
 # Fetch my relevant blog content
 python fetch_blog_content.py blog.db datasette dogsheep sqliteutils