A script to generate known versions data for the terraform binary (pa…

…ntsbuild#15958) Scrapes recent versions of Terraform, and uses it to update the known_versions. [ci skip-rust] [ci skip-build-wheels]
purajit · Jul 29, 2022 · 4785621 · 4785621
1 parent 4436017
commit 4785621
Show file tree

Hide file tree

Showing 10 changed files with 606 additions and 43 deletions.
diff --git a/3rdparty/python/BUILD b/3rdparty/python/BUILD
@@ -4,6 +4,8 @@
 python_requirements(
     module_mapping={
         "strawberry-graphql": ["strawberry"],
+        "beautifulsoup4": ["bs4"],
+        "python-gnupg": ["gnupg"],
     },
     overrides={
         "humbug": {"dependencies": ["#setuptools"]},

diff --git a/3rdparty/python/requirements.txt b/3rdparty/python/requirements.txt
@@ -48,3 +48,6 @@ fastapi==0.78.0
 starlette==0.19.1
 strawberry-graphql[fastapi]==0.114.0
 uvicorn[standard]==0.17.6
+
+beautifulsoup4==4.11.1 # Util for webcrawling for pulling in tool versions
+python-gnupg==0.4.9 # For validating signatures
diff --git a/3rdparty/python/user_reqs.lock b/3rdparty/python/user_reqs.lock
@@ -11,6 +11,7 @@
 //   "generated_with_requirements": [
 //     "PyYAML<7.0,>=6.0",
 //     "ansicolors==1.1.8",
+//     "beautifulsoup4==4.11.1",
 //     "chevron==0.14.0",
 //     "debugpy==1.6.0",
 //     "fastapi==0.78.0",
@@ -24,6 +25,7 @@
 //     "psutil==5.9.0",
 //     "pydevd-pycharm==203.5419.8",
 //     "pytest<7.1.0,>=6.2.4",
+//     "python-gnupg==0.4.9",
 //     "python-lsp-jsonrpc==1.0.0",
 //     "requests[security]>=2.28.1",
 //     "setproctitle==1.2.2",
@@ -202,6 +204,28 @@
           "requires_python": ">=3.6.0",
           "version": "1.0.2"
         },
+        {
+          "artifacts": [
+            {
+              "algorithm": "sha256",
+              "hash": "58d5c3d29f5a36ffeb94f02f0d786cd53014cf9b3b3951d42e0080d8a9498d30",
+              "url": "https://files.pythonhosted.org/packages/9c/d8/909c4089dbe4ade9f9705f143c9f13f065049a9d5e7d34c828aefdd0a97c/beautifulsoup4-4.11.1-py3-none-any.whl"
+            },
+            {
+              "algorithm": "sha256",
+              "hash": "ad9aa55b65ef2808eb405f46cf74df7fcb7044d5cbc26487f96eb2ef2e436693",
+              "url": "https://files.pythonhosted.org/packages/e8/b0/cd2b968000577ec5ce6c741a54d846dfa402372369b8b6861720aa9ecea7/beautifulsoup4-4.11.1.tar.gz"
+            }
+          ],
+          "project_name": "beautifulsoup4",
+          "requires_dists": [
+            "html5lib; extra == \"html5lib\"",
+            "lxml; extra == \"lxml\"",
+            "soupsieve>1.2"
+          ],
+          "requires_python": ">=3.6.0",
+          "version": "4.11.1"
+        },
         {
           "artifacts": [
             {
@@ -1190,6 +1214,24 @@
           "requires_python": ">=3.5",
           "version": "0.20"
         },
+        {
+          "artifacts": [
+            {
+              "algorithm": "sha256",
+              "hash": "012960bde4d25dad631bb7650f563dda5e7271248a73f3584240063a293d99d8",
+              "url": "https://files.pythonhosted.org/packages/a7/01/7cd32f2e2b1db9ee7eeeb0fb729bedf71903d00ccc2c13a590576a62aa91/python_gnupg-0.4.9-py2.py3-none-any.whl"
+            },
+            {
+              "algorithm": "sha256",
+              "hash": "aaa748795572591aaf127b4ac8985684f3673ff82b39f370c836b006e68fc537",
+              "url": "https://files.pythonhosted.org/packages/c8/cb/46fb80639cf0dd4251aeb075a1a5e2ebbb8c9656f28ddfe9d8c99b68b6da/python-gnupg-0.4.9.tar.gz"
+            }
+          ],
+          "project_name": "python-gnupg",
+          "requires_dists": [],
+          "requires_python": null,
+          "version": "0.4.9"
+        },
         {
           "artifacts": [
             {
@@ -1492,6 +1534,26 @@
           "requires_python": ">=3.5",
           "version": "1.2"
         },
+        {
+          "artifacts": [
+            {
+              "algorithm": "sha256",
+              "hash": "3b2503d3c7084a42b1ebd08116e5f81aadfaea95863628c80a3b774a11b7c759",
+              "url": "https://files.pythonhosted.org/packages/16/e3/4ad79882b92617e3a4a0df1960d6bce08edfb637737ac5c3f3ba29022e25/soupsieve-2.3.2.post1-py3-none-any.whl"
+            },
+            {
+              "algorithm": "sha256",
+              "hash": "fc53893b3da2c33de295667a0e19f078c14bf86544af307354de5fcf12a3f30d",
+              "url": "https://files.pythonhosted.org/packages/f3/03/bac179d539362319b4779a00764e95f7542f4920084163db6b0fd4742d38/soupsieve-2.3.2.post1.tar.gz"
+            }
+          ],
+          "project_name": "soupsieve",
+          "requires_dists": [
+            "backports-functools-lru-cache; python_version < \"3\""
+          ],
+          "requires_python": ">=3.6",
+          "version": "2.3.2.post1"
+        },
         {
           "artifacts": [
             {
@@ -2256,6 +2318,7 @@
   "requirements": [
     "PyYAML<7.0,>=6.0",
     "ansicolors==1.1.8",
+    "beautifulsoup4==4.11.1",
     "chevron==0.14.0",
     "debugpy==1.6.0",
     "fastapi==0.78.0",
@@ -2269,6 +2332,7 @@
     "psutil==5.9.0",
     "pydevd-pycharm==203.5419.8",
     "pytest<7.1.0,>=6.2.4",
+    "python-gnupg==0.4.9",
     "python-lsp-jsonrpc==1.0.0",
     "requests[security]>=2.28.1",
     "setproctitle==1.2.2",

diff --git a/build-support/bin/BUILD b/build-support/bin/BUILD
@@ -18,3 +18,8 @@ python_tests(
     name="py_tests",
     overrides={"reversion_test.py": {"timeout": 90, "dependencies": ["3rdparty/python#pex"]}},
 )
+
+pex_binary(
+    name="terraform_tool_versions",
+    entry_point="terraform_tool_versions.py",
+)