refactor(package): setup and config files for vax pipeline

scripts/MANIFEST.in

@@ -0,0 +1,6 @@
+include *.md
+include requirements.txt
+include requirements-flake.txt
+
+recursive-include altair *.py *.json *.ipynb *.html
+global-exclude *.py[co] __pycache__

scripts/config.yaml

@@ -0,0 +1,36 @@
+global:
+  project_dir: !ENV ${OWID_COVID_PROJECT_DIR}
+  credentials: !ENV ${OWID_COVID_VAX_CREDENTIALS_FILE}
+pipeline:
+  get-data:
+    parallel: True
+    countries:
+    njobs: -2
+    skip_countries:
+      - Gabon
+      - North Macedonia
+      - Northern Cyprus
+      - South Africa
+  process-data:
+    skip_complete:
+      - Pitcairn
+    skip_monotonic_check:
+    skip_anomaly_check:
+      Australia:
+        - date: 2021-05-24
+          metrics: people_vaccinated
+      Bahrain:
+        - date: 2021-03-06
+          metrics: total_vaccinations
+      Bolivia:
+        - date: 2021-03-06
+          metrics: people_vaccinated
+      Brazil:
+        - date: 2021-01-21
+          metrics: 
+           - total_vaccinations
+           - people_vaccinated
+      Nigeria:
+        - date: 2021-06-15
+          metrics: people_fully_vaccinated
+  generate-dataset:

scripts/requirements.txt

@@ -1,55 +1,31 @@
-aiohttp==3.7.4
-appnope==0.1.0
-async-timeout==3.0.1
-attrs==19.3.0
-backcall==0.1.0
 beautifulsoup4==4.9.3
-blessings==1.7
-bs4==0.0.1
-certifi~=2021.5.3
-chardet==3.0.4
-decorator==4.4.2
-et-xmlfile==1.0.1
-idna==2.10
-imutils==0.5.4
-inquirer==2.6.3
-jdcal==1.4.1
-jedi==0.16.0
-lxml==4.6.3
-multidict==4.7.6
-ndg-httpsclient==0.5.1
-numpy==1.21.0
-openpyxl==3.0.3
-pandas==1.3.0
-parso==0.6.2
-pexpect==4.8.0
-pickleshare==0.7.5
-prompt-tool-kit==1.0.14
-prompt-toolkit==3.0.4
-ptyprocess==0.6.0
-pyasn1==0.4.8
-Pygments==2.7.4
+chromedriver-binary==87.0.4280.88.0
+dateparser==1.0.0
+facebook-scraper==0.2.45
+gsheets==0.5.1
+joblib==1.0.1
+lxml>=4.6.3
+numpy~=1.21.0
+odfpy~=1.4.1
+openpyxl==3.0.7
+pandas~=1.3.0
+pdfreader==0.1.10
 PyMySQL==0.9.3
-pyOpenSSL==20.0.1
-pytesseract==0.3.7
-python-dateutil==2.8.1
-python-dotenv==0.14.0
-python-editor==1.0.4
-pytz==2019.3
+PyPDF2==1.26.0
+python-dotenv~=0.18.0
 PyYAML==5.4.1
-readchar==2.0.1
-regex==2020.2.20
-requests==2.25.1
-six==1.14.0
+pyaml-env~=1.1.0
+pyperclip==1.8.2
+pytz==2020.5
+requests~=2.25.1
+selenium==3.141.0
 slackclient==2.9.3
-soupsieve==2.2
+tabulate==0.8.9
+tabula-py==2.2.0
 termcolor==1.1.0
-tqdm==4.59.0
-traitlets==4.3.3
-typing-extensions==3.7.4.3
-Unidecode==1.1.1
-urllib3==1.26.5
-wcwidth==0.1.8
-xlrd==1.2.0
-XlsxWriter==1.3.7
-yarl==1.4.2
+tqdm==4.61.1
+tweepy~=3.10.0
+uk-covid19~=1.2.0
+Unidecode~=1.2.0
+xlrd==2.0.1
+XlsxWriter==1.4.3

scripts/setup.py

@@ -9,7 +9,7 @@
 # with open(os.path.join(this_directory, "README.md")) as f:
 #    long_description = f.read()
 
-with open(os.path.join(this_directory, "requirements-new.txt")) as f:
+with open(os.path.join(this_directory, "requirements.txt")) as f:
     requirements = f.readlines()
 
 
@@ -50,7 +50,9 @@
     python_requires=">=3.7",
     entry_points={
         "console_scripts": [
-            "cowidev-grapher-db=cowidev.grapher.db.__main__:main" ,
+            "cowidev-grapher-db=cowidev.grapher.db.__main__:main",
+            "cowid-vax=cowidev.vax.cmd.__main__:main",
+            "cowid-vax-track=cowidev.vax.tracking.__main__:main",
         ]
-    }
+    },
 )

scripts/tox.ini

@@ -0,0 +1,22 @@
+[tox]
+envlist = py39
+
+[testenv]
+basepython=python3.9
+skipsdist=True
+deps =
+    tox
+commands =
+    tox -e flake8
+
+[testenv:flake8]
+deps =
+    -rrequirements-flake.txt
+commands =
+    flake8 --max-complexity 10 --format=html --htmldir=reports/flake {posargs} "{envsitepackagesdir}/vax/"
+
+[flake8]
+max_line_length = 119
+max_complexity = 10
+docstring_convention = google
+ignore = D100,D101,D102,D103,D104,D107,S101

scripts/vax_dataset_config.json

@@ -0,0 +1,8 @@
+{
+    "greece_api_token": "b1ef5949bebace574a0d7e58b5cdf4018353121e",
+    "owid_cloud_table_post": "https://owid.cloud/wp/wp-admin/post.php?post=38102&action=edit",
+    "google_credentials": "/Users/lucasrodes/.config/gspread/credentials.json",
+    "google_spreadsheet_vax_id": "1ogjVP-0PwbRieUIy7nBkIP4gD6sZn2JqKWoY3ivJT5A",
+    "twitter_consumer_key": "liw78E8eg1n0QWo3QY3brMeeG",
+    "twitter_consumer_secret": "KYPsawtydDyXQ3VfGBM9R3GeFRITLuGesuP4ueeOq6S85o0AJK"
+}