Add 'integrations/airflow/' from commit '86b7fe8976bf500304ad17bccd93…

…1e96aa105f01'

git-subtree-dir: integrations/airflow
git-subtree-mainline: 80c3525
git-subtree-split: 86b7fe8

integrations/airflow/.circleci/config.yml

@@ -0,0 +1,75 @@
+version: 2
+
+defaults: &defaults
+  docker:
+    - image: circleci/python:3.8.5
+  working_directory: ~/marquez-airflow
+  environment:
+    SLUGIFY_USES_TEXT_UNIDECODE: 'yes'
+
+jobs:
+  build:
+    <<: *defaults
+    steps:
+      - checkout
+      - run:
+          name: Install airflow from package
+          command: pip install --user -e .
+      - run: pip install --user flake8
+      - run: python -m flake8 --exclude integration
+
+  test:
+    <<: *defaults
+    steps:
+      - checkout
+      - run:
+          name: Install test dependencies
+          command: pip install --user -r test-requirements.txt --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-1.10.12/constraints-3.7.txt"
+      - run:
+          name: Setup Airflow DB
+          command: airflow initdb
+      - run:
+          name: Install marquez-airflow package
+          command: pip install --user -e .
+      - run:
+          name:
+          environment:
+            AIRFLOW__CORE__DAGS_FOLDER: './tests/test_dags'
+          command: python -m pytest --ignore=tests/integration --cov=marquez_airflow tests/
+      - run: bash <(curl -s https://codecov.io/bash)
+
+  integration:
+    working_directory: ~/marquez-airflow/tests/integration
+    machine: true
+    steps:
+      - checkout:
+          path: ~/marquez-airflow
+      - run: ./../../.circleci/get-docker-compose.sh
+      - run: ./docker/up.sh
+
+  release:
+    <<: *defaults
+    steps:
+      - checkout
+      - run: ./.circleci/release_check.sh
+      - run: pip install --user twine
+      - run: python setup.py sdist
+      - run: python -m twine upload --verbose dist/*
+
+workflows:
+  version: 2
+  marquez_airflow:
+    jobs:
+      - build
+      - test:
+          requires:
+            - build
+      - integration:
+          requires:
+            - test
+      - release:
+          filters:
+            tags:
+              only: /^[0-9]+(\.[0-9]+){2}$/
+            branches:
+              ignore: /.*/

integrations/airflow/.circleci/get-docker-compose.sh

@@ -0,0 +1,24 @@
+
+#!/bin/bash
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Usage: $ ./get-docker-compose.sh
+
+set -e
+
+curl -L https://github.com/docker/compose/releases/download/1.25.3/docker-compose-`uname -s`-`uname -m` > ~/docker-compose
+chmod +x ~/docker-compose
+sudo mv ~/docker-compose /usr/local/bin/docker-compose
+
+echo "DONE!"

integrations/airflow/.circleci/release_check.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e -x -u
+
+expected_version=$CIRCLE_TAG
+current_version=$(python ./setup.py --version)
+echo "current_version is ${current_version}"
+echo "expected_version is ${expected_version}"
+
+if [[ "${current_version}" != "${expected_version}" ]]; then
+  echo "Packaging code ${current_version} does not equal to tagged version, ${expected_version}"
+  exit 1
+else
+  exit 0
+fi

integrations/airflow/.gitignore

@@ -0,0 +1,10 @@
+*.egg-info
+.coverage
+.idea/
+__pycache__
+bin/
+build/
+dist/
+lib/
+tests/integration/requirements.txt
+tests/integration/integration-requirements.txt

integrations/airflow/CHANGELOG.md

@@ -0,0 +1,36 @@
+# Changelog
+
+## [Unreleased](https://github.com/MarquezProject/marquez-airflow/compare/0.3.8...HEAD)
+
+## [0.3.8](https://github.com/MarquezProject/marquez-airflow/compare/0.3.7...0.3.8) - 2020-12-08
+
+### Added
+
+* Add schemas resolution to big query output datasets [@henneberger](https://github.com/henneberger)
+
+### Deprecated
+
+* Deprecate support for airflow 1.10.3
+
+### Fixed
+
+* Fix `bigquery.Client.close()`
+* Fix thrown exception on `Utils.get_location()`
+* Fix bug where all runs fail if DAG fails [@henneberger](https://github.com/henneberger)
+* Fix bug where big query operator creates duplicate job versions [@henneberger](https://github.com/henneberger)
+* Fix extract not reporting task state on callback & update api flow [@henneberger](https://github.com/henneberger)
+
+## [0.3.7](https://github.com/MarquezProject/marquez-airflow/compare/0.3.6...0.3.7) - 2020-11-09
+
+### Added
+
+* Add support to get postgres table schemas
+* Extract inputs / outputs from BigQuery jobs object (please see [Job](https://cloud.google.com/bigquery/docs/reference/rest/v2/Job) docs) [@julienledem](https://github.com/julienledem)
+
+### Changed
+
+* Bump `marquez-python` to [`0.7.11`](https://github.com/MarquezProject/marquez-python/releases/tag/0.7.11)
+
+### Fixed
+
+* Assure extractor associates only the output dataset on complete [@henneberger](https://github.com/henneberger)

integrations/airflow/CODE_OF_CONDUCT.md

@@ -0,0 +1,76 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to making participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, sex characteristics, gender identity and expression,
+level of experience, education, socio-economic status, nationality, personal
+appearance, race, religion, or sexual identity and orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+ advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+ address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+ professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community. Examples of
+representing a project or community include using an official project e-mail
+address, posting via an official social media account, or acting as an appointed
+representative at an online or offline event. Representation of a project may be
+further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at [email protected]. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
+
+[homepage]: https://www.contributor-covenant.org
+
+For answers to common questions about this code of conduct, see
+https://www.contributor-covenant.org/faq

integrations/airflow/CONTRIBUTING.md

@@ -0,0 +1,75 @@
+# How to Contribute
+
+We're excited you're interested in contributing to Marquez! We'd love your help, and there are plenty of ways to contribute:
+
+* Fix or [report](https://github.com/MarquezProject/marquez-airflow/issues/new) a bug
+* Fix or improve documentation
+* Pick up a ["good first issue"](https://github.com/MarquezProject/marquez-airflow/labels/good%20first%20issue), then send a pull request our way
+
+We feel that a welcoming community is important and we ask that you follow the [Contributor Covenant Code of Conduct](https://github.com/MarquezProject/marquez-airflow/blob/main/CODE_OF_CONDUCT.md) in all interactions with the community.
+
+# Development
+We recommend installing the following tools to run the tests:
+
+Install pyenv: `brew install pyenv pyenv-virtualenv`
+Add it to your profile: `~/.zshrc` or `~/.bashrc` depending on your shell
+```
+if command -v pyenv 1>/dev/null 2>&1; then
+  eval "$(pyenv init -)"
+  eval "$(pyenv virtualenv-init -)"
+fi
+```
+create a marquez-airflow virtual env:
+```
+pyenv virtualenv marquez-airflow
+pyenv activate marquez-airflow
+```
+(depending on the version of python you're running, you may need to alias pip to pip3)
+
+Install the test dependencies:
+```
+pip3 install -r test-requirements.txt
+```
+
+Init the airflow local db:
+```
+airflow initdb
+```
+
+now you can run the tests:
+`python -m pytest  --ignore=tests/integration tests/`
+
+To run the linter locally:
+install flake8
+```
+pip install flake8
+```
+and run it:
+```
+python -m flake8
+```
+
+# Submitting a [pull request](https://help.github.com/articles/about-pull-requests)
+
+1. [Fork](https://github.com/MarquezProject/marquez-airflow/fork) and clone the repository
+2. Make sure all tests pass locally: `python -m pytest --ignore=tests/integration`
+3. Create a new branch: `git checkout -b my-cool-new-branch`
+4. Make change on your cool new branch
+5. Write a test for your change
+6. Push change to your fork and [submit a pull request](https://github.com/MarquezProject/marquez-airflow/compare)
+7. Add the ["review"](https://github.com/MarquezProject/marquez-airflow/labels/review) label to your pull request
+8. Work with project maintainers to get your change reviewed and merged into the `main` branch
+9. Delete your branch
+
+To ensure your pull request is accepted, follow these guidelines:
+
+* All changes should be accompanied by tests
+* Do your best to have a [well-formed commit message](https://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html) for your change
+* [Keep diffs small](https://graysonkoonce.com/stacked-pull-requests-keeping-github-diffs-small) and self-contained
+* If your change fixes a bug, please [link the issue](https://help.github.com/articles/closing-issues-using-keywords) in your pull request description
+
+# Resources
+
+* [How to Contribute to Open Source](https://opensource.guide/how-to-contribute)
+* [Using the Fork-and-Branch Git Workflow](https://blog.scottlowe.org/2015/01/27/using-fork-branch-git-workflow)
+* [Keep a Changelog](https://keepachangelog.com)