[AIRFLOW-5180] Added static checks (yamllint) + auto-licences for yam…

…l file (apache#5790)
evinism · Aug 22, 2019 · e1cb8ce · e1cb8ce
1 parent 6344bb9
commit e1cb8ce
Show file tree

Hide file tree

Showing 19 changed files with 247 additions and 180 deletions.
diff --git a/.github/stale.yml b/.github/stale.yml
@@ -1,10 +1,27 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+---
 # Configuration for probot-stale - https://github.com/probot/stale
-
 # Number of days of inactivity before an Issue or Pull Request becomes stale
 daysUntilStale: 45
 
 # Number of days of inactivity before an Issue or Pull Request with the stale label is closed.
-# Set to false to disable. If disabled, issues still need to be closed manually, but will remain marked as stale.
+# Set to false to disable. If disabled, issues still need to be closed manually,
+# but will remain marked as stale.
 daysUntilClose: 7
 
 # Issues or Pull Requests with these labels will never be considered stale. Set to `[]` to disable

diff --git a/.hadolint.yaml b/.hadolint.yaml
@@ -1,4 +1,3 @@
-#
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -15,7 +14,7 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-#
+---
 ignored:
   - DL3006
   - DL3008

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -16,6 +16,9 @@
 # under the License.
 ---
 default_stages: [commit, push]
+default_language_version:
+  # force all unspecified python hooks to run python3
+  python: python3
 repos:
   - repo: https://github.com/Lucas-C/pre-commit-hooks
     rev: v1.1.6
@@ -77,6 +80,15 @@ repos:
           - "<!--||-->"
           - --license-filepath
           - license-templates/LICENSE.txt
+      - id: insert-license
+        name: Add licence for yaml files
+        exclude: ^\.github/.*$"|^airflow/_vendor/.*$
+        types: [yaml]
+        args:
+          - --comment-style
+          - "|#|"
+          - --license-filepath
+          - license-templates/LICENSE.txt
       - id: insert-license
         name: Add licence for all md files
         files: \.md$
@@ -109,6 +121,12 @@ repos:
       - id: check-xml
   - repo: local
     hooks:
+      - id: yamllint
+        name: Check yaml files with yamllint
+        entry: yamllint -c yamllint-config.yml
+        language: python
+        types: [yaml]
+        exclude: ^.*init_git_sync\.template\.yaml$|^.*airflow\.template\.yaml$
       - id: shellcheck
         name: Check Shell scripts syntax correctness
         language: docker_image

diff --git a/.readthedocs.yml b/.readthedocs.yml
@@ -1,4 +1,3 @@
-#
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -15,14 +14,14 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-
+---
 version: 2
 formats: []
 python:
     version: 3.7
     install:
-      - method: pip
-        path: .
-        extra_requirements:
-            - doc
+        - method: pip
+          path: .
+          extra_requirements:
+              - doc
     system_packages: true
diff --git a/.travis.yml b/.travis.yml
@@ -1,4 +1,3 @@
-#
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -16,6 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 #
+---
 dist: xenial
 language: python
 env:

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -688,6 +688,7 @@ In Linux you typically install them with `sudo apt install` on MacOS with `brew
 The current list of prerequisites:
 
 * xmllint: Linux - install via `sudo apt install xmllint`, MacOS - install via`brew install xmllint`
+* yamllint: install via `pip install yamllint`
 
 ## Pre-commit hooks installed
 
@@ -709,6 +710,7 @@ mixed-line-ending                Detects if mixed line ending is used (\r vs. \r
 mypy                             Run mypy
 pylint                           Run pylint
 shellcheck                       Check shell files with shellcheck
+yamllint                         Check yaml files with yamllint
 ```
 ## Using pre-commit hooks
 

diff --git a/scripts/ci/ci_before_install.sh b/scripts/ci/ci_before_install.sh
@@ -48,6 +48,6 @@ if [[ "${KUBERNETES_VERSION}" == "" ]]; then
     sudo service docker restart
 fi
 
-pip install pre-commit
+pip install pre-commit yamllint
 
 script_end
diff --git a/scripts/ci/docker-compose-mysql.yml b/scripts/ci/docker-compose-mysql.yml
@@ -1,19 +1,20 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
 #
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
+#   http://www.apache.org/licenses/LICENSE-2.0
 #
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+---
 version: "2.2"
 services:
   airflow-testing:

diff --git a/scripts/ci/docker-compose-postgres.yml b/scripts/ci/docker-compose-postgres.yml
@@ -1,19 +1,20 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
 #
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
+#   http://www.apache.org/licenses/LICENSE-2.0
 #
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+---
 version: "2.2"
 services:
   airflow-testing:

diff --git a/scripts/ci/docker-compose-sqlite.yml b/scripts/ci/docker-compose-sqlite.yml
@@ -1,19 +1,20 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
 #
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
+#   http://www.apache.org/licenses/LICENSE-2.0
 #
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+---
 version: "2.2"
 services:
   airflow-testing:

diff --git a/scripts/ci/docker-compose.yml b/scripts/ci/docker-compose.yml
@@ -1,19 +1,20 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
 #
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
+#   http://www.apache.org/licenses/LICENSE-2.0
 #
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+---
 version: "2.2"
 services:
 
@@ -61,7 +62,7 @@ services:
   airflow-testing:
     image: ${AIRFLOW_CONTAINER_DOCKER_IMAGE}
     init: true
-    entrypoint: [ "/bin/bash", "-c" ]
+    entrypoint: ["/bin/bash", "-c"]
     environment:
       - USER=root
       - ADDITIONAL_PATH=~/.local/bin

diff --git a/scripts/ci/kubernetes/kube/postgres.yaml b/scripts/ci/kubernetes/kube/postgres.yaml
@@ -1,20 +1,20 @@
-#  Licensed to the Apache Software Foundation (ASF) under one   *
-#  or more contributor license agreements.  See the NOTICE file *
-#  distributed with this work for additional information        *
-#  regarding copyright ownership.  The ASF licenses this file   *
-#  to you under the Apache License, Version 2.0 (the            *
-#  "License"); you may not use this file except in compliance   *
-#  with the License.  You may obtain a copy of the License at   *
-#                                                               *
-#    http://www.apache.org/licenses/LICENSE-2.0                 *
-#                                                               *
-#  Unless required by applicable law or agreed to in writing,   *
-#  software distributed under the License is distributed on an  *
-#  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY       *
-#  KIND, either express or implied.  See the License for the    *
-#  specific language governing permissions and limitations      *
-#  under the License.                                           *
-
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+---
 kind: Deployment
 apiVersion: extensions/v1beta1
 metadata:
@@ -48,25 +48,31 @@ spec:
             - name: PGDATA
               value: /var/lib/postgresql/data/pgdata
             - name: POD_IP
-              valueFrom: { fieldRef: { fieldPath: status.podIP } }
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
           livenessProbe:
             initialDelaySeconds: 60
             timeoutSeconds: 5
             failureThreshold: 5
             exec:
               command:
-              - /bin/sh
-              - -c
-              - exec pg_isready --host $POD_IP ||  if [[ $(psql -qtAc --host $POD_IP 'SELECT pg_is_in_recovery') != "f" ]]; then  exit 0 else; exit 1; fi
+                - /bin/sh
+                - -c
+                - >
+                  exec pg_isready --host $POD_IP ||
+                  if [[ $(psql -qtAc --host $POD_IP 'SELECT pg_is_in_recovery') != "f" ]];
+                    then  exit 0 else;
+                  exit 1; fi
           readinessProbe:
             initialDelaySeconds: 5
             timeoutSeconds: 5
             periodSeconds: 5
             exec:
               command:
-              - /bin/sh
-              - -c
-              - exec pg_isready --host $POD_IP
+                - /bin/sh
+                - -c
+                - exec pg_isready --host $POD_IP
           resources:
             requests:
               memory: .5Gi