-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #3 from DineshGanesan05/restore-commit-f1e2a15
UpdatedCodeWeatherReport
- Loading branch information
Showing
9,099 changed files
with
1,588,097 additions
and
0 deletions.
The diff you're trying to view is too large. We only load the first 3000 changed files.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,2 @@ | ||
project: | ||
name: dinesh |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
# Add dag files to exempt from parse test below. ex: dags/<test-file> |
141 changes: 141 additions & 0 deletions
141
airflow/WeatherAPI/.astro/test_dag_integrity_default.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,141 @@ | ||
"""Test the validity of all DAGs. **USED BY DEV PARSE COMMAND DO NOT EDIT**""" | ||
|
||
from contextlib import contextmanager | ||
import logging | ||
import os | ||
|
||
import pytest | ||
|
||
from airflow.models import DagBag, Variable, Connection | ||
from airflow.hooks.base import BaseHook | ||
from airflow.utils.db import initdb | ||
|
||
# init airflow database | ||
initdb() | ||
|
||
# The following code patches errors caused by missing OS Variables, Airflow Connections, and Airflow Variables | ||
|
||
|
||
# =========== MONKEYPATCH BaseHook.get_connection() =========== | ||
def basehook_get_connection_monkeypatch(key: str, *args, **kwargs): | ||
print( | ||
f"Attempted to fetch connection during parse returning an empty Connection object for {key}" | ||
) | ||
return Connection(key) | ||
|
||
|
||
BaseHook.get_connection = basehook_get_connection_monkeypatch | ||
# # =========== /MONKEYPATCH BASEHOOK.GET_CONNECTION() =========== | ||
|
||
|
||
# =========== MONKEYPATCH OS.GETENV() =========== | ||
def os_getenv_monkeypatch(key: str, *args, **kwargs): | ||
default = None | ||
if args: | ||
default = args[0] # os.getenv should get at most 1 arg after the key | ||
if kwargs: | ||
default = kwargs.get( | ||
"default", None | ||
) # and sometimes kwarg if people are using the sig | ||
|
||
env_value = os.environ.get(key, None) | ||
|
||
if env_value: | ||
return env_value # if the env_value is set, return it | ||
if ( | ||
key == "JENKINS_HOME" and default is None | ||
): # fix https://github.com/astronomer/astro-cli/issues/601 | ||
return None | ||
if default: | ||
return default # otherwise return whatever default has been passed | ||
return f"MOCKED_{key.upper()}_VALUE" # if absolutely nothing has been passed - return the mocked value | ||
|
||
|
||
os.getenv = os_getenv_monkeypatch | ||
# # =========== /MONKEYPATCH OS.GETENV() =========== | ||
|
||
# =========== MONKEYPATCH VARIABLE.GET() =========== | ||
|
||
|
||
class magic_dict(dict): | ||
def __init__(self, *args, **kwargs): | ||
self.update(*args, **kwargs) | ||
|
||
def __getitem__(self, key): | ||
return {}.get(key, "MOCKED_KEY_VALUE") | ||
|
||
|
||
_no_default = object() # allow falsey defaults | ||
|
||
|
||
def variable_get_monkeypatch(key: str, default_var=_no_default, deserialize_json=False): | ||
print( | ||
f"Attempted to get Variable value during parse, returning a mocked value for {key}" | ||
) | ||
|
||
if default_var is not _no_default: | ||
return default_var | ||
if deserialize_json: | ||
return magic_dict() | ||
return "NON_DEFAULT_MOCKED_VARIABLE_VALUE" | ||
|
||
|
||
Variable.get = variable_get_monkeypatch | ||
# # =========== /MONKEYPATCH VARIABLE.GET() =========== | ||
|
||
|
||
@contextmanager | ||
def suppress_logging(namespace): | ||
""" | ||
Suppress logging within a specific namespace to keep tests "clean" during build | ||
""" | ||
logger = logging.getLogger(namespace) | ||
old_value = logger.disabled | ||
logger.disabled = True | ||
try: | ||
yield | ||
finally: | ||
logger.disabled = old_value | ||
|
||
|
||
def get_import_errors(): | ||
""" | ||
Generate a tuple for import errors in the dag bag, and include DAGs without errors. | ||
""" | ||
with suppress_logging("airflow"): | ||
dag_bag = DagBag(include_examples=False) | ||
|
||
def strip_path_prefix(path): | ||
return os.path.relpath(path, os.environ.get("AIRFLOW_HOME")) | ||
|
||
# Initialize an empty list to store the tuples | ||
result = [] | ||
|
||
# Iterate over the items in import_errors | ||
for k, v in dag_bag.import_errors.items(): | ||
result.append((strip_path_prefix(k), v.strip())) | ||
|
||
# Check if there are DAGs without errors | ||
for file_path in dag_bag.dags: | ||
# Check if the file_path is not in import_errors, meaning no errors | ||
if file_path not in dag_bag.import_errors: | ||
result.append((strip_path_prefix(file_path), "No import errors")) | ||
|
||
return result | ||
|
||
|
||
@pytest.mark.parametrize( | ||
"rel_path, rv", get_import_errors(), ids=[x[0] for x in get_import_errors()] | ||
) | ||
def test_file_imports(rel_path, rv): | ||
"""Test for import errors on a file""" | ||
if os.path.exists(".astro/dag_integrity_exceptions.txt"): | ||
with open(".astro/dag_integrity_exceptions.txt", "r") as f: | ||
exceptions = f.readlines() | ||
print(f"Exceptions: {exceptions}") | ||
if (rv != "No import errors") and rel_path not in exceptions: | ||
# If rv is not "No import errors," consider it a failed test | ||
raise Exception(f"{rel_path} failed to import with message \n {rv}") | ||
else: | ||
# If rv is "No import errors," consider it a passed test | ||
print(f"{rel_path} passed the import test") |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,8 @@ | ||
astro | ||
.git | ||
.env | ||
airflow_settings.yaml | ||
logs/ | ||
.venv | ||
airflow.db | ||
airflow.cfg |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,11 @@ | ||
.git | ||
.env | ||
.DS_Store | ||
airflow_settings.yaml | ||
__pycache__/ | ||
astro | ||
.venv | ||
airflow-webserver.pid | ||
webserver_config.py | ||
airflow.cfg | ||
airflow.db |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,5 @@ | ||
{ | ||
"recommendations": [ | ||
"kiranshah.chatgpt-helper" | ||
] | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,3 @@ | ||
{ | ||
"CodeGPT.apiKey": "CodeGPT Plus Beta" | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
FROM quay.io/astronomer/astro-runtime:12.5.0 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,48 @@ | ||
Overview | ||
======== | ||
|
||
Welcome to Astronomer! This project was generated after you ran 'astro dev init' using the Astronomer CLI. This readme describes the contents of the project, as well as how to run Apache Airflow on your local machine. | ||
|
||
Project Contents | ||
================ | ||
|
||
Your Astro project contains the following files and folders: | ||
|
||
- dags: This folder contains the Python files for your Airflow DAGs. By default, this directory includes one example DAG: | ||
- `example_astronauts`: This DAG shows a simple ETL pipeline example that queries the list of astronauts currently in space from the Open Notify API and prints a statement for each astronaut. The DAG uses the TaskFlow API to define tasks in Python, and dynamic task mapping to dynamically print a statement for each astronaut. For more on how this DAG works, see our [Getting started tutorial](https://www.astronomer.io/docs/learn/get-started-with-airflow). | ||
- Dockerfile: This file contains a versioned Astro Runtime Docker image that provides a differentiated Airflow experience. If you want to execute other commands or overrides at runtime, specify them here. | ||
- include: This folder contains any additional files that you want to include as part of your project. It is empty by default. | ||
- packages.txt: Install OS-level packages needed for your project by adding them to this file. It is empty by default. | ||
- requirements.txt: Install Python packages needed for your project by adding them to this file. It is empty by default. | ||
- plugins: Add custom or community plugins for your project to this file. It is empty by default. | ||
- airflow_settings.yaml: Use this local-only file to specify Airflow Connections, Variables, and Pools instead of entering them in the Airflow UI as you develop DAGs in this project. | ||
|
||
Deploy Your Project Locally | ||
=========================== | ||
|
||
1. Start Airflow on your local machine by running 'astro dev start'. | ||
|
||
This command will spin up 4 Docker containers on your machine, each for a different Airflow component: | ||
|
||
- Postgres: Airflow's Metadata Database | ||
- Webserver: The Airflow component responsible for rendering the Airflow UI | ||
- Scheduler: The Airflow component responsible for monitoring and triggering tasks | ||
- Triggerer: The Airflow component responsible for triggering deferred tasks | ||
|
||
2. Verify that all 4 Docker containers were created by running 'docker ps'. | ||
|
||
Note: Running 'astro dev start' will start your project with the Airflow Webserver exposed at port 8080 and Postgres exposed at port 5432. If you already have either of those ports allocated, you can either [stop your existing Docker containers or change the port](https://www.astronomer.io/docs/astro/cli/troubleshoot-locally#ports-are-not-available-for-my-local-airflow-webserver). | ||
|
||
3. Access the Airflow UI for your local Airflow project. To do so, go to http://localhost:8080/ and log in with 'admin' for both your Username and Password. | ||
|
||
You should also be able to access your Postgres Database at 'localhost:5432/postgres'. | ||
|
||
Deploy Your Project to Astronomer | ||
================================= | ||
|
||
If you have an Astronomer account, pushing code to a Deployment on Astronomer is simple. For deploying instructions, refer to Astronomer documentation: https://www.astronomer.io/docs/astro/deploy-code/ | ||
|
||
Contact | ||
======= | ||
|
||
The Astronomer CLI is maintained with love by the Astronomer team. To report a bug or suggest a change, reach out to our support. |
Empty file.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,16 @@ | ||
version: '3' | ||
services: | ||
postgres: | ||
image: postgres:13 | ||
container_name: postgres_db | ||
environment: | ||
POSTGRES_USER: postgres | ||
POSTGRES_PASSWORD: postgres | ||
POSTGRES_DB: postgres | ||
ports: | ||
- "5433:5432" | ||
volumes: | ||
- postgres_data:/var/lib/postgresql/data | ||
|
||
volumes: | ||
postgres_data: |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,104 @@ | ||
from airflow import DAG | ||
from airflow.providers.http.hooks.http import HttpHook | ||
from airflow.providers.postgres.hooks.postgres import PostgresHook | ||
from airflow.decorators import task | ||
from airflow.utils.dates import days_ago | ||
|
||
## Latitude and longitude | ||
LATITUDE = '51.5074' | ||
LONGITUDE = '-0.1278' | ||
POSTGRES_CONN_ID = 'default_conn' | ||
API_CON_ID = 'open_meteo_api' | ||
|
||
default_args = { | ||
'owner': 'AIRFLOW', | ||
'start_date': days_ago(1) | ||
} | ||
|
||
## DAG | ||
with DAG(dag_id='weather_etl_pipeline', | ||
default_args=default_args, | ||
schedule_interval='@daily', | ||
catchup=False) as dag: | ||
|
||
@task() | ||
def extract_weather_data(): | ||
"""Extract weather data""" | ||
|
||
## Use HTTP hook | ||
http_hook = HttpHook(http_conn_id=API_CON_ID, method='GET') | ||
## End Point | ||
endpoint = f'/v1/forecast?latitude={LATITUDE}&longitude={LONGITUDE}¤t_weather=true' | ||
|
||
## Make the request via HTTPHOOK | ||
response = http_hook.run(endpoint) | ||
|
||
if response.status_code == 200: | ||
# Process the response data here | ||
return response.json() # Assuming you want to return the JSON data | ||
else: | ||
raise Exception(f"Failed to fetch data: {response.status_code}") | ||
|
||
@task() | ||
def transform_weather_data(weather_data): | ||
"""Transform the extracted weather data.""" | ||
current_weather = weather_data['current_weather'] | ||
transformed_data = { | ||
'latitude': LATITUDE, | ||
'longitude': LONGITUDE, | ||
'temperature': current_weather['temperature'], | ||
'windspeed': current_weather['windspeed'], | ||
'winddirection': current_weather['winddirection'], | ||
'weathercode': current_weather['weathercode'] | ||
} | ||
return transformed_data | ||
@task() | ||
def load_weather_data(transformed_data): | ||
"""Load transformed data into PostgreSQL.""" | ||
pg_hook = PostgresHook(postgres_conn_id=POSTGRES_CONN_ID) | ||
conn = pg_hook.get_conn() | ||
cursor = conn.cursor() | ||
|
||
# Create table if it doesn't exist | ||
cursor.execute(""" | ||
CREATE TABLE IF NOT EXISTS weather_data ( | ||
latitude FLOAT, | ||
longitude FLOAT, | ||
temperature FLOAT, | ||
windspeed FLOAT, | ||
winddirection FLOAT, | ||
weathercode INT, | ||
timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP | ||
); | ||
""") | ||
|
||
# Insert transformed data into the table | ||
cursor.execute(""" | ||
INSERT INTO weather_data (latitude, longitude, temperature, windspeed, winddirection, weathercode) | ||
VALUES (%s, %s, %s, %s, %s, %s) | ||
""", ( | ||
transformed_data['latitude'], | ||
transformed_data['longitude'], | ||
transformed_data['temperature'], | ||
transformed_data['windspeed'], | ||
transformed_data['winddirection'], | ||
transformed_data['weathercode'] | ||
)) | ||
|
||
conn.commit() | ||
cursor.close() | ||
|
||
##DAG workflow ETL Pipeline | ||
weather_date = extract_weather_data() | ||
transformed_data = transform_weather_data(weather_date) | ||
load_weather_data = load_weather_data(transformed_data) | ||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
2024-12-06 |
Empty file.
Oops, something went wrong.