Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[GITHUB-578] Exclude MongoDB-Redshift from valid Fastsync pairs. #2

Merged
merged 1 commit into from
Nov 27, 2020
Merged

[GITHUB-578] Exclude MongoDB-Redshift from valid Fastsync pairs. #2

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 9 additions & 8 deletions pipelinewise/cli/pipelinewise.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,13 @@
from . import commands
from .commands import TapParams, TargetParams, TransformParams
from .config import Config
from .alert_sender import AlertSender
from .alert_handlers.base_alert_handler import BaseAlertHandler

FASTSYNC_PAIRS = {
'tap-mysql': {'target-snowflake', 'target-redshift', 'target-postgres'},
'tap-postgres': {'target-snowflake', 'target-redshift', 'target-postgres'},
'tap-s3-csv': {'target-snowflake', 'target-redshift', 'target-postgres'},
'tap-mongodb': {'target-snowflake', 'target-postgres'}
}

# pylint: disable=too-many-lines,too-many-instance-attributes,too-many-public-methods
class PipelineWise:
Expand Down Expand Up @@ -132,8 +136,7 @@ def create_filtered_tap_properties(self, target_type, tap_type, tap_properties,
# Get filter conditions with default values from input dictionary
# Nothing selected by default
f_selected = filters.get('selected', None)
f_target_type = filters.get('target_type', None)
f_tap_type = filters.get('tap_type', None)
f_tap_target_pairs = filters.get('tap_target_pairs', None)
f_replication_method = filters.get('replication_method', None)
f_initial_sync_required = filters.get('initial_sync_required', None)

Expand Down Expand Up @@ -199,8 +202,7 @@ def create_filtered_tap_properties(self, target_type, tap_type, tap_properties,
# pylint: disable=too-many-boolean-expressions,bad-continuation
if (
(f_selected is None or selected == f_selected) and
(f_target_type is None or target_type in f_target_type) and
(f_tap_type is None or tap_type in f_tap_type) and
(f_tap_target_pairs is None or target_type in f_tap_target_pairs.get(tap_type, {})) and
(f_replication_method is None or replication_method in f_replication_method) and
(f_initial_sync_required is None or initial_sync_required == f_initial_sync_required)
):
Expand Down Expand Up @@ -976,8 +978,7 @@ def run_tap(self):
tap_properties,
tap_state, {
'selected': True,
'target_type': ['target-snowflake', 'target-redshift', 'target-postgres'],
'tap_type': ['tap-mysql', 'tap-postgres', 'tap-s3-csv', 'tap-mongodb'],
'tap_target_pairs': FASTSYNC_PAIRS,
'initial_sync_required': True
},
create_fallback=True)
Expand Down
37 changes: 35 additions & 2 deletions tests/units/cli/test_cli.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -184,8 +184,7 @@ def test_create_filtered_tap_props(self):
os.path.dirname(__file__)),
filters={
'selected': True,
'target_type': ['target-snowflake'],
'tap_type': ['tap-mysql', 'tap-postgres'],
'tap_target_pairs': {'tap-mysql': {'target-snowflake'}, 'tap-postgres': {'target-snowflake'}},
'initial_sync_required': True
},
create_fallback=True)
Expand All @@ -202,6 +201,40 @@ def test_create_filtered_tap_props(self):
assert fastsync_stream_ids == ['db_test_mysql-table_one', 'db_test_mysql-table_two']
assert singer_stream_ids == ['db_test_mysql-table_one', 'db_test_mysql-table_two']

# pylint: disable=bad-continuation
def test_create_filtered_tap_props_fallback(self):
"""Test creating fastsync and singer specific properties file"""
(
tap_properties_fastsync,
fastsync_stream_ids,
tap_properties_singer,
singer_stream_ids
) = self.pipelinewise.create_filtered_tap_properties(
target_type='target-snowflake',
tap_type='tap-mysql',
tap_properties='{}/resources/sample_json_config/target_one/tap_one/properties.json'.format(
os.path.dirname(__file__)),
tap_state='{}/resources/sample_json_config/target_one/tap_one/state.json'.format(
os.path.dirname(__file__)),
filters={
'selected': True,
'tap_target_pairs': {'tap-postgres': {'target-snowflake'}},
'initial_sync_required': True
},
create_fallback=True)

# Fastsync and singer properties should be created
assert os.path.isfile(tap_properties_fastsync)
assert os.path.isfile(tap_properties_singer)

# Delete generated properties file
os.remove(tap_properties_fastsync)
os.remove(tap_properties_singer)

# Fastsync and singer properties should be created
assert fastsync_stream_ids == []
assert singer_stream_ids == ['db_test_mysql-table_one', 'db_test_mysql-table_two']

def test_merge_empty_catalog(self):
"""Merging two empty singer schemas should be another empty"""
# TODO: Check if pipelinewise.merge_schemas is required at all or not
Expand Down