From 7d24bcaa6b02ee0c3fbddb0a605fdf89d0334166 Mon Sep 17 00:00:00 2001 From: Maxime Beauchemin Date: Tue, 4 Oct 2016 18:16:07 -0700 Subject: [PATCH] Fixing the tests --- caravel/source_registry.py | 2 +- caravel/views.py | 5 +- run_specific_test.sh | 3 +- tests/base_tests.py | 11 ++-- tests/core_tests.py | 7 ++- tests/druid_tests.py | 110 ++++++++++++++++++++----------------- 6 files changed, 77 insertions(+), 61 deletions(-) diff --git a/caravel/source_registry.py b/caravel/source_registry.py index e28dd3b7c6fbc..dc1a170d7c68e 100644 --- a/caravel/source_registry.py +++ b/caravel/source_registry.py @@ -14,7 +14,7 @@ def register_sources(cls, datasource_config): cls.sources[source_class.type] = source_class @classmethod - def get_datasource(cls, datasource_type, datasource_id, session=None): + def get_datasource(cls, datasource_type, datasource_id, session): return ( session.query(cls.sources[datasource_type]) .filter_by(id=datasource_id) diff --git a/caravel/views.py b/caravel/views.py index 7a6b858e5c391..117f5cef55353 100755 --- a/caravel/views.py +++ b/caravel/views.py @@ -243,7 +243,8 @@ def apply(self, query, func): # noqa druid_datasources = [] for perm in perms: match = re.search(r'\(id:(\d+)\)', perm) - druid_datasources.append(match.group(1)) + if match: + druid_datasources.append(match.group(1)) qry = query.filter(self.model.id.in_(druid_datasources)) return qry @@ -1132,7 +1133,7 @@ def explore_json(self, datasource_type, datasource_id): if not self.datasource_access(viz_obj.datasource): return Response( json.dumps( - {'error': "You don't have access to this datasource"}), + {'error': _("You don't have access to this datasource")}), status=404, mimetype="application/json") return Response( diff --git a/run_specific_test.sh b/run_specific_test.sh index 46c84390aabbe..52c6af371fdaa 100755 --- a/run_specific_test.sh +++ b/run_specific_test.sh @@ -5,4 +5,5 @@ export CARAVEL_CONFIG=tests.caravel_test_config set -e caravel/bin/caravel version -v export SOLO_TEST=1 -nosetests tests.core_tests:CoreTests.test_endpoints_for_a_slice +#nosetests tests.core_tests:CoreTests.test_slice_endpoint +nosetests tests.druid_tests diff --git a/tests/base_tests.py b/tests/base_tests.py index 5c8ae354a868e..1e22934e27bf4 100644 --- a/tests/base_tests.py +++ b/tests/base_tests.py @@ -81,6 +81,12 @@ def __init__(self, *args, **kwargs): utils.init(caravel) + def get_or_create(self, cls, criteria, session): + obj = session.query(cls).filter_by(**criteria).first() + if not obj: + obj = cls(**criteria) + return obj + def login(self, username='admin', password='general'): resp = self.client.post( '/login/', @@ -133,11 +139,6 @@ def get_access_requests(self, username, ds_type, ds_id): def logout(self): self.client.get('/logout/', follow_redirects=True) - def test_welcome(self): - self.login() - resp = self.client.get('/caravel/welcome') - assert 'Welcome' in resp.data.decode('utf-8') - def setup_public_access_for_dashboard(self, table_name): public_role = appbuilder.sm.find_role('Public') perms = db.session.query(ab_models.PermissionView).all() diff --git a/tests/core_tests.py b/tests/core_tests.py index 3bf4c39946d9c..01c502d82c2f3 100644 --- a/tests/core_tests.py +++ b/tests/core_tests.py @@ -44,9 +44,14 @@ def setUp(self): def tearDown(self): pass + def test_welcome(self): + self.login() + resp = self.client.get('/caravel/welcome') + assert 'Welcome' in resp.data.decode('utf-8') + def test_slice_endpoint(self): self.login(username='admin') - slc = self.get_slice("Energy Sankey", db.session) + slc = self.get_slice("Girls", db.session) resp = self.get_resp('/caravel/slice/{}/'.format(slc.id)) assert 'Time Column' in resp assert 'List Roles' in resp diff --git a/tests/druid_tests.py b/tests/druid_tests.py index 9d1857be13b1c..b1e70486e7936 100644 --- a/tests/druid_tests.py +++ b/tests/druid_tests.py @@ -14,7 +14,6 @@ from caravel.models import DruidCluster, DruidDatasource from .base_tests import CaravelTestCase -from flask_appbuilder.security.sqla import models as ab_models SEGMENT_METADATA = [{ @@ -118,25 +117,40 @@ def test_client(self, PyDruid): datasource_id)) assert "[test_cluster].[test_datasource]" in resp.data.decode('utf-8') - resp = self.client.get( - '/caravel/explore/druid/{}/?viz_type=table&granularity=one+day&' + url = ( + '/caravel/explore_json/druid/{}/?viz_type=table&granularity=one+day&' 'druid_time_origin=&since=7+days+ago&until=now&row_limit=5000&' 'include_search=false&metrics=count&groupby=name&flt_col_0=dim1&' 'flt_op_0=in&flt_eq_0=&slice_id=&slice_name=&collapsed_fieldsets=&' 'action=&datasource_name=test_datasource&datasource_id={}&' - 'datasource_type=druid&previous_viz_type=table&json=true&' + 'datasource_type=druid&previous_viz_type=table&' 'force=true'.format(datasource_id, datasource_id)) - assert "Canada" in resp.data.decode('utf-8') + resp = self.get_resp(url) + assert "Canada" in resp def test_druid_sync_from_config(self): + CLUSTER_NAME = 'new_druid' self.login() - cluster = DruidCluster(cluster_name="new_druid") - db.session.add(cluster) + cluster = self.get_or_create( + DruidCluster, + {'cluster_name': CLUSTER_NAME}, + db.session) + + db.session.merge(cluster) + db.session.commit() + + ds = ( + db.session.query(DruidDatasource) + .filter_by(datasource_name='test_click') + .first() + ) + if ds: + db.session.delete(ds) db.session.commit() cfg = { "user": "admin", - "cluster": "new_druid", + "cluster": CLUSTER_NAME, "config": { "name": "test_click", "dimensions": ["affiliate_id", "campaign", "first_seen"], @@ -152,30 +166,24 @@ def test_druid_sync_from_config(self): } } } - resp = self.client.post('/caravel/sync_druid/', data=json.dumps(cfg)) - - druid_ds = db.session.query(DruidDatasource).filter_by( - datasource_name="test_click").first() - assert set([c.column_name for c in druid_ds.columns]) == set( - ["affiliate_id", "campaign", "first_seen"]) - assert set([m.metric_name for m in druid_ds.metrics]) == set( - ["count", "sum"]) - assert resp.status_code == 201 - - # datasource exists, not changes required - resp = self.client.post('/caravel/sync_druid/', data=json.dumps(cfg)) - druid_ds = db.session.query(DruidDatasource).filter_by( - datasource_name="test_click").first() - assert set([c.column_name for c in druid_ds.columns]) == set( - ["affiliate_id", "campaign", "first_seen"]) - assert set([m.metric_name for m in druid_ds.metrics]) == set( - ["count", "sum"]) - assert resp.status_code == 201 + def check(): + resp = self.client.post('/caravel/sync_druid/', data=json.dumps(cfg)) + druid_ds = db.session.query(DruidDatasource).filter_by( + datasource_name="test_click").first() + col_names = set([c.column_name for c in druid_ds.columns]) + assert {"affiliate_id", "campaign", "first_seen"} == col_names + metric_names = {m.metric_name for m in druid_ds.metrics} + assert {"count", "sum"} == metric_names + assert resp.status_code == 201 + + check() + # checking twice to make sure a second sync yields the same results + check() # datasource exists, add new metrics and dimentions cfg = { "user": "admin", - "cluster": "new_druid", + "cluster": CLUSTER_NAME, "config": { "name": "test_click", "dimensions": ["affiliate_id", "second_seen"], @@ -200,26 +208,33 @@ def test_druid_sync_from_config(self): assert resp.status_code == 201 def test_filter_druid_datasource(self): - gamma_ds = DruidDatasource( - datasource_name="datasource_for_gamma", - ) - db.session.add(gamma_ds) - no_gamma_ds = DruidDatasource( - datasource_name="datasource_not_for_gamma", - ) - db.session.add(no_gamma_ds) - db.session.commit() + CLUSTER_NAME = 'new_druid' + cluster = self.get_or_create( + DruidCluster, + {'cluster_name': CLUSTER_NAME}, + db.session) + db.session.merge(cluster) + + gamma_ds = self.get_or_create( + DruidDatasource, {'datasource_name': 'datasource_for_gamma'}, + db.session) + gamma_ds.cluster = cluster + db.session.merge(gamma_ds) + + no_gamma_ds = self.get_or_create( + DruidDatasource, {'datasource_name': 'datasource_not_for_gamma'}, + db.session) + no_gamma_ds.cluster = cluster + db.session.merge(no_gamma_ds) + utils.merge_perm(sm, 'datasource_access', gamma_ds.perm) utils.merge_perm(sm, 'datasource_access', no_gamma_ds.perm) + db.session.commit() - gamma_ds_permission_view = ( - db.session.query(ab_models.PermissionView) - .join(ab_models.ViewMenu) - .filter(ab_models.ViewMenu.name == gamma_ds.perm) - .first() - ) - sm.add_permission_role(sm.find_role('Gamma'), gamma_ds_permission_view) + perm = sm.find_permission_view_menu('datasource_access', gamma_ds.perm) + sm.add_permission_role(sm.find_role('Gamma'), perm) + db.session.commit() self.login(username='gamma') url = '/druiddatasourcemodelview/list/' @@ -227,13 +242,6 @@ def test_filter_druid_datasource(self): assert 'datasource_for_gamma' in resp assert 'datasource_not_for_gamma' not in resp - def test_add_filter(self, username='admin'): - # navigate to energy_usage slice with "Electricity,heat" in filter values - data = ( - "/caravel/explore/table/1/?viz_type=table&groupby=source&metric=count&flt_col_1=source&flt_op_1=in&flt_eq_1=%27Electricity%2Cheat%27" - "&userid=1&datasource_name=energy_usage&datasource_id=1&datasource_type=tablerdo_save=saveas") - assert "source" in self.get_resp(data) - if __name__ == '__main__': unittest.main()