Fixing the tests

caravel/source_registry.py

@@ -14,7 +14,7 @@ def register_sources(cls, datasource_config):
                 cls.sources[source_class.type] = source_class
 
     @classmethod
-    def get_datasource(cls, datasource_type, datasource_id, session=None):
+    def get_datasource(cls, datasource_type, datasource_id, session):
         return (
             session.query(cls.sources[datasource_type])
             .filter_by(id=datasource_id)

caravel/views.py

@@ -243,7 +243,8 @@ def apply(self, query, func):  # noqa
         druid_datasources = []
         for perm in perms:
             match = re.search(r'\(id:(\d+)\)', perm)
-            druid_datasources.append(match.group(1))
+            if match:
+                druid_datasources.append(match.group(1))
         qry = query.filter(self.model.id.in_(druid_datasources))
         return qry
 
@@ -1132,7 +1133,7 @@ def explore_json(self, datasource_type, datasource_id):
         if not self.datasource_access(viz_obj.datasource):
             return Response(
                 json.dumps(
-                    {'error': "You don't have access to this datasource"}),
+                    {'error': _("You don't have access to this datasource")}),
                 status=404,
                 mimetype="application/json")
         return Response(

run_specific_test.sh

@@ -5,4 +5,5 @@ export CARAVEL_CONFIG=tests.caravel_test_config
 set -e
 caravel/bin/caravel version -v
 export SOLO_TEST=1
-nosetests tests.core_tests:CoreTests.test_endpoints_for_a_slice
+#nosetests tests.core_tests:CoreTests.test_slice_endpoint
+nosetests tests.druid_tests

tests/base_tests.py

@@ -81,6 +81,12 @@ def __init__(self, *args, **kwargs):
 
         utils.init(caravel)
 
+    def get_or_create(self, cls, criteria, session):
+        obj = session.query(cls).filter_by(**criteria).first()
+        if not obj:
+            obj = cls(**criteria)
+        return obj
+
     def login(self, username='admin', password='general'):
         resp = self.client.post(
             '/login/',
@@ -133,11 +139,6 @@ def get_access_requests(self, username, ds_type, ds_id):
     def logout(self):
         self.client.get('/logout/', follow_redirects=True)
 
-    def test_welcome(self):
-        self.login()
-        resp = self.client.get('/caravel/welcome')
-        assert 'Welcome' in resp.data.decode('utf-8')
-
     def setup_public_access_for_dashboard(self, table_name):
         public_role = appbuilder.sm.find_role('Public')
         perms = db.session.query(ab_models.PermissionView).all()

tests/core_tests.py

@@ -44,9 +44,14 @@ def setUp(self):
     def tearDown(self):
         pass
 
+    def test_welcome(self):
+        self.login()
+        resp = self.client.get('/caravel/welcome')
+        assert 'Welcome' in resp.data.decode('utf-8')
+
     def test_slice_endpoint(self):
         self.login(username='admin')
-        slc = self.get_slice("Energy Sankey", db.session)
+        slc = self.get_slice("Girls", db.session)
         resp = self.get_resp('/caravel/slice/{}/'.format(slc.id))
         assert 'Time Column' in resp
         assert 'List Roles' in resp

tests/druid_tests.py

@@ -14,7 +14,6 @@
 from caravel.models import DruidCluster, DruidDatasource
 
 from .base_tests import CaravelTestCase
-from flask_appbuilder.security.sqla import models as ab_models
 
 
 SEGMENT_METADATA = [{
@@ -118,25 +117,40 @@ def test_client(self, PyDruid):
             datasource_id))
         assert "[test_cluster].[test_datasource]" in resp.data.decode('utf-8')
 
-        resp = self.client.get(
-            '/caravel/explore/druid/{}/?viz_type=table&granularity=one+day&'
+        url = (
+            '/caravel/explore_json/druid/{}/?viz_type=table&granularity=one+day&'
             'druid_time_origin=&since=7+days+ago&until=now&row_limit=5000&'
             'include_search=false&metrics=count&groupby=name&flt_col_0=dim1&'
             'flt_op_0=in&flt_eq_0=&slice_id=&slice_name=&collapsed_fieldsets=&'
             'action=&datasource_name=test_datasource&datasource_id={}&'
-            'datasource_type=druid&previous_viz_type=table&json=true&'
+            'datasource_type=druid&previous_viz_type=table&'
             'force=true'.format(datasource_id, datasource_id))
-        assert "Canada" in resp.data.decode('utf-8')
+        resp = self.get_resp(url)
+        assert "Canada" in resp
 
     def test_druid_sync_from_config(self):
+        CLUSTER_NAME = 'new_druid'
         self.login()
-        cluster = DruidCluster(cluster_name="new_druid")
-        db.session.add(cluster)
+        cluster = self.get_or_create(
+            DruidCluster,
+            {'cluster_name': CLUSTER_NAME},
+            db.session)
+
+        db.session.merge(cluster)
+        db.session.commit()
+
+        ds = (
+            db.session.query(DruidDatasource)
+            .filter_by(datasource_name='test_click')
+            .first()
+        )
+        if ds:
+            db.session.delete(ds)
         db.session.commit()
 
         cfg = {
             "user": "admin",
-            "cluster": "new_druid",
+            "cluster": CLUSTER_NAME,
             "config": {
                 "name": "test_click",
                 "dimensions": ["affiliate_id", "campaign", "first_seen"],
@@ -152,30 +166,24 @@ def test_druid_sync_from_config(self):
                 }
             }
         }
-        resp = self.client.post('/caravel/sync_druid/', data=json.dumps(cfg))
-
-        druid_ds = db.session.query(DruidDatasource).filter_by(
-            datasource_name="test_click").first()
-        assert set([c.column_name for c in druid_ds.columns]) == set(
-            ["affiliate_id", "campaign", "first_seen"])
-        assert set([m.metric_name for m in druid_ds.metrics]) == set(
-            ["count", "sum"])
-        assert resp.status_code == 201
-
-        # datasource exists, not changes required
-        resp = self.client.post('/caravel/sync_druid/', data=json.dumps(cfg))
-        druid_ds = db.session.query(DruidDatasource).filter_by(
-            datasource_name="test_click").first()
-        assert set([c.column_name for c in druid_ds.columns]) == set(
-            ["affiliate_id", "campaign", "first_seen"])
-        assert set([m.metric_name for m in druid_ds.metrics]) == set(
-            ["count", "sum"])
-        assert resp.status_code == 201
+        def check():
+            resp = self.client.post('/caravel/sync_druid/', data=json.dumps(cfg))
+            druid_ds = db.session.query(DruidDatasource).filter_by(
+                datasource_name="test_click").first()
+            col_names = set([c.column_name for c in druid_ds.columns])
+            assert {"affiliate_id", "campaign", "first_seen"} == col_names
+            metric_names = {m.metric_name for m in druid_ds.metrics}
+            assert {"count", "sum"} == metric_names
+            assert resp.status_code == 201
+
+        check()
+        # checking twice to make sure a second sync yields the same results
+        check()
 
         # datasource exists, add new metrics and dimentions
         cfg = {
             "user": "admin",
-            "cluster": "new_druid",
+            "cluster": CLUSTER_NAME,
             "config": {
                 "name": "test_click",
                 "dimensions": ["affiliate_id", "second_seen"],
@@ -200,40 +208,40 @@ def test_druid_sync_from_config(self):
         assert resp.status_code == 201
 
     def test_filter_druid_datasource(self):
-        gamma_ds = DruidDatasource(
-            datasource_name="datasource_for_gamma",
-        )
-        db.session.add(gamma_ds)
-        no_gamma_ds = DruidDatasource(
-            datasource_name="datasource_not_for_gamma",
-        )
-        db.session.add(no_gamma_ds)
-        db.session.commit()
+        CLUSTER_NAME = 'new_druid'
+        cluster = self.get_or_create(
+            DruidCluster,
+            {'cluster_name': CLUSTER_NAME},
+            db.session)
+        db.session.merge(cluster)
+
+        gamma_ds = self.get_or_create(
+            DruidDatasource, {'datasource_name': 'datasource_for_gamma'},
+            db.session)
+        gamma_ds.cluster = cluster
+        db.session.merge(gamma_ds)
+
+        no_gamma_ds = self.get_or_create(
+            DruidDatasource, {'datasource_name': 'datasource_not_for_gamma'},
+            db.session)
+        no_gamma_ds.cluster = cluster
+        db.session.merge(no_gamma_ds)
+
         utils.merge_perm(sm, 'datasource_access', gamma_ds.perm)
         utils.merge_perm(sm, 'datasource_access', no_gamma_ds.perm)
+
         db.session.commit()
 
-        gamma_ds_permission_view = (
-            db.session.query(ab_models.PermissionView)
-            .join(ab_models.ViewMenu)
-            .filter(ab_models.ViewMenu.name == gamma_ds.perm)
-            .first()
-        )
-        sm.add_permission_role(sm.find_role('Gamma'), gamma_ds_permission_view)
+        perm = sm.find_permission_view_menu('datasource_access', gamma_ds.perm)
+        sm.add_permission_role(sm.find_role('Gamma'), perm)
+        db.session.commit()
 
         self.login(username='gamma')
         url = '/druiddatasourcemodelview/list/'
         resp = self.get_resp(url)
         assert 'datasource_for_gamma' in resp
         assert 'datasource_not_for_gamma' not in resp
 
-    def test_add_filter(self, username='admin'):
-        # navigate to energy_usage slice with "Electricity,heat" in filter values
-        data = (
-            "/caravel/explore/table/1/?viz_type=table&groupby=source&metric=count&flt_col_1=source&flt_op_1=in&flt_eq_1=%27Electricity%2Cheat%27"
-            "&userid=1&datasource_name=energy_usage&datasource_id=1&datasource_type=tablerdo_save=saveas")
-        assert "source" in self.get_resp(data)
-
 
 if __name__ == '__main__':
     unittest.main()