Use a key-value store model for sharing long queries (apache#1951)

* Add KeyValue model for storing id-value pairs
use it for storing shared queries

* Change string to text and added test

* Put getQueryLink in one place

* Changed migration down version

* Changes based on comments

* Update bcf3126872fc_add_keyvalue.py

superset/assets/javascripts/SqlLab/actions.js

@@ -298,3 +298,23 @@ export function removeTable(table) {
 export function refreshQueries(alteredQueries) {
   return { type: REFRESH_QUERIES, alteredQueries };
 }
+
+export function popStoredQuery(urlId) {
+  return function (dispatch) {
+    $.ajax({
+      type: 'GET',
+      url: `/kv/${urlId}`,
+      success: (data) => {
+        const newQuery = JSON.parse(data);
+        const queryEditorProps = {
+          title: newQuery.title ? newQuery.title : 'shared query',
+          dbId: newQuery.dbId ? parseInt(newQuery.dbId, 10) : null,
+          schema: newQuery.schema ? newQuery.schema : null,
+          autorun: newQuery.autorun ? newQuery.autorun : false,
+          sql: newQuery.sql ? newQuery.sql : 'SELECT ...',
+        };
+        dispatch(addQueryEditor(queryEditorProps));
+      },
+    });
+  };
+}

superset/assets/javascripts/SqlLab/components/CopyQueryTabUrl.jsx

@@ -1,6 +1,6 @@
 import React from 'react';
 import CopyToClipboard from '../../components/CopyToClipboard';
-import { getShortUrl } from '../../../utils/common';
+import { storeQuery } from '../../../utils/common';
 
 const propTypes = {
   queryEditor: React.PropTypes.object.isRequired,
@@ -9,16 +9,14 @@ const propTypes = {
 export default class CopyQueryTabUrl extends React.PureComponent {
   getUrl(callback) {
     const qe = this.props.queryEditor;
-    const params = [];
-    if (qe.dbId) params.push('dbid=' + qe.dbId);
-    if (qe.title) params.push('title=' + encodeURIComponent(qe.title));
-    if (qe.schema) params.push('schema=' + encodeURIComponent(qe.schema));
-    if (qe.autorun) params.push('autorun=' + qe.autorun);
-    if (qe.sql) params.push('sql=' + encodeURIComponent(qe.sql));
-
-    const queryString = params.join('&');
-    const queryLink = window.location.pathname + '?' + queryString;
-    getShortUrl(queryLink, callback);
+    const sharedQuery = {
+      dbId: qe.dbId,
+      title: qe.title,
+      schema: qe.schema,
+      autorun: qe.autorun,
+      sql: qe.sql,
+    };
+    storeQuery(sharedQuery, callback);
   }
 
   render() {

superset/assets/javascripts/SqlLab/components/QueryTable.jsx

@@ -10,7 +10,7 @@ import ModalTrigger from '../../components/ModalTrigger';
 import HighlightedSql from './HighlightedSql';
 import { STATE_BSSTYLE_MAP } from '../constants';
 import { fDuration } from '../../modules/dates';
-import { getLink } from '../../../utils/common';
+import { storeQuery } from '../../../utils/common';
 
 const propTypes = {
   columns: React.PropTypes.array,
@@ -38,10 +38,17 @@ class QueryTable extends React.PureComponent {
       activeQuery: null,
     };
   }
-  getQueryLink(dbId, sql) {
-    const params = ['dbid=' + dbId, 'sql=' + sql, 'title=Untitled Query'];
-    const link = getLink(this.state.cleanUri, params);
-    return encodeURI(link);
+  callback(url) {
+    window.open(url);
+  }
+  openQuery(dbId, schema, sql) {
+    const newQuery = {
+      dbId,
+      title: 'Untitled Query',
+      schema,
+      sql,
+    };
+    storeQuery(newQuery, this.callback);
   }
   hideVisualizeModal() {
     this.setState({ showVisualizeModal: false });
@@ -98,12 +105,12 @@ class QueryTable extends React.PureComponent {
       q.started = moment(q.startDttm).format('HH:mm:ss');
       q.querylink = (
         <div style={{ width: '100px' }}>
-          <a
-            href={this.getQueryLink(q.dbId, q.sql)}
-            className="btn btn-primary btn-xs"
+          <button
+            className="btn btn-link btn-xs"
+            onClick={this.openQuery.bind(this, q.dbId, q.schema, q.sql)}
           >
             <i className="fa fa-external-link" />Open in SQL Editor
-          </a>
+          </button>
         </div>
       );
       q.sql = (

superset/assets/javascripts/SqlLab/components/TabbedSqlEditors.jsx

@@ -4,9 +4,9 @@ import { connect } from 'react-redux';
 import { bindActionCreators } from 'redux';
 import * as Actions from '../actions';
 import SqlEditor from './SqlEditor';
-import { getParamFromQuery } from '../../../utils/common';
 import CopyQueryTabUrl from './CopyQueryTabUrl';
 import { areArraysShallowEqual } from '../../reduxUtils';
+import { getParamFromQuery } from '../../../utils/common';
 
 const propTypes = {
   actions: React.PropTypes.object.isRequired,
@@ -28,34 +28,39 @@ let queryCount = 1;
 class TabbedSqlEditors extends React.PureComponent {
   constructor(props) {
     super(props);
-    const uri = window.location.toString();
-    const search = window.location.search;
-    const cleanUri = search ? uri.substring(0, uri.indexOf('?')) : uri;
-    const query = search.substring(1);
+    const sqlLabUrl = '/superset/sqllab';
     this.state = {
-      uri,
-      cleanUri,
-      query,
+      sqlLabUrl,
       queriesArray: [],
       dataPreviewQueries: [],
       hideLeftBar: false,
     };
   }
-  componentWillMount() {
-    if (this.state.query) {
-      queryCount++;
-      const queryEditorProps = {
-        title: getParamFromQuery(this.state.query, 'title'),
-        dbId: parseInt(getParamFromQuery(this.state.query, 'dbid'), 10),
-        schema: getParamFromQuery(this.state.query, 'schema'),
-        autorun: getParamFromQuery(this.state.query, 'autorun'),
-        sql: getParamFromQuery(this.state.query, 'sql'),
-      };
-      this.props.actions.addQueryEditor(queryEditorProps);
-      // Clean the url in browser history
-      window.history.replaceState({}, document.title, this.state.cleanUri);
+  componentDidMount() {
+    const search = window.location.search;
+    if (search) {
+      const queryString = search.substring(1);
+      const urlId = getParamFromQuery(queryString, 'id');
+      if (urlId) {
+        this.props.actions.popStoredQuery(urlId);
+      } else {
+        const newQueryEditor = {
+          title: getParamFromQuery(queryString, 'title'),
+          dbId: getParamFromQuery(queryString, 'dbid'),
+          schema: getParamFromQuery(queryString, 'schema'),
+          autorun: getParamFromQuery(queryString, 'autorun'),
+          sql: getParamFromQuery(queryString, 'sql'),
+        };
+        this.props.actions.addQueryEditor(newQueryEditor);
+      }
+      this.popNewTab();
     }
   }
+  popNewTab() {
+    queryCount++;
+    // Clean the url in browser history
+    window.history.replaceState({}, document.title, this.state.sqlLabUrl);
+  }
   componentWillReceiveProps(nextProps) {
     const nextActiveQeId = nextProps.tabHistory[nextProps.tabHistory.length - 1];
     const queriesArray = [];

superset/assets/utils/common.js

@@ -39,8 +39,20 @@ export function getParamFromQuery(query, param) {
   return null;
 }
 
-export function getLink(baseUrl, params) {
-  return baseUrl + '?' + params.join('&');
+export function storeQuery(query, callback) {
+  $.ajax({
+    type: 'POST',
+    url: '/kv/store/',
+    async: false,
+    data: {
+      data: JSON.stringify(query),
+    },
+    success: (data) => {
+      const baseUrl = window.location.origin + window.location.pathname;
+      const url = `${baseUrl}?id=${JSON.parse(data).id}`;
+      callback(url);
+    },
+  });
 }
 
 export function getParamsFromUrl() {

superset/migrations/versions/bcf3126872fc_add_keyvalue.py

@@ -0,0 +1,30 @@
+"""Add keyvalue table
+
+Revision ID: bcf3126872fc
+Revises: f18570e03440
+Create Date: 2017-01-10 11:47:56.306938
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = 'bcf3126872fc'
+down_revision = 'f18570e03440'
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade():
+    ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('keyvalue',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('value', sa.Text(), nullable=False),
+    sa.PrimaryKeyConstraint('id')
+    )
+    ### end Alembic commands ###
+
+
+def downgrade():
+    ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('keyvalue')
+    ### end Alembic commands ###

superset/models.py

@@ -211,6 +211,15 @@ class Url(Model, AuditMixinNullable):
     url = Column(Text)
 
 
+class KeyValue(Model):
+
+    """Used for any type of key-value store"""
+
+    __tablename__ = 'keyvalue'
+    id = Column(Integer, primary_key=True)
+    value = Column(Text, nullable=False)
+
+
 class CssTemplate(Model, AuditMixinNullable):
 
     """CSS templates for dashboards"""

superset/views.py

@@ -18,7 +18,7 @@
 import sqlalchemy as sqla
 
 from flask import (
-    g, request, redirect, flash, Response, render_template, Markup)
+    g, request, redirect, flash, Response, render_template, Markup, url_for)
 from flask_appbuilder import ModelView, CompactCRUDMixin, BaseView, expose
 from flask_appbuilder.actions import action
 from flask_appbuilder.models.sqla.interface import SQLAInterface
@@ -1127,6 +1127,37 @@ def ping():
     return "OK"
 
 
+class KV(BaseSupersetView):
+
+    """Used for storing and retrieving key value pairs"""
+
+    @log_this
+    @expose("/store/", methods=['POST'])
+    def store(self):
+        try:
+            value = request.form.get('data')
+            obj = models.KeyValue(value=value)
+            db.session.add(obj)
+            db.session.commit()
+        except Exception as e:
+            return json_error_response(e)
+        return Response(
+            json.dumps({'id': obj.id}),
+            status=200)
+
+    @log_this
+    @expose("/<key_id>/", methods=['GET'])
+    def get_value(self, key_id):
+        kv = None
+        try:
+            kv = db.session.query(models.KeyValue).filter_by(id=key_id).one()
+        except Exception as e:
+            return json_error_response(e)
+        return Response(kv.value, status=200)
+
+appbuilder.add_view_no_menu(KV)
+
+
 class R(BaseSupersetView):
 
     """used for short urls"""
@@ -1788,7 +1819,7 @@ def tables(self, db_id, schema):
         )
         tables = [t for t in database.all_table_names(schema) if
                   self.datasource_access_by_name(database, t, schema=schema)]
-        views = [v for v in database.all_table_names(schema) if
+        views = [v for v in database.all_view_names(schema) if
                  self.datasource_access_by_name(database, v, schema=schema)]
         payload = {'tables': tables, 'views': views}
         return Response(

tests/core_tests.py

@@ -256,6 +256,32 @@ def test_shortner(self):
         resp = self.client.post('/r/shortner/', data=data)
         assert '/r/' in resp.data.decode('utf-8')
 
+    def test_kv(self):
+        self.logout()
+        self.login(username='admin')
+
+        try:
+            resp = self.client.post('/kv/store/', data=dict())
+        except Exception as e:
+            self.assertRaises(TypeError)
+
+        value = json.dumps({'data': 'this is a test'})
+        resp = self.client.post('/kv/store/', data=dict(data=value))
+        self.assertEqual(resp.status_code, 200)
+        kv = db.session.query(models.KeyValue).first()
+        kv_value = kv.value
+        self.assertEqual(json.loads(value), json.loads(kv_value))
+
+        resp = self.client.get('/kv/{}/'.format(kv.id))
+        self.assertEqual(resp.status_code, 200)
+        self.assertEqual(json.loads(value),
+            json.loads(resp.data.decode('utf-8')))
+
+        try:
+            resp = self.client.get('/kv/10001/')
+        except Exception as e:
+            self.assertRaises(TypeError)
+
     def test_save_dash(self, username='admin'):
         self.login(username=username)
         dash = db.session.query(models.Dashboard).filter_by(