Merge pull request fhirbase#138 from KainosSoftwareLtd/new-param-to-i…

…mprove-calculating-count

introduce totalMethod param to get fast but approx count

README.md

@@ -213,6 +213,19 @@ SELECT fhir_valueset_expand('{"id": "issue-types", "filter": "err"}');
 
 SELECT fhir_conformance('{"default": "values"}');
 -- return simple Conformance resource, based on created stores
+
+---
+
+-- use different methods to calculate total elements to improve performance: no _totalMethod or _totalMethod=exact uses standard approach
+SELECT fhir_search('{"resourceType": "Patient", "queryString": "name=smith"}');
+SELECT fhir_search('{"resourceType": "Patient", "queryString": "name=smith&_totalMethod=exact"}');
+
+-- _totalMethod=extimated - faster but 'total' is estimated.
+SELECT fhir_search('{"resourceType": "Patient", "queryString": "name=smith&_totalMethod=estimated"}');
+
+-- _totalMethod=no - fastest but no 'total' is returned.
+SELECT fhir_search('{"resourceType": "Patient", "queryString": "name=smith&_totalMethod=no"}');
+
 ```
 
 ## Contributing

src/fhir/query_string.coffee

@@ -122,7 +122,10 @@ specials =
     query.sort ||= []
     query.sort.push ['$param', key, '']
     query
-  format:  (query, left, right)->
+  totalMethod: (query, left, right)->
+    query.total_method = right;
+    query
+  format: (query, left, right)->
     query
   lastUpdated: (query, left, right)->
     unless query.lastUpdateds

src/fhir/search.litcoffee

@@ -33,7 +33,6 @@ so the code split as much as possible  into small modules
     sql = require('../honey')
     utils = require('../core/utils')
 
-
 For every search type we have dedicated module,
 with indexing and building search expression implementation.
 
@@ -104,27 +103,36 @@ Then we are returning  resulting Bundle.
 
       resource_rows = utils.exec(plv8, honey)
       resources = resource_rows.map((x)-> compat.parse(plv8, x.resource))
-      count = utils.exec(plv8, countize_query(honey))[0].count
+
+      should_include_total = (expr.total_method != "no")
+
+      if should_include_total
+        count = get_count(plv8, honey, expr)
 
       base_url = "#{query.resourceType}/#{query.queryString}"
 
       if expr.summary or expr.elements
         resources = mask_resources(plv8, expr, idx_db, resources)
 
-      if expr.include && count > 0
+      if expr.include && (count > 0 || !should_include_total)
         includes = search_include.load_includes(plv8, expr.include, resources)
         resources = resources.concat(includes)
 
-      if expr.revinclude && count > 0
+      if expr.revinclude && (count > 0 || !should_include_total)
         includes = search_include.load_revincludes(plv8, expr.revinclude, resources)
         resources = resources.concat(includes)
 
+      if should_include_total
+        resourceType: 'Bundle'
+        type: 'searchset'
+        total: count
+        link: helpers.search_links(query, expr, count)
+        entry: resources.map(to_entry)
+      else
+        resourceType: 'Bundle'
+        type: 'searchset'
+        entry: resources.map(to_entry)
 
-      resourceType: 'Bundle'
-      type: 'searchset'
-      total: count
-      link: helpers.search_links(query, expr, count)
-      entry: resources.map(to_entry)
 
 Helper function to convert resource into entry bundle:
 TODO: add links
@@ -344,6 +352,27 @@ we just strip limit, offset, order and rewrite select clause:
       q.select = [':count(*) as count']
       q
 
+    get_count = (plv8, honey, query_obj) ->
+      if !query_obj.total_method || query_obj.total_method is "exact"
+        query_obj.total_method = "exact"
+
+        utils.exec(plv8, countize_query(honey))[0].count
+      else if query_obj.total_method is "estimated"
+        sql_query= sql(honey)
+        query = sql_query[0].replace /\$(\d+)/g, (match, number) ->
+            if typeof sql_query[number] is "string"
+              return '\''+sql_query[number]+'\''
+            else if typeof sql_query[number] isnt 'undefined'
+              return sql_query[number]
+            else
+              return match
+        query = query.replace /LIMIT \d+/, ""
+        query = query.replace /\'/g, '\'\''
+        query = "SELECT count_estimate('#{query}');"
+        tmp = plv8.execute(query)
+        tmp[0].count_estimate
+      else
+        throw new Error("Invalid value of totalMethod. only 'exact', 'estimated' and 'no' allowed.")
 
 We cache FHIR meta-data index per connection using plv8 object:
 

test/fhir/query_string_spec.edn

@@ -26,6 +26,15 @@
  [["Patient" "_count=11"]
   {:query "Patient" :count 11}]
 
+ [["Patient" "_totalMethod=exact"]
+  {:query "Patient" :total_method "exact"}]
+
+ [["Patient" "_totalMethod=no"]
+  {:query "Patient" :total_method "no"}]
+
+ [["Patient" "_totalMethod=estimated"]
+  {:query "Patient" :total_method "estimated"}]
+
  [["Patient" "_page=12"]
   {:query "Patient" :page 12}]
 
@@ -34,7 +43,7 @@
    :joins [(chained
             (param {:resourceType "Patient" :name "careprovider" :join "Practitioner"} {:value "$id"})
             (param {:resourceType "Practitioner" :name "name"} {:value "igor"}))]}]
- 
+
  [["Patient" "careprovider:Practitioner.organization:Organization.name=hl7"]
   {:query "Patient"
    :joins [(chained

test/fhir/search/pagination_search.yaml

@@ -63,3 +63,23 @@ queries:
         result: "/Encounter?patient=Patient/nicola&_sort=patient&_page=0&_count=1"
       - path: ['link', 3, 'url']
         result: "/Encounter?patient=Patient/nicola&_sort=patient&_page=3&_count=1"
+
+  - query: {resourceType: 'Encounter', queryString: 'patient=Patient/nicola&_sort=patient&_page=1&_count=1&_totalMethod=exact'}
+    total: 3
+    probes:
+      - path: ['entry', 'length']
+        result: 1
+      - path: ['link', 0, 'url']
+        result: "/Encounter?patient=Patient/nicola&_sort=patient&_page=1&_count=1&_totalMethod=exact"
+      - path: ['link', 1, 'url']
+        result: "/Encounter?patient=Patient/nicola&_sort=patient&_page=2&_count=1&_totalMethod=exact"
+      - path: ['link', 2, 'url']
+        result: "/Encounter?patient=Patient/nicola&_sort=patient&_page=0&_count=1&_totalMethod=exact"
+      - path: ['link', 3, 'url']
+        result: "/Encounter?patient=Patient/nicola&_sort=patient&_page=3&_count=1&_totalMethod=exact"
+
+  - query: {resourceType: 'Encounter', queryString: 'patient=Patient/nicola&_sort=patient&_page=1&_count=1&_totalMethod=no'}
+    total: _undefined
+    probes:
+      - path: ['link']
+        result: '_undefined'

test/fhir/search_spec.coffee

@@ -63,9 +63,16 @@ fs.readdirSync("#{__dirname}/search").filter(match(FILTER)).forEach (yml)->
         plv8.execute "SET enable_seqscan = ON;" if (q.indexed or q.indexed_order)
 
         if q.total || q.total == 0
-          assert.equal(res.total, q.total)
-
-        (q.probes || []).forEach (probe)-> assert.equal(get_in(res, probe.path), probe.result)
+          if q.total == "_undefined"
+            assert.equal(res.total, undefined)
+          else
+            assert.equal(res.total, q.total)
+
+        (q.probes || []).forEach (probe)->
+          if probe.result == "_undefined"
+            assert.equal(get_in(res, probe.path), undefined)
+          else
+            assert.equal(get_in(res, probe.path), probe.result)
 
         # console.log(explain)
 

utils/generate_schema.coffee

@@ -75,6 +75,22 @@ CREATE AGGREGATE FIRST (
   basetype = anyelement,
   stype    = anyelement
 );
+
+-- Create function to estimate rows of a query. It's used to improve performance of search function.
+CREATE OR REPLACE FUNCTION count_estimate(query text) RETURNS INTEGER AS
+$func$
+DECLARE
+    rec   record;
+    ROWS  INTEGER;
+BEGIN
+    FOR rec IN EXECUTE 'EXPLAIN ' || query LOOP
+        ROWS := SUBSTRING(rec."QUERY PLAN" FROM ' rows=([[:digit:]]+)');
+        EXIT WHEN ROWS IS NOT NULL;
+    END LOOP;
+
+    RETURN ROWS;
+END
+$func$ LANGUAGE plpgsql;
 """
 
 is_first = true