Firming up support for passing query_str

This patch addresses several issues with passing a generic pandas
query string to sqobjects and below.
* Asserts should not take a query string yet as they cause all sorts
  of problems. The GUI will have filters not affect the assert output.
  When we move asserts to their own page, this dicrepancy should fix
  itself.
* engineobj needed to handle query_str only in get_valid_df and not
  individually in other calls or functions.
* Add tests to REST testing to validate that query string is working
  correctly

suzieq/engines/pandas/engineobj.py

@@ -55,6 +55,11 @@ def get_valid_df(self, table, **kwargs) -> pd.DataFrame:
         active_only = kwargs.pop('active_only', True)
         query_str = kwargs.pop('query_str', '')
 
+        # The REST API provides the query_str enclosed in ". Strip that
+        if query_str:
+            if query_str.startswith('"') and query_str.endswith('"'):
+                query_str = query_str[1:-1]
+
         fields = sch.get_display_fields(columns)
         key_fields = sch.key_fields()
         drop_cols = []
@@ -357,15 +362,9 @@ def top(self, **kwargs):
 
     def _init_summarize(self, table, **kwargs):
         kwargs.pop('columns', None)
-        query_str = kwargs.pop('query_str', '')
         columns = ['*']
 
         df = self.get(columns=columns, **kwargs)
-        if not df.empty:
-            try:
-                df = df.query(query_str)
-            except Exception:
-                pass
 
         self.summary_df = df
         if df.empty:

suzieq/engines/pandas/ospf.py

@@ -257,59 +257,67 @@ def aver(self, **kwargs):
             return ospf_df[['namespace', 'hostname', 'vrf', 'ifname',
                             'assertReason', 'assert']]
 
-        ospf_df = ospf_df.merge(int_df,
+        peer_df = ospf_df.merge(int_df,
                                 left_on=["namespace", "hostname", "ifname"],
                                 right_on=["namespace", "peerHostname",
                                           "peerIfname"]) \
             .dropna(how="any")
 
-        # Now start comparing the various parameters
-        ospf_df["assertReason"] += ospf_df.apply(
-            lambda x: ["subnet mismatch"]
-            if (
-                (x["isUnnumbered_x"] != x["isUnnumbered_y"])
-                and (
-                    IPv4Network(x["ipAddress_x"], strict=False)
-                    != IPv4Network(x["ipAddress_y"], strict=False)
+        if peer_df.empty:
+            ospf_df = ospf_df[~(ospf_df.ifname.str.contains('loopback') |
+                                ospf_df.ifname.str.contains('Vlan'))]
+            ospf_df['assertReason'] = 'No LLDP peering info'
+            ospf_df['assert'] = 'fail'
+        else:
+            ospf_df = peer_df
+            # Now start comparing the various parameters
+            ospf_df["assertReason"] += ospf_df.apply(
+                lambda x: ["subnet mismatch"]
+                if (
+                    (x["isUnnumbered_x"] != x["isUnnumbered_y"])
+                    and (
+                        IPv4Network(x["ipAddress_x"], strict=False)
+                        != IPv4Network(x["ipAddress_y"], strict=False)
+                    )
                 )
+                else [],
+                axis=1,
+            )
+            ospf_df["assertReason"] += ospf_df.apply(
+                lambda x: ["area mismatch"]
+                if (x["area_x"] != x["area_y"] and
+                    x["areaStub_x"] != x["areaStub_y"])
+                else [],
+                axis=1,
+            )
+            ospf_df["assertReason"] += ospf_df.apply(
+                lambda x: ["Hello timers mismatch"]
+                if x["helloTime_x"] != x["helloTime_y"]
+                else [],
+                axis=1,
+            )
+            ospf_df["assertReason"] += ospf_df.apply(
+                lambda x: ["Dead timer mismatch"]
+                if x["deadTime_x"] != x["deadTime_y"]
+                else [],
+                axis=1,
+            )
+            ospf_df["assertReason"] += ospf_df.apply(
+                lambda x: ["network type mismatch"]
+                if x["networkType_x"] != x["networkType_y"]
+                else [],
+                axis=1,
+            )
+            ospf_df["assertReason"] += ospf_df.apply(
+                lambda x: ["passive config mismatch"]
+                if x["passive_x"] != x["passive_y"]
+                else [],
+                axis=1,
+            )
+            ospf_df["assertReason"] += ospf_df.apply(
+                lambda x: ["vrf mismatch"] if x["vrf_x"] != x["vrf_y"] else [],
+                axis=1,
             )
-            else [],
-            axis=1,
-        )
-        ospf_df["assertReason"] += ospf_df.apply(
-            lambda x: ["area mismatch"]
-            if (x["area_x"] != x["area_y"] and x["areaStub_x"] != x["areaStub_y"])
-            else [],
-            axis=1,
-        )
-        ospf_df["assertReason"] += ospf_df.apply(
-            lambda x: ["Hello timers mismatch"]
-            if x["helloTime_x"] != x["helloTime_y"]
-            else [],
-            axis=1,
-        )
-        ospf_df["assertReason"] += ospf_df.apply(
-            lambda x: ["Dead timer mismatch"]
-            if x["deadTime_x"] != x["deadTime_y"]
-            else [],
-            axis=1,
-        )
-        ospf_df["assertReason"] += ospf_df.apply(
-            lambda x: ["network type mismatch"]
-            if x["networkType_x"] != x["networkType_y"]
-            else [],
-            axis=1,
-        )
-        ospf_df["assertReason"] += ospf_df.apply(
-            lambda x: ["passive config mismatch"]
-            if x["passive_x"] != x["passive_y"]
-            else [],
-            axis=1,
-        )
-        ospf_df["assertReason"] += ospf_df.apply(
-            lambda x: ["vrf mismatch"] if x["vrf_x"] != x["vrf_y"] else [],
-            axis=1,
-        )
 
         # Fill up a single assert column now indicating pass/fail
         ospf_df['assert'] = ospf_df.apply(lambda x: 'pass'

suzieq/gui/pages/xplore.py

@@ -278,7 +278,6 @@ def page_work(state_container, page_flip: bool):
             assert_df = xplore_run_assert(sqobjs[state.table],
                                           start_time=state.start_time,
                                           end_time=state.end_time,
-                                          query_str=state.query,
                                           namespace=state.namespace.split())
         else:
             assert_df = pd.DataFrame()

suzieq/sqobjects/bgp.py

@@ -12,8 +12,7 @@ def __init__(self, **kwargs):
             'state': ['Established', 'NotEstd', ''],
             'status': ['all', 'pass', 'fail'],
         }
-        self._valid_assert_args = ['namespace', 'hostname', 'vrf', 'status',
-                                   'query_str']
+        self._valid_assert_args = ['namespace', 'hostname', 'vrf', 'status']
 
     def aver(self, **kwargs):
         """Assert that the BGP state is OK"""

suzieq/sqobjects/evpnVni.py

@@ -7,8 +7,7 @@ def __init__(self, **kwargs):
         super().__init__(table='evpnVni', **kwargs)
         self._valid_get_args = ['namespace', 'hostname', 'columns', 'vni',
                                 'query_str']
-        self._valid_assert_args = ['namespace', 'hostname', 'status',
-                                   'query_str']
+        self._valid_assert_args = ['namespace', 'hostname', 'status']
         self._valid_arg_vals = {
             'status': ['all', 'pass', 'fail'],
         }

suzieq/sqobjects/interfaces.py

@@ -10,7 +10,7 @@ def __init__(self, **kwargs):
         self._valid_get_args = ['namespace', 'hostname', 'ifname', 'columns',
                                 'state', 'type', 'mtu', 'query_str']
         self._valid_assert_args = ['namespace', 'hostname', 'ifname',
-                                   'what', 'matchval', 'status', 'query_str']
+                                   'what', 'matchval', 'status']
         self._valid_arg_vals = {
             'state': ['up', 'down', ''],
             'status': ['all', 'pass', 'fail'],

suzieq/sqobjects/ospf.py

@@ -12,8 +12,7 @@ def __init__(self, **kwargs):
         self._addnl_nbr_fields = ['state']
         self._valid_get_args = ['namespace', 'hostname', 'columns',
                                 'vrf', 'ifname', 'state', 'query_str']
-        self._valid_assert_args = ['namespace', 'vrf', 'status',
-                                   'query_str']
+        self._valid_assert_args = ['namespace', 'vrf', 'status']
         self._valid_arg_vals = {
             'state': ['full', 'other', 'passive', ''],
             'status': ['all', 'pass', 'fail'],
@@ -56,7 +55,8 @@ def humanize_fields(self, df: pd.DataFrame, subset=None) -> pd.DataFrame:
             return df
 
         if 'lastChangeTime' in df.columns:
-            df['lastChangeTime'] = humanize_timestamp(df.lastChangeTime)
+            df['lastChangeTime'] = humanize_timestamp(
+                df.lastChangeTime.fillna(0))
 
             if 'adjState' in df.columns:
                 df['lastChangeTime'] = np.where(df.adjState == "passive", "-",

suzieq/sqobjects/routes.py

@@ -24,7 +24,9 @@ def summarize(self, namespace=[], vrf=[], hostname=[], query_str=''):
         """Summarize routing info for one or more namespaces"""
         if self.columns != ["default"]:
             self.summarize_df = pd.DataFrame(
-                {'error': ['ERROR: You cannot specify columns with summarize']})
+                {'error':
+                 ['ERROR: You cannot specify columns with summarize']})
             return self.summarize_df
         return self.engine_obj.summarize(namespace=namespace, vrf=vrf,
+                                         query_str=query_str,
                                          hostname=hostname)

suzieq/sqobjects/topology.py

@@ -22,11 +22,13 @@ def __init__(self, **kwargs):
                                 'polled_neighbor', 'query_str']
 
     # overriding parent because we want to take more arguments than the standard
-    def summarize(self, namespace=[], hostname=[],
-                  polled_neighbor=None) -> pd.DataFrame:
+    def summarize(self, namespace: typing.List[str] = [],
+                  hostname: typing.List[str] = [],
+                  polled_neighbor=None, query_str: str = '') -> pd.DataFrame:
         if self.columns != ["default"]:
             self.summarize_df = pd.DataFrame(
                 {'error': ['ERROR: You cannot specify columns with summarize']})
             return self.summarize_df
         return self.engine_obj.summarize(namespace=namespace, hostname=hostname,
+                                         query_str=query_str,
                                          polled_neighbor=polled_neighbor)

tests/integration/test_rest.py

@@ -49,6 +49,8 @@
            'status=whatever',
            'vlanName=vlan13',
            'state=active',
+           'query_str="hostname == \"leaf01\""',
+           'query_str="hostname == \"leaf01\" and 1000 < mtu < 2000"'
            ]
 
 # these should only succeed for the specific service/verb tuples
@@ -145,6 +147,8 @@
                     'unique?': 405,
                     'unique?namespace=ospf-ibgp': 405,
                     'unique?view=latest': 405,
+                    'assert?query_str="hostname == \"leaf01\""': 405,
+                    'assert?query_str="hostname == \"leaf01\" and 1000 < mtu < 2000"': 405,
                     }
 
 # these service/verb/filter tuples should return errors
@@ -154,20 +158,26 @@
     'address/summarize?address=10.127.1.2&view=all': 405,
     'address/summarize?ipvers=v4': 405,
     'address/summarize?vrf=default': 405,
+    'address/show?query_str="hostname == \"leaf01\" and 1000 < mtu < 2000"': 500,
+    'address/summarize?query_str="hostname == \"leaf01\" and 1000 < mtu < 2000"': 500,
     'arpnd/summarize?macaddr=c2:6d:17:7a:bd:03': 405,
     'arpnd/summarize?macaddr=8a:92:4b:c1:ea:03%20c2:6d:17:7a:bd:03': 405,
     'arpnd/summarize?ipAddress=10.127.1.2': 405,
     'arpnd/summarize?oif=eth1.4': 405,
+    'arpnd/show?query_str="hostname == \"leaf01\" and 1000 < mtu < 2000"': 500,
+    'arpnd/summarize?query_str="hostname == \"leaf01\" and 1000 < mtu < 2000"': 500,
     'bgp/show?state=pass': 405,
     'bgp/show?status=pass': 405,
     'bgp/show?status=fail': 405,
     'bgp/show?status=all': 405,
     'bgp/show?status=whatever': 405,
     'bgp/show?state=active': 405,
+    'bgp/show?query_str="hostname == \"leaf01\" and 1000 < mtu < 2000"': 500,
     'bgp/assert?peer=eth1.2': 405,
     'bgp/assert?state=pass': 405,
     'bgp/assert?state=whatever': 405,
     'bgp/assert?state=active': 405,
+    'bgp/summarize?query_str="hostname == \"leaf01\" and 1000 < mtu < 2000"': 500,
     'bgp/summarize?peer=eth1.2': 405,
     'bgp/summarize?state=pass': 405,
     'bgp/summarize?state=Established': 405,
@@ -178,10 +188,13 @@
     'bgp/summarize?status=fail': 405,
     'bgp/summarize?status=all': 405,
     'bgp/summarize?status=whatever': 405,
+    'device/summarize?query_str="hostname == \"leaf01\" and 1000 < mtu < 2000"': 500,
+    'device/show?query_str="hostname == \"leaf01\" and 1000 < mtu < 2000"': 500,
     'evpnVni/show?status=pass': 405,
     'evpnVni/show?status=fail': 405,
     'evpnVni/show?status=all': 405,
     'evpnVni/show?status=whatever': 405,
+    'evpnVni/show?query_str="hostname == \"leaf01\" and 1000 < mtu < 2000"': 500,
     'evpnVni/assert?vni=13': 405,
     'evpnVni/assert?vni=13%2024': 405,
     'evpnVni/summarize?vni=13': 405,
@@ -190,8 +203,11 @@
     'evpnVni/summarize?status=fail': 405,
     'evpnVni/summarize?status=all': 405,
     'evpnVni/summarize?status=whatever': 405,
+    'evpnVni/summarize?query_str="hostname == \"leaf01\" and 1000 < mtu < 2000"': 500,
+    'fs/show?query_str="hostname == \"leaf01\" and 1000 < mtu < 2000"': 500,
     'fs/summarize?usedPercent=8': 405,
     'fs/summarize?mountPoint=/': 405,
+    'fs/summarize?query_str="hostname == \"leaf01\" and 1000 < mtu < 2000"': 500,
     'interface/assert?state=active': 405,
     'interface/show?state=pass': 405,
     'interface/show?status=pass': 405,
@@ -210,23 +226,31 @@
     'interface/summarize?status=whatever': 405,
     'interface/summarize?state=active': 405,
     'lldp/summarize?ifname=swp1': 405,
+    'lldp/show?query_str="hostname == \"leaf01\" and 1000 < mtu < 2000"': 500,
+    'lldp/summarize?query_str="hostname == \"leaf01\" and 1000 < mtu < 2000"': 500,
+    'mac/show?query_str="hostname == \"leaf01\" and 1000 < mtu < 2000"': 500,
+    'mac/summarize?query_str="hostname == \"leaf01\" and 1000 < mtu < 2000"': 500,
     'mac/summarize?bd=': 405,
     'mac/summarize?macaddr=c2:6d:17:7a:bd:03': 405,
     'mac/summarize?macaddr=8a:92:4b:c1:ea:03%20c2:6d:17:7a:bd:03': 405,
     'mac/summarize?localOnly=True': 405,
     'mac/summarize?remoteVtepIp=10.0.0.101': 405,
     'mac/summarize?vlan=13': 405,
+    'mlag/show?query_str="hostname == \"leaf01\" and 1000 < mtu < 2000"': 500,
+    'mlag/summarize?query_str="hostname == \"leaf01\" and 1000 < mtu < 2000"': 500,
     'ospf/show?state=pass': 405,
     'ospf/show?status=pass': 405,
     'ospf/show?status=fail': 405,
     'ospf/show?status=all': 405,
     'ospf/show?state=active': 405,
     'ospf/show?status=whatever': 405,
+    'ospf/show?query_str="hostname == \"leaf01\" and 1000 < mtu < 2000"': 500,
     'ospf/assert?hostname=leaf01': 405,
     'ospf/assert?hostname=leaf01%20spine01': 405,
     'ospf/assert?state=pass': 405,
     'ospf/assert?state=active': 405,
     'ospf/assert?ifname=swp1': 405,
+    'ospf/summarize?query_str="hostname == \"leaf01\" and 1000 < mtu < 2000"': 500,
     'ospf/summarize?ifname=swp1': 405,
     'ospf/summarize?state=pass': 405,
     'ospf/summarize?vrf=default': 405,
@@ -237,12 +261,17 @@
     'ospf/summarize?status=whatever': 405,
     'ospf/summarize?state=active': 405,
     'ospf/show?status=all': 405,
+    'path/show?query_str="hostname == \"leaf01\" and 1000 < mtu < 2000"': 404,
     'path/summarize?columns=namespace': 404,
+    'path/summarize?query_str="hostname == \"leaf01\" and 1000 < mtu < 2000"': 404,
     'path/unique?columns=namespace': 404,
+    'route/show?query_str="hostname == \"leaf01\" and 1000 < mtu < 2000"': 500,
+    'route/summarize?query_str="hostname == \"leaf01\" and 1000 < mtu < 2000"': 500,
     'route/lpm?': 404,
     'route/lpm?columns=namespace': 404,
     'route/lpm?hostname=leaf01': 404,
     'route/lpm?namespace=ospf-ibgp': 404,
+    'route/summarize?query_str="hostname == \"leaf01\" and 1000 < mtu < 2000"': 500,
     'route/summarize?address=10.127.1.2': 405,
     'route/summarize?address=10.127.1.2&view=all': 405,
     'route/summarize?ipvers=v4': 405,
@@ -251,13 +280,19 @@
     'route/summarize?protocol=bgp': 405,
     'route/show?ipvers=v4': 405,
     'route/lpm?view=latest': 404,
+    'sqpoller/show?query_str="hostname == \"leaf01\" and 1000 < mtu < 2000"': 500,
     'sqpoller/show?status=whatever': 405,
+    'sqpoller/summarize?query_str="hostname == \"leaf01\" and 1000 < mtu < 2000"': 500,
     'sqpoller/summarize?status=pass': 405,
     'sqpoller/summarize?status=fail': 405,
     'sqpoller/summarize?status=all': 405,
     'sqpoller/summarize?status=whatever': 405,
     'sqpoller/summarize?service=device': 405,
+    'topology/show?query_str="hostname == \"leaf01\" and 1000 < mtu < 2000"': 500,
+    'topology/summarize?query_str="hostname == \"leaf01\" and 1000 < mtu < 2000"': 500,
     'vlan/show?state=pass': 405,
+    'vlan/show?query_str="hostname == \"leaf01\" and 1000 < mtu < 2000"': 500,
+    'vlan/summarize?query_str="hostname == \"leaf01\" and 1000 < mtu < 2000"': 500,
     'vlan/summarize?vlan=13': 405,
     'vlan/summarize?state=pass': 405,
     'vlan/summarize?state=active': 405,