Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Include new template query with ipv6/mac. Update mermaind show bgp. t… #129

Merged
merged 1 commit into from
Aug 14, 2024
Merged

Include new template query with ipv6/mac. Update mermaind show bgp. t… #129

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 11 additions & 8 deletions autogole-api/src/python/RTMon/worker.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,7 @@ def running_exe(self, filename, fout):
self.logger.info('Dashboard is present in Grafana: %s', dashbName)
# Check if we need to re-issue ping test
tmpOut = self.sr_submit_ping(instance=fout.get('instance', {}), manifest=fout.get('manifest', {}))
if tmpOut:
if tmpOut and fout.get('dashbInfo', {}):
fout['ping'] = tmpOut
self.g_submitAnnotation(sitermOut=tmpOut, dashbInfo=fout["dashbInfo"])
self._updateState(filename, fout)
Expand Down Expand Up @@ -227,13 +227,16 @@ def _startwork(self):
# Load all grafana dashboards
self.g_loadAll()
for key, val in self.config.get('sense_endpoints', {}).items():
startTime = int(time.time())
os.environ['SENSE_AUTH_OVERRIDE_NAME'] = key
os.environ['SENSE_AUTH_OVERRIDE'] = val
self.s_reloadClient()
self._getAllInstances()
endTime = int(time.time())
timings[key] = endTime - startTime
try:
startTime = int(time.time())
os.environ['SENSE_AUTH_OVERRIDE_NAME'] = key
os.environ['SENSE_AUTH_OVERRIDE'] = val
self.s_reloadClient()
self._getAllInstances()
endTime = int(time.time())
timings[key] = endTime - startTime
except SENSEOFailure as ex:
self.logger.error('SENSEOFailure: %s', ex)
startTime = int(time.time())
self.logger.info('Running Main')
self.main()
Expand Down
47 changes: 27 additions & 20 deletions autogole-api/src/python/RTMonLibs/SenseAPI.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,26 +47,33 @@ def s_getDiscoverApi(self):
def s_getManifest(self, instance):
"""Create a manifest in SENSE-0"""
template = {"Ports": [{
"Port": "?terminal?",
"Name": "?port_name?",
"Vlan": "?vlan?",
"Node": "?node_name?",
"Peer": "?peer?",
"Site": "?site?",
"Host": [{
"Interface": "?host_port_name?",
"Name": "?host_name?",
"IPv4": "?ipv4?",
"IPv6": "?ipv6?",
"MAC": "?mac?",
"sparql": "SELECT DISTINCT ?host_port ?ipv4 ?ipv6 ?mac WHERE { ?vlan_port nml:isAlias ?host_vlan_port. ?host_port nml:hasBidirectionalPort ?host_vlan_port. OPTIONAL {?host_vlan_port mrs:hasNetworkAddress ?ipv4na. ?ipv4na mrs:type \"ipv4-address\". ?ipv4na mrs:value ?ipv4.} OPTIONAL {?host_vlan_port mrs:hasNetworkAddress ?ipv6na. ?ipv6na mrs:type \"ipv6-address\". ?ipv6na mrs:value ?ipv6.} OPTIONAL {?host_vlan_port mrs:hasNetworkAddress ?macna. ?macna mrs:type \"mac-address\". ?macna mrs:value ?mac.} FILTER NOT EXISTS {?sw_svc mrs:providesSubnet ?vlan_subnt. ?vlan_subnt nml:hasBidirectionalPort ?host_vlan_port.} }",
"sparql-ext": "SELECT DISTINCT ?host_name ?host_port_name WHERE {?host a nml:Node. ?host nml:hasBidirectionalPort ?host_port. OPTIONAL {?host nml:name ?host_name.} OPTIONAL {?host_port mrs:hasNetworkAddress ?na_pn. ?na_pn mrs:type \"sense-rtmon:name\". ?na_pn mrs:value ?host_port_name.} }",
"required": "false"
}],
"sparql": "SELECT DISTINCT ?vlan_port ?vlan WHERE { ?subnet a mrs:SwitchingSubnet. ?subnet nml:hasBidirectionalPort ?vlan_port. ?vlan_port nml:hasLabel ?vlan_l. ?vlan_l nml:value ?vlan. }",
"sparql-ext": "SELECT DISTINCT ?terminal ?port_name ?node_name ?peer ?site WHERE { {?node a nml:Node. ?node nml:name ?node_name. ?node nml:hasBidirectionalPort ?terminal. ?terminal nml:hasBidirectionalPort ?vlan_port. OPTIONAL {?terminal mrs:hasNetworkAddress ?na_pn. ?na_pn mrs:type \"sense-rtmon:name\". ?na_pn mrs:value ?port_name.} OPTIONAL {?terminal nml:isAlias ?peer.} OPTIONAL {?site nml:hasNode ?node.} OPTIONAL {?site nml:hasTopology ?sub_site. ?sub_site nml:hasNode ?node.} } UNION { ?site a nml:Topology. ?site nml:name ?node_name. ?site nml:hasBidirectionalPort ?terminal. ?terminal nml:hasBidirectionalPort ?vlan_port. OPTIONAL {?terminal mrs:hasNetworkAddress ?na_pn. ?na_pn mrs:type \"sense-rtmon:name\". ?na_pn mrs:value ?port_name.} OPTIONAL {?terminal nml:isAlias ?peer.}}}",
"required": "true"
}]}
"Port": "?terminal?",
"Name": "?port_name?",
"Vlan": "?vlan?",
"Mac": "?port_mac?",
"IPv6": "?port_ipv6?",
"IPv4": "?port_ipv4?",
"Node": "?node_name?",
"Peer": "?peer?",
"Site": "?site?",
"Host": [
{
"Interface": "?host_port_name?",
"Name": "?host_name?",
"IPv4": "?ipv4?",
"IPv6": "?ipv6?",
"Mac": "?mac?",
"sparql": "SELECT DISTINCT ?host_port ?ipv4 ?ipv6 ?mac WHERE { ?host_vlan_port nml:isAlias ?vlan_port. ?host_port nml:hasBidirectionalPort ?host_vlan_port. OPTIONAL {?host_vlan_port mrs:hasNetworkAddress ?ipv4na. ?ipv4na mrs:type \"ipv4-address\". ?ipv4na mrs:value ?ipv4.} OPTIONAL {?host_vlan_port mrs:hasNetworkAddress ?ipv6na. ?ipv6na mrs:type \"ipv6-address\". ?ipv6na mrs:value ?ipv6.} OPTIONAL {?host_vlan_port mrs:hasNetworkAddress ?macana. ?macana mrs:type \"mac-address\". ?macana mrs:value ?mac.} FILTER NOT EXISTS {?sw_svc mrs:providesSubnet ?vlan_subnt. ?vlan_subnt nml:hasBidirectionalPort ?host_vlan_port.} }",
"sparql-ext": "SELECT DISTINCT ?host_name ?host_port_name WHERE {?host a nml:Node. ?host nml:hasBidirectionalPort ?host_port. OPTIONAL {?host nml:name ?host_name.} OPTIONAL {?host_port mrs:hasNetworkAddress ?na_pn. ?na_pn mrs:type \"sense-rtmon:name\". ?na_pn mrs:value ?host_port_name.} }",
"required": "false"
}
],
"sparql": "SELECT DISTINCT ?vlan_port ?vlan WHERE { ?subnet a mrs:SwitchingSubnet. ?subnet nml:hasBidirectionalPort ?vlan_port. ?vlan_port nml:hasLabel ?vlan_l. ?vlan_l nml:value ?vlan. }",
"sparql-ext": "SELECT DISTINCT ?terminal ?port_name ?node_name ?peer ?site ?port_mac ?port_ipv4 ?port_ipv6 WHERE { { ?node a nml:Node. ?node nml:name ?node_name. ?node nml:hasBidirectionalPort ?terminal. ?terminal nml:hasBidirectionalPort ?vlan_port. OPTIONAL { ?terminal mrs:hasNetworkAddress ?na_pn. ?na_pn mrs:type \"sense-rtmon:name\". ?na_pn mrs:value ?port_name. } OPTIONAL { ?terminal nml:isAlias ?peer. } OPTIONAL { ?site nml:hasNode ?node. } OPTIONAL { ?site nml:hasTopology ?sub_site. ?sub_site nml:hasNode ?node. } OPTIONAL { ?terminal mrs:hasNetworkAddress ?naportmac. ?naportmac mrs:type \"mac-address\". ?naportmac mrs:value ?port_mac. } OPTIONAL { ?vlan_port mrs:hasNetworkAddress ?ipv4na. ?ipv4na mrs:type \"ipv4-address\". ?ipv4na mrs:value ?port_ipv4. } OPTIONAL { ?vlan_port mrs:hasNetworkAddress ?ipv6na. ?ipv6na mrs:type \"ipv6-address\". ?ipv6na mrs:value ?port_ipv6. } } UNION { ?site a nml:Topology. ?site nml:name ?node_name. ?site nml:hasBidirectionalPort ?terminal. ?terminal nml:hasBidirectionalPort ?vlan_port. OPTIONAL { ?terminal mrs:hasNetworkAddress ?na_pn. ?na_pn mrs:type \"sense-rtmon:name\". ?na_pn mrs:value ?port_name. } OPTIONAL { ?terminal nml:isAlias ?peer. } OPTIONAL { ?terminal mrs:hasNetworkAddress ?naportmac. ?naportmac mrs:type \"mac-address\". ?naportmac mrs:value ?port_mac. } OPTIONAL { ?vlan_port mrs:hasNetworkAddress ?ipv4na. ?ipv4na mrs:type \"ipv4-address\". ?ipv4na mrs:value ?port_ipv4. } OPTIONAL { ?vlan_port mrs:hasNetworkAddress ?ipv6na. ?ipv6na mrs:type \"ipv6-address\". ?ipv6na mrs:value ?port_ipv6. } } }",
"required": "true"
}
]
}
wApi = self.s_getWorkflowApi()
wApi.si_uuid = instance['referenceUUID']
response = wApi.manifest_create(dumpJson(template, self.logger))
Expand Down
9 changes: 7 additions & 2 deletions autogole-api/src/python/RTMonLibs/SiteRMApi.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,8 +62,13 @@ def sr_submit_ping(self, **kwargs):
for key, defval in [("IPv4", "?ipv4?"), ("IPv6", "?ipv6?")]:
if host.get(key) and host[key] != defval:
hostspl = host.get("Name").split(':')
allDebugActions = self.sr_get_debug_actions(**{'sitename': hostspl[0],
'hostname': hostspl[1]})
try:
allDebugActions = self.sr_get_debug_actions(**{'sitename': hostspl[0],
'hostname': hostspl[1]})
except Exception as e:
self.logger.error(f"Failed to get debug actions for {hostspl[0]}:{hostspl[1]}: {e}")
allDebugActions = []
continue
for ip in allIPs.get(key, []):
hostip = host[key].split('/')[0]
if hostip == ip:
Expand Down
40 changes: 37 additions & 3 deletions autogole-api/src/python/RTMonLibs/Template.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ def __init__(self, **kwargs):
self.mac_addresses = {}
self.orderlist = []
self.orderlistports = []
self.instance = None

def _m_cleanCache(self):
"""Clean Cache"""
Expand All @@ -40,6 +41,7 @@ def _m_cleanCache(self):
self.vlans = {}
self.orderlist = []
self.orderlistports = []
self.instance = None

def _m_addLink(self, val1, val2):
if [val1, val2] not in self.links and [val2, val1] not in self.links:
Expand All @@ -62,6 +64,22 @@ def _m_recordMac(self, hostdict):
if 'MAC' in hostdict and hostdict['MAC'] not in self.mac_addresses and hostdict['MAC'] != "?mac?":
self.mac_addresses.setdefault(hostname, hostdict['MAC'])

def _m_addBGP(self, item, ipkey, bgppeer):
"""Add BGP into Mermaid graph"""
if not item.get('Site', None):
return
for intitem in self.instance.get('intents', []):
for connections in intitem.get('json', {}).get('data', {}).get('connections', []):
for terminal in connections.get('terminals', []):
if 'uri' not in terminal:
continue
if item['Site'] == terminal['uri'] and terminal.get(f'{ipkey.lower()}_prefix_list', None):
val = terminal[f'{ipkey.lower()}_prefix_list']
self.mermaid.append(f' {bgppeer}_bgp{ipkey}(BGP_{ipkey})')
self._m_addLink(bgppeer, f'{bgppeer}_bgp{ipkey}')
self.mermaid.append(f' {bgppeer}_bgp{ipkey}_peer({val})')
self._m_addLink(f'{bgppeer}_bgp{ipkey}', f'{bgppeer}_bgp{ipkey}_peer')

def _m_addSwitch(self, item):
uniqname = _processName(f'{item["Node"]}_{item["Name"]}')
self.m_groups['Switches'].setdefault(item["Node"], {}).setdefault(item["Name"], {})
Expand All @@ -72,12 +90,23 @@ def _m_addSwitch(self, item):
else:
self.mermaid.append(f' subgraph "{item["Node"]}"')
self.mermaid.append(f' {uniqname}("{item["Name"]}")')
self.mermaid.append(' end')
if 'Peer' in item and item['Peer'] != "?peer?":
self._m_addLink(uniqname, _processName(item['Peer']))
if 'Vlan' in item and item['Vlan']:
self._m_addVlan(f'{uniqname}_{item["Peer"]}', item['Vlan'])
self._m_addPorts(_processName(item['Port']), uniqname)
# Add IPv4/IPv6 on the switch
for ipkey, ipdef in {'IPv4': '?port_ipv4?', 'IPv6': '?port_ipv6?'}.items():
if ipkey in item and item[ipkey] != ipdef:
self.mermaid.append(f' {uniqname}_{ipkey}({item[ipkey]})')
if item.get('Vlan'):
self.mermaid.append(f' {uniqname}_vlan{item["Vlan"]}(vlan.{item["Vlan"]})')
self._m_addLink(uniqname, f'{uniqname}_vlan{item["Vlan"]}')
self._m_addLink(f'{uniqname}_vlan{item["Vlan"]}', f'{uniqname}_{ipkey}')
# Add BGP Peering information
self._m_addBGP(item, ipkey, f'{uniqname}_{ipkey}')
# TODO: Here we should save IP for issuing ping between SiteRM endpoints (switches)
self.mermaid.append(' end')
return uniqname

def _m_addHost(self, host):
Expand Down Expand Up @@ -228,9 +257,10 @@ def findorder(self, manifest):
nexthop = manifest["Ports"][0]["Node"]
counter = 50

def m_getMermaidContent(self, manifest):
def m_getMermaidContent(self, instance, manifest):
"""Create Mermaid Template"""
self._m_cleanCache()
self.instance = instance
self.findorder(manifest)
for item in self.orderlist:
self._m_addItem(item)
Expand Down Expand Up @@ -414,6 +444,10 @@ def findIntf(interfaces):
intfs.append(splitdata[2])
else:
intfs.append(intfname)
# Add also lowercase and space removed intf names
# https://github.com/esnet/sense-rtmon/issues/128
intfs.append(intfname.lower())
intfs.append(intfname.replace(" ", ""))
intfline = "|".join(intfs)
return intfline
out = []
Expand Down Expand Up @@ -446,7 +480,7 @@ def t_createMermaid(self, *args):
self.so_mappeers(args[1])
row = self.t_addRow(*args, title="End-to-End Flow Monitoring")
panel = self._t_loadTemplate("mermaid.json")
mermaid = self.m_getMermaidContent(args[1])
mermaid = self.m_getMermaidContent(*args)
panel["options"]["content"] = "\n".join(mermaid)
# Need to add correct size for the panel
totalHeight = 12 + len(self.m_groups['Hosts']) + len(self.m_groups['Switches'])
Expand Down