#!/usr/bin/env python3 # # Copyright (c) 2017-2020 Joe Clarke # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF # SUCH DAMAGE. from builtins import str from builtins import range import os import re import sys import time import json import paramiko from multiprocessing import Pool import traceback import CLEUCreds CACHE_FILE = "/home/jclarke/mac_counts.dat" CACHE_FILE_TMP = CACHE_FILE + ".tmp" IDF_FILE = "/home/jclarke/idf-devices.json" commands = { "macCore": { "command": "show mac address-table count | inc Dynamic Address Count", "pattern": r"Dynamic Address Count:\s+(\d+)", "metric": "totalMacs", }, "macIdf": {"command": "show mac address-table dynamic | inc Total", "pattern": r"Total.*: (\d+)", "metric": "totalMacs",}, "arpEntries": {"command": "show ip arp summary | inc IP ARP", "pattern": r"(\d+) IP ARP entries", "metric": "arpEntries",}, "ndEntries": {"command": "show ipv6 neighbors statistics | inc Entries", "pattern": r"Entries (\d+),", "metric": "ndEntries",}, "natTrans": { "command": "show ip nat translations total", "pattern": r"Total number of translations: (\d+)", "metric": "natTranslations", }, "umbrella1Trans": { "command": "show platform hardware qfp active feature nat datapath limit", "pattern": r"limit_type 5 limit_id 0xa64fd06.*curr_count (\d+)", "metric": "umbrella1NatTrans", }, "umbrella2Trans": { "command": "show platform hardware qfp active feature nat datapath limit", "pattern": r"limit_type 5 limit_id 0xa64fe06.*curr_count (\d+)", "metric": "umbrella2NatTrans", }, "natPoolDefault1": { "command": "show ip nat statistics | begin NAT-POOL-DEFAULT-1", "pattern": r"total addresses (\d+), allocated (\d+)[^,]+, misses (\d+)", "metrics": ["natPoolDefault1Addresses", "natPoolDefault1Allocated", "natPoolDefault1Misses"], }, "natPoolDefault2": { "command": "show ip nat statistics | begin NAT-POOL-DEFAULT-2", "pattern": r"total addresses (\d+), allocated (\d+)[^,]+, misses (\d+)", "metrics": ["natPoolDefault2Addresses", "natPoolDefault2Allocated", "natPoolDefault2Misses"], }, "natPoolDns": { "command": "show ip nat statistics | begin NAT-POOL-DNS", "pattern": r"total addresses (\d+), allocated (\d+)[^,]+, misses (\d+)", "metrics": ["natPoolDnsAddresses", "natPoolDnsAllocated", "natPoolDnsMisses"], }, "natPoolLabs": { "command": "show ip nat statistics | begin NAT-POOL-LABS", "pattern": r"total addresses (\d+), allocated (\d+)[^,]+, misses (\d+)", "metrics": ["natPoolLabsAddresses", "natPoolLabsAllocated", "natPoolLabsMisses"], }, "natHealthStats": { "command": "show ip nat statistics | begin In-to-out", "pattern": r"In-to-out-drops: (\d+)\s+Out-to-in-drops: (\d+).*Pool stats drop: (\d+)\s+Mapping stats drop: (\d+).*Port block alloc fail: (\d+).*IP alias add fail: (\d+).*Limit entry add fail: (\d+)", "metrics": [ "natHealthInOutDrops", "natHealthOutInDrops", "natHealthStatsDrops", "natHealthPortBlockAllocFail", "natHealthAliasAddFail", "natHealthEntryAddFail", ], }, "qfpUtil": { "command": "show platform hardware qfp active datapath utilization summary", "pattern": r"Processing: Load \(pct\)\s+(\d+)", "metric": "qfpUtil", }, } devices = [ {"pattern": "CORE{}-L3C", "range": {"min": 1, "max": 2}, "commands": ["arpEntries", "ndEntries"],}, {"file": IDF_FILE, "commands": ["macIdf", "arpEntries", "ndEntries"],}, {"pattern": "CORE{}-WA", "range": {"min": 1, "max": 2}, "commands": ["macIdf", "arpEntries", "ndEntries"],}, { "pattern": "CORE{}-EDGE", "range": {"min": 1, "max": 2}, "commands": [ "natTrans", "qfpUtil", "umbrella1Trans", "umbrella2Trans", "natPoolDefault1", "natPoolDefault2", "natPoolDns", "natPoolLabs", "natHealthStats", ], }, ] def send_command(chan, command): chan.sendall(command + "\n") time.sleep(0.5) output = "" i = 0 while i < 60: r = chan.recv(65535) if len(r) == 0: raise EOFError("Remote host has closed the connection") r = r.decode("utf-8", "ignore") output += r if re.search(r"[#>]$", r.strip()): break time.sleep(1) return output def get_results(dev): global commands ssh_client = paramiko.SSHClient() ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) response = [] try: ssh_client.connect( dev["device"], username=CLEUCreds.NET_USER, password=CLEUCreds.NET_PASS, timeout=5, allow_agent=False, look_for_keys=False, ) chan = ssh_client.invoke_shell() try: send_command(chan, "term width 0") send_command(chan, "term length 0") for command in dev["commands"]: cmd = commands[command]["command"] pattern = commands[command]["pattern"] metric = None if "metric" in commands[command]: metric = commands[command]["metric"] output = "" try: output = send_command(chan, cmd) except Exception as iie: response.append("") sys.stderr.write("Failed to get result for {} from {}: {}\n".format(cmd, dev["device"], iie)) traceback.print_exc() m = re.search(pattern, output) if m: if metric: response.append('{}{{idf="{}"}} {}'.format(metric, dev["device"], m.group(1))) else: metrics = commands[command]["metrics"] i = 1 for metric in metrics: response.append('{}{{idf="{}"}} {}'.format(metric, dev["device"], m.group(i))) i += 1 else: # sys.stderr.write( # 'Failed to find pattern "{}" in "{}"\n'.format(pattern, output) # ) if metric: response.append('{}{{idf="{}"}} {}'.format(metric, dev["device"], 0)) else: metrics = commands[command]["metrics"] for metric in metrics: response.append('{}{{idf="{}"}} {}'.format(metric, dev["device"], 0)) except Exception as ie: for command in dev["commands"]: response.append("") sys.stderr.write("Failed to setup SSH on {}: {}\n".format(dev["device"], ie)) traceback.print_exc() except Exception as e: for command in dev["commands"]: response.append("") sys.stderr.write("Failed to connect to {}: {}\n".format(dev["device"], e)) ssh_client.close() return response def get_metrics(pool): response = [] targets = [] for device in devices: if "list" in device: for dev in device["list"]: targets.append({"device": dev, "commands": device["commands"]}) elif "range" in device or "subs" in device: if "range" in device: for i in range(device["range"]["min"], device["range"]["max"] + 1): targets.append( {"device": device["pattern"].format(str(i)), "commands": device["commands"],} ) else: for sub in device["subs"]: targets.append( {"device": device["pattern"].format(sub), "commands": device["commands"],} ) else: with open(device["file"]) as fd: for dev in json.load(fd): targets.append({"device": dev, "commands": device["commands"]}) results = [pool.apply_async(get_results, [d]) for d in targets] for res in results: retval = res.get() if retval is not None: response += retval return response if __name__ == "__main__": pool = Pool(20) response = get_metrics(pool) fd = open(CACHE_FILE_TMP, "w") json.dump(response, fd, indent=4) fd.close() os.rename(CACHE_FILE_TMP, CACHE_FILE)