File: //lib/fm-agent/plugins/bandwidth.py
import agent_util
import sys
import os
from datetime import datetime
from agent_util import float
import fnmatch
class BandwidthPlugin(agent_util.Plugin):
textkey = "bandwidth"
label = "Bandwidth"
@classmethod
def get_metadata(self, config):
def add_to_interfaces(iface):
if any(fnmatch.fnmatch(iface, f) for f in filter_interfaces):
return
if type(interfaces) == set:
interfaces.add(iface)
else:
interfaces.append(iface)
status = agent_util.SUPPORTED
msg = None
interfaces = set()
filter_interfaces = []
filter_interfaces_config = config.get("filter_interfaces")
if filter_interfaces_config:
if filter_interfaces_config[0] == "[":
filter_interfaces_config = filter_interfaces_config.strip("[]")
for f in filter_interfaces_config.split(","):
f = f.strip()
if f:
if f[0] == '"':
f = f.strip('"')
elif f[0] == "'":
f = f.strip("'")
if f:
filter_interfaces.append(f)
if "freebsd" in sys.platform or "darwin" in sys.platform:
netstat_binary = agent_util.which("netstat")
if not netstat_binary:
self.log.info("netstat not found")
status = agent_util.UNSUPPORTED
msg = "Please install netstat."
return {}
if status is agent_util.SUPPORTED:
ret, output = agent_util.execute_command("%s -ib" % netstat_binary)
self.log.debug("BANDWIDTH INFO")
self.log.debug(output)
if config.get("debug", False):
self.log.debug(
"#####################################################"
)
self.log.debug("Bandwidth command 'netstat -ib' output:")
self.log.debug(output)
self.log.debug(
"#####################################################"
)
output = output.splitlines()[1:]
for line in output:
if line == "" or "lo" in line:
continue
stuff = line.strip().split()
iface = stuff[0]
add_to_interfaces(iface)
elif "aix" in sys.platform:
if not agent_util.which("netstat"):
self.log.info("netstat not found")
status = agent_util.UNSUPPORTED
msg = "Please install netstat."
return {}
# Get the list of network devices
interfaces = []
ret, output = agent_util.execute_command(
"netstat -v | grep 'ETHERNET STATISTICS'"
)
output = output.strip().split("\n")
for line in output:
fields = line.split()
try:
add_to_interfaces(fields[2].strip("(").strip(")"))
except:
pass
metadata = {}
metadata["bandwidth.kbytes.in"] = {
"label": "Kilobytes IN per second",
"options": interfaces,
"status": status,
"error_message": msg,
"unit": "kB",
}
metadata["bandwidth.kbytes.out"] = {
"label": "Kilobytes OUT per second",
"options": interfaces,
"status": status,
"error_message": msg,
"unit": "kB",
}
metadata["bandwidth.packets.in"] = {
"label": "Packets IN per second",
"options": interfaces,
"status": status,
"error_message": msg,
"unit": "packets",
}
metadata["bandwidth.packets.out"] = {
"label": "Packets OUT per second",
"options": interfaces,
"status": status,
"error_message": msg,
"unit": "packets",
}
return metadata
elif "sunos" in sys.platform:
if not agent_util.which("netstat"):
self.log.info("netstat not found")
status = agent_util.UNSUPPORTED
msg = "Please install netstat."
return {}
# Get the list of network devices
interfaces = []
ret, output = agent_util.execute_command("netstat -i")
output = output.strip().split("\n")
for line in output:
fields = line.split()
if not fields:
continue
if fields[0] in ("lo", "inet", "ether", "Name"):
continue
try:
add_to_interfaces(fields[0])
except:
pass
metadata = {}
metadata["bandwidth.kbytes.in"] = {
"label": "Kilobytes IN per second",
"options": interfaces,
"status": status,
"error_message": msg,
"unit": "kB",
}
metadata["bandwidth.kbytes.out"] = {
"label": "Kilobytes OUT per second",
"options": interfaces,
"status": status,
"error_message": msg,
"unit": "kB",
}
metadata["bandwidth.packets.in"] = {
"label": "Packets IN per second",
"options": interfaces,
"status": status,
"error_message": msg,
"unit": "packets",
}
metadata["bandwidth.packets.out"] = {
"label": "Packets OUT per second",
"options": interfaces,
"status": status,
"error_message": msg,
"unit": "packets",
}
return metadata
elif "vmware" in sys.platform:
if not agent_util.which("esxcli"):
self.log.info("esxcli not found")
status = agent_util.UNSUPPORTED
msg = "Please confirm esxcli is installed."
return {}
interfaces = []
ret, out = agent_util.execute_command("esxcli network nic list")
iface_table = out.split("\n")
for iface in iface_table:
# skip the headers, dividers and any empty items
if iface.startswith("Name") or iface.startswith("--") or iface == "":
continue
add_to_interfaces(iface.split()[0])
metadata = {}
metadata["bandwidth.kbytes.in"] = {
"label": "Kilobytes IN per second",
"options": interfaces,
"status": status,
"error_message": msg,
"unit": "kB",
}
metadata["bandwidth.kbytes.out"] = {
"label": "Kilobytes OUT per second",
"options": interfaces,
"status": status,
"error_message": msg,
"unit": "kB",
}
metadata["bandwidth.packets.in"] = {
"label": "Packets IN per second",
"options": interfaces,
"status": status,
"error_message": msg,
"unit": "packets",
}
metadata["bandwidth.packets.out"] = {
"label": "Packets OUT per second",
"options": interfaces,
"status": status,
"error_message": msg,
"unit": "packets",
}
return metadata
elif "hp-ux" in sys.platform:
ret, out = agent_util.execute_command("nwmgr -g")
iface_table = out.splitlines()
interfaces = []
for line in iface_table:
if not line.lower().startswith("lan"):
continue
iface = line.split()
if iface[1] != "UP":
continue
add_to_interfaces(iface[0])
metadata = {}
metadata["bandwidth.kbytes.in"] = {
"label": "Kilobytes IN per second",
"options": interfaces,
"status": status,
"error_message": msg,
"unit": "kB",
}
metadata["bandwidth.kbytes.out"] = {
"label": "Kilobytes OUT per second",
"options": interfaces,
"status": status,
"error_message": msg,
"unit": "kB",
}
metadata["bandwidth.packets.in"] = {
"label": "Packets IN per second",
"options": interfaces,
"status": status,
"error_message": msg,
"unit": "packets",
}
metadata["bandwidth.packets.out"] = {
"label": "Packets OUT per second",
"options": interfaces,
"status": status,
"error_message": msg,
"unit": "packets",
}
return metadata
else:
# Default Linux options
if not os.path.exists("/proc/net/dev"):
self.log.info("/proc/net/dev not found")
status = agent_util.UNSUPPORTED
msg = "/proc/net/dev not found"
return {}
if status is agent_util.SUPPORTED:
# get the interfaces
output = open("/proc/net/dev", "r").read()
output = output.splitlines()
if config.get("debug", False):
self.log.debug(
"#####################################################"
)
self.log.debug("Content of file '/proc/net/dev':")
self.log.debug(str(output))
self.log.debug(
"#####################################################"
)
for line in output[2:]:
stuff = line.strip().split()
iface, bytes_read = stuff[0].split(":")
add_to_interfaces(iface)
interfaces = list(interfaces)
interfaces.sort()
if status is agent_util.SUPPORTED and not interfaces:
status = agent_util.MISCONFIGURED
msg = "No network interfaces found."
metadata = {}
metadata["bandwidth.kbytes.in"] = {
"label": "Kilobytes IN per second",
"options": interfaces,
"status": status,
"error_message": msg,
"unit": "kB",
}
metadata["bandwidth.kbytes.out"] = {
"label": "Kilobytes OUT per second",
"options": interfaces,
"status": status,
"error_message": msg,
"unit": "kB",
}
metadata["bandwidth.packets.in"] = {
"label": "Packets IN per second",
"options": interfaces,
"status": status,
"error_message": msg,
"unit": "packets",
}
metadata["bandwidth.packets.out"] = {
"label": "Packets OUT per second",
"options": interfaces,
"status": status,
"error_message": msg,
"unit": "packets",
}
if "freebsd" not in sys.platform and "darwin" not in sys.platform:
metadata["bandwidth.monthly.in"] = {
"label": "Kilobytes IN for the month",
"options": interfaces,
"status": status,
"error_message": msg,
"unit": "kB",
}
metadata["bandwidth.monthly.out"] = {
"label": "Kilobytes OUT for the month",
"options": interfaces,
"status": status,
"error_message": msg,
"unit": "kB",
}
return metadata
def check(self, textkey, interface, config):
interface_found = False
if "freebsd" in sys.platform:
ret, output = agent_util.execute_command("netstat -ib")
lines = output.splitlines()[1:]
for line in lines:
if "lo" in line or line == "":
continue
line = line.split()
(
si,
mtu,
network,
addr,
packets_read,
ierr,
idrop,
bytes_read,
packets_written,
oerr,
bytes_written,
coll,
) = line
packets_read = int(packets_read)
packets_written = int(packets_written)
kbytes_read = int(bytes_read) / 1024
kbytes_written = int(bytes_written) / 1024
if si == interface and network.startswith("<Link"):
interface_found = True
break
elif "darwin" in sys.platform:
netstat = agent_util.which("netstat")
ret, output = agent_util.execute_command("%s -ibn" % netstat)
lines = output.splitlines()[1:]
for line in lines:
if "lo" in line or line == "":
continue
line = line.split()
si = line[0]
network = line[2]
packets_read = int(line[-7])
bytes_read = int(line[-5])
packets_written = int(line[-4])
bytes_written = int(line[-2])
kbytes_read = bytes_read / 1024
kbytes_written = bytes_written / 1024
if si == interface and network.startswith("<Link"):
interface_found = True
break
elif "aix" in sys.platform:
ret, output = agent_util.execute_command(
'netstat -v %s | grep "^Bytes:"' % interface
)
fields = output.split()
kbytes_written = int(fields[1]) / 1024
kbytes_read = int(fields[3]) / 1024
ret, output = agent_util.execute_command(
'netstat -v %s | grep "^Packets:"' % interface
)
fields = output.split()
packets_written = int(fields[1]) / 1024
packets_read = int(fields[3]) / 1024
interface_found = True
elif "sunos" in sys.platform:
ret, output = agent_util.execute_command("netstat -i")
for line in output.strip().split("\n"):
fields = line.strip().split()
if not fields:
continue
if fields[0] == interface:
interface_found = True
packets_read = int(fields[4])
packets_written = int(fields[6])
kbytes_read = 0
kbytes_written = 0
ret, output = agent_util.execute_command(
"kstat -n %s 1 2 | egrep 'bytes64' | uniq" % interface
)
for line in output.split("\n"):
if not line:
continue
if "obytes" in line:
fields = line.split()
kbytes_read = int(fields[1]) / 1024
elif "rbytes" in line:
fields = line.split()
kbytes_written = int(fields[1]) / 1024
elif "vmware" in sys.platform:
ret, out = agent_util.execute_command(
"esxcli network nic stats get -n %s" % interface
)
if ret == 0:
interface_found = True
iface_stats = {}
fields = out.split("\n")
for i in fields:
trans = i.strip().split(":")
if len(trans) != 2:
continue
iface_stats[trans[0]] = trans[1].strip()
kbytes_written = int(iface_stats["Bytes sent"]) / 1024
kbytes_read = int(iface_stats["Bytes received"]) / 1024
packets_written = int(iface_stats["Packets sent"])
packets_read = int(iface_stats["Packets received"])
elif "hp-ux" in sys.platform:
interface_found = True
# NOTE, we need to first divide by 8 to get bytes from octets
# then divide by 1024 to get KB
ret, out = agent_util.execute_command("nwmgr --st -c %s" % interface)
nwmgr = out.splitlines()
iface_stats = {}
for line in nwmgr:
if "=" not in line:
continue
l = line.split("=")
iface_stats[l[0].strip()] = l[1].strip()
self.log.debug(iface_stats)
kbytes_read = (float(iface_stats["Inbound Octets"]) / 8) / 1024
kbytes_written = (float(iface_stats["Outbound Octets"]) / 8) / 1024
packets_read = total_pkts = (
float(iface_stats["Inbound Unicast Packets"])
+ float(iface_stats["Inbound Multicast Packets"])
+ float(iface_stats["Inbound Broadcast Packets"])
)
packets_written = total_pkts = (
float(iface_stats["Outbound Unicast Packets"])
+ float(iface_stats["Outbound Multicast Packets"])
+ float(iface_stats["Outbound Broadcast Packets"])
)
else:
output = open("/proc/net/dev", "r").read()
self.log.debug("/proc/net/dev output: %s" % str(output))
output = output.splitlines()
for line in output[2:]:
stuff = line.strip().split()
if not stuff[0].endswith(":"):
stuff[0], new_insert = stuff[0].split(":")
stuff.insert(1, new_insert)
else:
stuff[0] = stuff[0].rstrip(":")
iface = stuff[0]
bytes_read = int(stuff[1])
bytes_written = int(stuff[9])
kbytes_read = int(bytes_read) / 1024
kbytes_written = int(stuff[9]) / 1024
packets_read = int(stuff[2])
packets_written = int(stuff[10])
if interface == iface:
interface_found = True
break
# Special handling for monthly bandwidth - each time through we bank the
# difference in the current reading vs. the previous, taking into account
# cases where we cross a month boundary and when the byte counters wrap
if textkey.startswith("bandwidth.monthly"):
if textkey == "bandwidth.monthly.in":
current_bytes = bytes_read
else:
current_bytes = bytes_written
# First get the cached values
c = self.get_cache_results(textkey + ":current_bytes", interface)
previous_bytes = c and c[0][1] or current_bytes
c = self.get_cache_results(textkey + ":banked_bytes", interface)
banked_bytes = c and c[0][1] or 0
c = self.get_cache_results(textkey + ":current_month", interface)
current_month = c and c[0][1] or datetime.now().month
c = self.get_cache_results(textkey + ":current_year", interface)
current_year = c and c[0][1] or datetime.now().year
now = datetime.now()
if now.year != current_year or now.month != current_month:
# If we"ve crossed a month boundary, zero out bank and reset counters
banked_bytes = 0
current_year = now.year
current_month = now.month
previous_bytes = current_bytes
elif current_bytes < previous_bytes:
# The OS counters wrapped, need to handle this
banked_bytes += current_bytes
previous_bytes = current_bytes
else:
# Standard case, just bank the difference between current and last
banked_bytes += current_bytes - previous_bytes
previous_bytes = current_bytes
# Cache the new values
self.cache_result(
textkey + ":current_bytes", interface, current_bytes, replace=True
)
self.cache_result(
textkey + ":banked_bytes", interface, banked_bytes, replace=True
)
self.cache_result(
textkey + ":current_month", interface, current_month, replace=True
)
self.cache_result(
textkey + ":current_year", interface, current_year, replace=True
)
return banked_bytes / 1024
# Because of AIX's interface naming convention, every
# command provides a different interface name, we need to ignore that
if "aix" in sys.platform:
pass
else:
if not interface_found:
self.log.warning("interface %s not found!?" % interface)
return None
# try to get the old cache
cache = self.get_cache_results(textkey, interface)
# cache our result always
if textkey == "bandwidth.kbytes.in":
result = kbytes_read
elif textkey == "bandwidth.kbytes.out":
result = kbytes_written
elif textkey == "bandwidth.packets.in":
result = packets_read
elif textkey == "bandwidth.packets.out":
result = packets_written
else:
self.log.error("UNKNOWN BANDWIDTH TEXTKEY- %s" % textkey)
self.cache_result(textkey, interface, result)
if not cache:
return 0.0
delta, cached_result = cache[0]
self.log.debug("kbytes read: %d" % kbytes_read)
self.log.debug("kbytes written: %d" % kbytes_written)
self.log.debug("packets read: %d" % packets_read)
self.log.debug("packets written: %d" % packets_written)
new_value = result - cached_result
if new_value < 0.0:
self.log.warning(
"Metric {} current {} less than cached {} - maybe reboot or counter overflow".format(
textkey, result, cached_result
)
)
return None
return new_value / float(delta)