Open hanynowsky opened 9 years ago
def masac_poller(device_type, timeout, retries, sqlitedbtable, banned):
""" Fetch SNMP metrics using fastsnmp module and send them to Carbon server """
# oids in group must be with same indexes
node = platform.node().replace('.', '-')
tags = ("Vlan","unrouted","thernet")
oid_group = {"1.3.6.1.2.1.2.2.1.2": "ifDescr", "1.3.6.1.2.1.2.2.1.10": "ifInOctets","1.3.6.1.2.1.2.2.1.16": "ifOutOctets",}
count = 0
for h in get_db_device_list(sqlitedbtable, device_type, banned):
timestamp = int(time.time())
hostnames = []
hostnames.append(h['ip'])
snmp_data = snmp_poller.poller(hostnames, (oid_group.keys(),), h['snmp_community'], int(timeout), int(retries))
if len(list(snmp_data)) < 1 or h['snmp_community'] == '':
update_device_in_db(sqlitedbtable, h['login'], device_type, 1)
print("Banned " + h['ip'])
continue
snmp_data = snmp_poller.poller(hostnames, (oid_group.keys(),), h['snmp_community'], int(timeout), int(retries))
e = {}
lines= []
for d in snmp_data:
metrics = []
oid = oid_group[d[1]]
host = d[0]
index = d[2]
value = d[3]
e['hostname'] = host
if "ifDescr" in oid:
e['ifDescr'] = value
e['index'] = index
if "ifInOctets" in oid:
pline = ("host=%s oid=%s.%s value=%s" % (host, oid, index, value))
e['ifHCInOctets'] = value
e['index'] = index
if "ifOutOctets" in oid:
e['ifHCOutOctets'] = value
e['index'] = index
metrics.insert(int(e['index']), e)
for device in metrics:
if len(device) == 5:
identity = h['login'].replace(".","_").replace("@","_at_")
if "sdsl" in device_type:
identity = str(h['hostname'].replace(".","_").replace("@","_at_")) + "_" + str(h['ip'])
for metric in ("ifHCInOctets","ifHCOutOctets"):
line = '%s.%s.%s.%s.%s %d %d' % (METRIC_PREFIX, device_type, identity, device['ifDescr'].replace("/","_").replace(" ","_"), metric, device[metric], timestamp)
if tags[0] not in device['ifDescr'] and tags[1] not in device['ifDescr'] and tags[2] in device['ifDescr']:
lines.append(line)
message = '\n'.join(lines) + '\n'
sock = socket.socket()
sock.connect((CARBON_SERVER, CARBON_PORT))
print(message)
sock.sendall(message.encode('UTF-8'))
sock.close()
hostnames.pop() # Pop the hostnames after fetching needed metrics
return count
Check this example https://github.com/gescheit/fastsnmp/blob/master/examples/graphite_sender.py
@gescheit Real thanks. I am going to check that soon and feedback. By the first read, I can tell this is some real python programming.
Hi, gesheit,
Do you have a benchmark for fastsnmp? Don't have to be accurate. For example, to poll one OID, how many hosts can it poll per second? 100, 1K, etc. To poll 4 OIDS, how many host can it poll per second?
Thanks
Hi! Main bottleneck of performance is decoding of SNMP-payload. My laptop is able to decode about 30k OIDs per second on localhost tests on 1 CPU. But in real life it must be considered with network parameters like delay, limited passthrough and packet-drops. Also network equipment typically have weak CPU and can answer very slowly or doesn't answer if it is on high load. So SNMP-poller must wait answer, resend query in case of timeout. Direct answer is next: in ideal world my laptop can pool 1k hosts with 4 OIDs in 1 second:) In real world, for example, 1k host would be polled in 1-30 seconds, 10k in 3-30 seconds, 100k 15-45...
Thank you for your answer.
I am very interested in your work. I may give it a try.
From: gescheit notifications@github.com<mailto:notifications@github.com> Reply-To: gescheit/fastsnmp reply@reply.github.com<mailto:reply@reply.github.com> Date: Monday, October 19, 2015 at 5:53 PM To: gescheit/fastsnmp fastsnmp@noreply.github.com<mailto:fastsnmp@noreply.github.com> Cc: Henry Lu henry_lu@cable.comcast.com<mailto:henry_lu@cable.comcast.com> Subject: Re: [fastsnmp] How to gather metrics with less complexity? (#2)
Hi! Main bottleneck of performance is decoding of SNMP-payload. My laptop is able to decode about 30k OIDs per second on localhost tests on 1 CPU. But in real life it must be considered with network parameters like delay, limited passthrough and packet-drops. Also network equipment typically have weak CPU and can answer very slowly or doesn't answer if it is on high load. So SNMP-poller must wait answer, resend query in case of timeout. Direct answer is next: in ideal world my laptop can pool 1k hosts with 4 OIDs in 1 second:) In real world, for example, 1k host would be polled in 1-30 seconds, 10k in 3-30 seconds.
Reply to this email directly or view it on GitHubhttps://github.com/gescheit/fastsnmp/issues/2#issuecomment-149358899.
Hello, As you can see in the following example, it's very expensive to get metrics with less complexity.