Castome

[rubenpapovyan]

Hi, I created custom monitor to collect data from network connection count by types. See below Execution of this script taking about 1,2 seconds. But diamond using one trade 100% for 20 seconds Please tell me whats wrong

#!/usr/bin/python
from decimal import *
import commands

import diamond.collector
class ConnectionCounterCollector(diamond.collector.Collector):
        def collect(self):

            all_connections = commands.getoutput("netstat -ant")

         connection_list = {"ESTABLISHED" : 0 , "SYN_SENT" : 0 ,  "SYN_RECV" : 0 ,  "FIN_WAIT1" : 0 ,  "FIN_WAIT2" : 0 ,  "TIME_WAIT" : 0 , "CLOSE" : 0 , "CLOSE_WAIT" : 0 , "LAST_ACK" : 0 , "LISTEN" : 0 , "CLOSING" : 0 , "UNKNOWN" : 0 }

            for line in all_connections.splitlines():
                #print line
                if line.find("ESTABLISHED") != -1:
                    #print "ESTABLISHED"
                    connection_list["ESTABLISHED"]+=1

                elif line.find("SYN_SENT") != -1:
                    #print "SYN_SENT"
                    connection_list["SYN_SENT"]+=1

                elif line.find("SYN_RECV") != -1:
                    #print "SYN_RECV"
                    connection_list["SYN_RECV"]+=1

                elif line.find("FIN_WAIT1") != -1:
                    #print "FIN_WAIT1"
                    connection_list["FIN_WAIT1"]+=1

                elif line.find("FIN_WAIT2") != -1:
                    #print "FIN_WAIT2"
                    connection_list["FIN_WAIT2"]+=1

                elif line.find("TIME_WAIT") != -1:
                    #print "TIME_WAIT"
                    connection_list["TIME_WAIT"]+=1

                elif line.find("CLOSE") != -1:
                    #print "CLOSE"
                    connection_list["CLOSE"]+=1

                elif line.find("CLOSE_WAIT") != -1:
                    #print "CLOSE_WAIT"
                    connection_list["CLOSE_WAIT"]+=1

                elif line.find("LAST_ACK") != -1:
                    #print "LAST_ACK"
                    connection_list["LAST_ACK"]+=1

                elif line.find("LISTEN") != -1:
                    #print "LISTEN"
                    connection_list["LISTEN"]+=1

                elif line.find("CLOSING") != -1:
                    #print "CLOSING"
                    connection_list["CLOSING"]+=1

                elif line.find("UNKNOWN") != -1:
                    #print "UNKNOWN"
                    connection_list["UNKNOWN"]+=1

                # PUSH METRICS
                self.publish("ESTABLISHED" , connection_list["ESTABLISHED"])
                self.publish("SYN_SENT"    , connection_list["SYN_SENT"]   )
                self.publish("SYN_RECV"    , connection_list["SYN_RECV"]   )
                self.publish("FIN_WAIT1"   , connection_list["FIN_WAIT1"]  )
                self.publish("FIN_WAIT2"   , connection_list["FIN_WAIT2"]  )
                self.publish("TIME_WAIT"   , connection_list["TIME_WAIT"]  )
                self.publish("CLOSE"       , connection_list["CLOSE"]      )
                self.publish("CLOSE_WAIT"  , connection_list["CLOSE_WAIT"] )
                self.publish("LAST_ACK"    , connection_list["LAST_ACK"]   )
                self.publish("LISTEN"      , connection_list["LISTEN"]     )
                self.publish("CLOSING"     , connection_list["CLOSING"]    )
                self.publish("UNKNOWN"     , connection_list["UNKNOWN"]    )

[jaingaurav]

What do you mean by "one trade 100% for 20 seconds"? Are you saying that diamond occasionally takes 20 seconds? My guess is the netstat -ant is taking a long time. How many connections are you typically working with?

[jaingaurav]

Also please note that development of diamond has moved to https://github.com/python-diamond/Diamond

[rubenpapovyan]

Im executing this script it takes 2 sec When i'm putting same same code to dimond class it using CPU a lot

#!/usr/bin/python
from decimal import *
import commands

import diamond.collector

all_connections = commands.getoutput("netstat -ant")

connection_list = {"ESTABLISHED" : 0 , "SYN_SENT" : 0 ,  "SYN_RECV" : 0 ,  "FIN_WAIT1" : 0 ,  "FIN_WAIT2" : 0 ,  "TIME_WAIT" : 0 , "CLOSE" : 0 , "CLOSE_WAIT" : 0 , "LAST_ACK" : 0 , "LISTEN" : 0 , "CLOSING" : 0 , "UNKNOWN" : 0 }

for line in all_connections.splitlines():
    #print line
    if line.find("ESTABLISHED") != -1:
        #print "ESTABLISHED"
        connection_list["ESTABLISHED"]+=1

    elif line.find("SYN_SENT") != -1:
        #print "SYN_SENT"
        connection_list["SYN_SENT"]+=1

    elif line.find("SYN_RECV") != -1:
        #print "SYN_RECV"
        connection_list["SYN_RECV"]+=1

    elif line.find("FIN_WAIT1") != -1:
        #print "FIN_WAIT1"
        connection_list["FIN_WAIT1"]+=1

    elif line.find("FIN_WAIT2") != -1:
        #print "FIN_WAIT2"
        connection_list["FIN_WAIT2"]+=1

    elif line.find("TIME_WAIT") != -1:
        #print "TIME_WAIT"
        connection_list["TIME_WAIT"]+=1

    elif line.find("CLOSE") != -1:
        #print "CLOSE"
        connection_list["CLOSE"]+=1

    elif line.find("CLOSE_WAIT") != -1:
        #print "CLOSE_WAIT"
        connection_list["CLOSE_WAIT"]+=1

    elif line.find("LAST_ACK") != -1:
        #print "LAST_ACK"
        connection_list["LAST_ACK"]+=1

    elif line.find("LISTEN") != -1:
        #print "LISTEN"
        connection_list["LISTEN"]+=1

    elif line.find("CLOSING") != -1:
        #print "CLOSING"
        connection_list["CLOSING"]+=1

    elif line.find("UNKNOWN") != -1:
        #print "UNKNOWN"
        connection_list["UNKNOWN"]+=1

print "ESTABLISHED : " , connection_list["ESTABLISHED"]
print "SYN_SENT    : " , connection_list["SYN_SENT"]   
print "SYN_RECV    : " , connection_list["SYN_RECV"]   
print "FIN_WAIT1   : " , connection_list["FIN_WAIT1"]  
print "FIN_WAIT2   : " , connection_list["FIN_WAIT2"]  
print "TIME_WAIT   : " , connection_list["TIME_WAIT"]  
print "CLOSE       : " , connection_list["CLOSE"]      
print "CLOSE_WAIT  : " , connection_list["CLOSE_WAIT"] 
print "LAST_ACK    : " , connection_list["LAST_ACK"]   
print "LISTEN      : " , connection_list["LISTEN"]     
print "CLOSING     : " , connection_list["CLOSING"]    
print "UNKNOWN     : " , connection_list["UNKNOWN"] 

[jaingaurav]

@rubenpapovyan there really shouldn't be any reason why diamond would use more resources than a standalone script. You might have to paste your entire config for a better understanding. How are you integrating your script? As a custom collector or as a script in the UserScriptCollector?

[kormoc]

This repo has moved to https://github.com/python-diamond/Diamond

Please open a new ticket there if this issue is not resolved with the current code there.

Thanks!