Message collection pprint with PY:

[Jitoxxx]

Hello, I have got some settings running on 2 servers, they are both identical apart from one running 4.0.5 and the other one running 4.0.6 of exabgp.

Now the issue I am having is related to my log collector, the files are exactly the same for both versions: The only difference is that version 4.0.5 gets a pprint which is a normal format for tag rd, as followed: "rd": "XXXXX:XXXXX" The 4.0.6 gets a pprint which is as followd : "rd": "?\xfd\x91\x00\x00\xfd\xfa\x00". This stops my exabgp session and I can't log the files.

Script I'm using is as followed:

#!/usr/bin/env python
import sys
import socket
import json
from pprint import pprint

sock = socket.socket()
sock.connect( ('IP', 5150) )

def get_attrs(json_in):
    attrs = {}
    attrs['neighbor'] =     json_in['neighbor']['address']['peer']
    attrs['peeras'] =       json_in['neighbor']['asn']['peer']

    msgtype =               json_in['neighbor']['message'].keys()[0]
    attrs['msgtype'] = msgtype

    if 'announce' in        json_in['neighbor']['message'][msgtype].keys():
      action = 'announce'
    elif 'withdraw' in      json_in['neighbor']['message'][msgtype].keys():
      action = 'withdraw'
    else:
      print "funky action in this message, %s" % json_in
    attrs['action'] = action

    afisafi  =              json_in['neighbor']['message'][msgtype][action].keys()[0]
    attrs['afi'] = afisafi.split()[0]
    attrs['safi'] = afisafi.split()[1]

    if action == 'announce':
      nexthop =             json_in['neighbor']['message'][msgtype][action][afisafi].keys()[0]
      attrs['nexthop'] =      nexthop

      # simple mapping, but substitute hyphens since elastic doesn't like em
      simple_attributes = ['local-preference','origin','med','originator-id','cluster-list']
      for simple_attribute in simple_attributes:
        if json_in['neighbor']['message'][msgtype]['attribute'].has_key(simple_attribute):
          attrs[simple_attribute.replace('-','')] = json_in['neighbor']['message'][msgtype]['attribute'][simple_attribute]

      # as-path needs to be casted from list to string
      if json_in['neighbor']['message'][msgtype]['attribute'].has_key('as-path'):
        if type(json_in['neighbor']['message'][msgtype]['attribute']['as-path']) == type([]):
          attrs['aspath'] = ' '.join(str(x) for x in json_in['neighbor']['message'][msgtype]['attribute']['as-path'])
        else:
          attrs['aspath'] =   json_in['neighbor']['message'][msgtype]['attribute']['as-path']

      # ext comms are string/value pairs by exa, only keep strings
      if json_in['neighbor']['message'][msgtype]['attribute'].has_key('extended-community'):
        extcomms = []
        for extcomm in json_in['neighbor']['message'][msgtype]['attribute']['extended-community']:
          extcomms.append(extcomm['string'])
        attrs['extcomms'] = extcomms

      # regular comms are lists, rewrite to colon=separated
      if json_in['neighbor']['message'][msgtype]['attribute'].has_key('community'):
        comms = []
        for comm in json_in['neighbor']['message'][msgtype]['attribute']['community']:
          comms.append("%s:%s" % (comm[0],comm[1]))
        attrs['community'] = comms
    return attrs

def get_evpn_routes(announce_in):
    announce = {}
    announce['rd'] =           announce_in['rd']
    routetype =                announce_in['code']
    routetypename =            announce_in['name']
    announce['routetype'] =    routetype
    announce['routetypename'] =    routetypename
    if routetype == 1:
      announce['esi'] = announce_in['esi']
      announce['ethernettag'] = announce_in['ethernet-tag']
      announce['label'] = announce_in['label']
    elif routetype == 2:
      announce['esi'] = announce_in['esi']
      announce['mac'] = announce_in['mac']
      announce['ethernettag'] = announce_in['ethernet-tag']
      try:
        announce['ip'] = announce_in['ip']
      except KeyError:
        pass
      announce['label'] = announce_in['label']
    elif routetype == 3:
      announce['ethernettag'] = announce_in['ethernet-tag']
      announce['ip'] = announce_in['ip']
    elif routetype == 4:
      announce['ip'] = announce_in['ip']
      announce['esi'] = announce_in['esi']
    elif routetype == 5:
      pass
    else: # not supported
      pass
    return announce

def get_mpls_vpn_routes(announce_in):
    announce = {}
    announce['rd'] =           announce_in['rd']
    announce['label'] = announce_in['label']
    announce['prefix'] = announce_in['nlri']

    return announce

def get_unicast_routes(announce_in):
    announce = {}
    announce['prefix'] = announce_in['nlri']

    return announce

def get_flow_routes(announce_in):
    announce = {}
    announce['rule'] = announce_in['string']
    return announce

logfile = open('/tmp/transform.log','a')
counter = 0
while True:
    try:
        line = sys.stdin.readline().strip()
        if line == "":
            counter += 1
            if counter > 100:
                break
            continue
        counter = 0
        json_in = json.loads(line)
        json_out_list = []

        #logfile.write(line)
        attrs = get_attrs(json_in)
        attrs['orig'] = line # only when developing/troubleshooting
        #print "ATTRS:",attrs
        routes_to_parse = []
        if attrs['action'] == 'announce':
          routes_to_parse = json_in['neighbor']['message'][attrs['msgtype']][attrs['action']][attrs['afi'] + ' ' + attrs['safi']][attrs['nexthop']]
        elif attrs['action'] == 'withdraw':
          routes_to_parse = json_in['neighbor']['message'][attrs['msgtype']][attrs['action']][attrs['afi'] + ' ' + attrs['safi']]
        else:
          print "ERROR: we're not supposed to get here, funky action in ", line
        #print "RTP:", routes_to_parse
        for route in routes_to_parse:
          newroute = {}
          newroute.update(attrs)
          if attrs['safi'] == 'evpn':
            newroute.update(get_evpn_routes(route))
            json_newroute = json.dumps(newroute)
            json_out_list.append(json_newroute)
          elif attrs['safi'] == 'mpls-vpn':
            newroute.update(get_mpls_vpn_routes(route))
            json_newroute = json.dumps(newroute)
            json_out_list.append(json_newroute)
          elif attrs['safi'] == 'unicast':
            newroute.update(get_unicast_routes(route))
            json_newroute = json.dumps(newroute)
            json_out_list.append(json_newroute)
          elif attrs['safi'] == 'flow':
            newroute.update(get_flow_routes(route))
            json_newroute = json.dumps(newroute)
            json_out_list.append(json_newroute)
          else:
            logfile.write(line)
        #print "OUT LIST:",len(json_out_list), pprint(json_out_list)
        msg = '\n'.join(json_out_list)
        #print "MSG", type(msg),len(msg),msg
        sock.sendall(msg + '\n')
    except KeyboardInterrupt:
        sock.close()
#        logfile.close()
    except IOError:
        # most likely a signal during readline
        sock.close()
#        logfile.close()

sock.close()
#logfile.close()

I'm guessing there ain't anything wrong with the script, since I get the error befor it even goes through it. I'm checking further for errors but my environment is the same in both and is as followed:

[exabgp.api]
encoder = text
highres = false
respawn = false

[exabgp.bgp]
openwait = 60

[exabgp.cache]
attributes = true
nexthops = true

[exabgp.daemon]
pid = '/var/run/exabgp.pid'
user = 'root'

[exabgp.log]
all = false
configuration = true
daemon = true
destination = '/var/exalog/exa.log'
enable = true
level = INFO
message = false
network = true
packets = false
parser = false
processes = true
reactor = true
rib = false
routes = false
short = false
timers = false

[exabgp.pdb]
enable = false

[exabgp.profile]
enable = false

[exabgp.reactor]
speed = 1.0

[exabgp.tcp]
acl = false
delay = 0
once = false
port = 179

I run the following file with exabgp:

process parsed-route-backend {
        run /etc/exabgp/bin/transform-exabgp-json.py;
        encoder json;
}

template {
    neighbor IG {
        router-id SERVERIP;
        local-address SERVERIP;
        local-as xxx;
        peer-as xxx;
        hold-time 180;
        family {
                ipv4 unicast;
                ipv6 unicast;
                ipv4 flow;
                ipv6 flow;
        }
        api speaking1 {
            processes [ parsed-route-backend ];
            receive {
                parsed;
                update;
                #keepalive;
            }
        }
    }
    neighbor VPN-RR {
        router-id SERVERIP;
        local-address SERVERIP;
        local-as xxx;
        peer-as xxx;
        hold-time 180;
        family {
                 ipv4 mpls-vpn;
                 ipv6 mpls-vpn;
        }
        api speaking2 {
            processes [ parsed-route-backend ];
            receive {
                parsed;
                update;
                #keepalive;
            }
        }
    }
        neighbor EVPN-RR {
        router-id SERVERIP;
        local-address SERVERIP;
        local-as xxx;
        peer-as xxx;
        hold-time 180;
        family {
                l2vpn evpn;
                }
        api speaking3 {
            processes [ parsed-route-backend ];
            receive {
                parsed;
                update;
            }
        }
    }
}

neighbor IP {
        inherit IG;
}
neighbor IP {
        inherit IG;
}
neighbor IP {
        inherit IG;
}
neighbor IP {
        inherit EVPN-RR;
}
neighbor IP {
        inherit EVPN-RR;
}
neighbor IP {
        inherit VPN-RR;
}
neighbor IP{
        inherit VPN-RR;
}

This is the error I get for the VPN-RR ( the others work ):

13:46:21 | 31761  | welcome       | Thank you for using ExaBGP
13:46:21 | 31761  | version       | 4.0.6-daa3d6ba
13:46:21 | 31761  | interpreter   | 2.7.12 (default, Dec  4 2017, 14:50:18)  [GCC 5.4.0 20160609]
13:46:21 | 31761  | os            | Linux driven-urchin 4.4.0-124-generic #148-Ubuntu SMP Wed May 2 13:00:18 UTC 2018 x86_64
13:46:21 | 31761  | installation  | /usr/local
13:46:21 | 31761  | cli           | could not find the named pipes (exabgp.in and exabgp.out) required for the cli
13:46:21 | 31761  | cli           | we scanned the following folders (the number is your PID):
13:46:21 | 31761  | cli control   |  - /run/exabgp/
13:46:21 | 31761  | cli control   |  - /run/0/
13:46:21 | 31761  | cli control   |  - /run/
13:46:21 | 31761  | cli control   |  - /var/run/exabgp/
13:46:21 | 31761  | cli control   |  - /var/run/0/
13:46:21 | 31761  | cli control   |  - /var/run/
13:46:21 | 31761  | cli control   |  - /usr/local/run/exabgp/
13:46:21 | 31761  | cli control   |  - /usr/local/run/0/
13:46:21 | 31761  | cli control   |  - /usr/local/run/
13:46:21 | 31761  | cli control   |  - /usr/local/var/run/exabgp/
13:46:21 | 31761  | cli control   |  - /usr/local/var/run/0/
13:46:21 | 31761  | cli control   |  - /usr/local/var/run/
13:46:21 | 31761  | cli control   | please make them in one of the folder with the following commands:
13:46:21 | 31761  | cli control   | > mkfifo /root/run/exabgp.{in,out}
13:46:21 | 31761  | cli control   | > chmod 600 /root/run/exabgp.{in,out}
13:46:21 | 31761  | configuration | performing reload of exabgp 4.0.6-daa3d6ba
13:46:21 | 31761  | configuration | > process          | 'parsed-route-backend'
13:46:21 | 31761  | configuration | . run              | '/etc/exabgp/bin/transform-exabgp-json.py'
13:46:21 | 31761  | configuration | . encoder          | 'json'
13:46:21 | 31761  | reactor       | loaded new configuration successfully
13:46:21 | 31761  | process       | forked process parsed-route-backend
Traceback (most recent call last):
  File "/etc/exabgp/bin/transform-exabgp-json.py", line 126, in <module>
    json_in = json.loads(line)
  File "/usr/lib/python2.7/json/__init__.py", line 339, in loads
    return _default_decoder.decode(s)
  File "/usr/lib/python2.7/json/decoder.py", line 364, in decode
    obj, end = self.raw_decode(s, idx=_w(s, 0).end())
  File "/usr/lib/python2.7/json/decoder.py", line 380, in raw_decode
    obj, end = self.scan_once(s, idx)
ValueError: Invalid control character at: line 1 column 995 (char 994)

this is what char 994 looks like:

"rd": "?\xfd\x91\x00\x00\xfd\xfa\x00"

and it should be (X=digit 0-9):

"rd": "XXXXX:XXXXX"

[thomas-mangin]

"rd": "?\xfd\x91\x00\x00\xfd\xfa\x00" means that the first short of the RD is 16381.

> unpack('!H',"?\xfd")
(16381,)

Therefore the following code use the string representation of the binary of what looks like AFAIK an invalid RD ( https://en.wikipedia.org/wiki/Route_distinguisher )

    def _str (self):
        t,c1,c2,c3 = unpack('!HHHH',self.rd)
        if t == 0:
            rd = '%d:%d' % (c1,(c2 << 16)+c3)
        elif t == 1:
            rd = '%d.%d.%d.%d:%d' % (c1 >> 8,c1 & 0xFF,c2 >> 8,c2 & 0xFF,c3)
        elif t == 2:
            rd = '%d:%d' % ((c1 << 16) + c2,c3)
        else:
            rd = str(self.rd)
        return rd

which indeed does not generate some valid JSON.

>>> s = """{"rd": "?\xfd\x91\x00\x00\xfd\xfa\x00"}"""
>>> json.loads(s)
Traceback (most recent call last):

It was changed to be an hex string of the value.

>>> hexstring("?\xfd\x91\x00\x00\xfd\xfa\x00")
'0x3FFD910000FDFA00'