Closed bvivi57 closed 3 years ago
Hi, @bvivi57
Please share your /etc/nfcol/nfc.yaml
configuration file, influxdb log and also output of this command show databases
in your InfluxDB Shell
Hi, Thank for your help
This is my conf file
# # # # # # # # # # # # # # # # # #
# Netflow Collector #
# Configuration #
# # # # # # # # # # # # # # # # # #
server:
# Listen Address
address: 0.0.0.0
# Listen UDP Port
port: 6859
# If true, nfc will write flow data into stdout
dump: false
# Activate forwarder
forwarder: false
# Host to forward - Can be separated by ; (semi-colon) eg: 127.0.0.1;192.168.100.1
forwarderHost: 127.0.0.1
# Forwarder UDP Port
forwarderPort: 7161
# # # # # # # # # # # # # # # # # #
# IP2Location Configuration #
# # # # # # # # # # # # # # # # # #
ip2location:
# IP2Location command path
cmd: /usr/local/bin/ip2location
# Path to Local GEO Database. Read more at:
# https://github.com/javadmohebbi/IP2Location#local-database-format
local: /etc/ip2location/local.csv
# # # # # # # # # # # # # # # # # #
# Exporter Configuration #
# # # # # # # # # # # # # # # # # #
exporter:
# Enable if it's true
enable: true
# Currently Only InfluxDB (1.x) supported
type: influxdb
# # # # # # # # # # # # # # # # # #
# InfluxDB Configuration #
# # # # # # # # # # # # # # # # # #
influxDB:
# InfluxDB Host
host: localhost
# InfluxDB Port
port: 8086
# InfluxDB Username. Can be null
username: user_sup_netflow
# InfluxDB Password. Can be null
password: mypassword
# InfluxDB Database
# InfluxDB command example for creating database:
# CREATE DATABASE "netflowDB" WITH DURATION 10d REPLICATION 1 SHARD DURATION 1h NAME "nfc"
database: netflowDB
# Temp Dir for InfluxDB Metrics. MUST be ended with / (Linux) or \ (Windows)
tmpDir: /tmp/nfcol/
# # # # # # # # # # # # # # # # # #
# Measurements #
# # # # # # # # # # # # # # # # # #
measurements:
# Netflow Summary Measurement Name
summaryProto: sum_proto
# Netflow GEO Summary Measurement Name
summaryProtoGeo: sum_proto_geo
This is an extract influxdblog
Aug 03 11:14:00 infsrv001 influxd[862]: [httpd] 192.168.10.40 - jeedom_usr [03/Aug/2020:11:14:00 +0200] "POST /write?db=jeedom_db HTTP/1.1" 204 0 "-" "Telegraf/1.14.4" aaad975b-d569-11ea-8729-005056903494 4079
Aug 03 11:14:06 infsrv001 influxd[862]: [httpd] 127.0.0.1 - user_sup_netflow [03/Aug/2020:11:14:06 +0200] "POST /write?consistency=&db=netflowDB&precision=ns&rp= HTTP/1.1" 204 0 "-" "InfluxDBClient" ae417dd5-d569-11ea-872a-005056903494 110
Aug 03 11:14:06 infsrv001 influxd[862]: [httpd] 127.0.0.1 - user_sup_netflow [03/Aug/2020:11:14:06 +0200] "POST /write?consistency=&db=netflowDB&precision=ns&rp= HTTP/1.1" 204 0 "-" "InfluxDBClient" ae418df6-d569-11ea-872b-005056903494 58
Aug 03 11:14:10 infsrv001 influxd[862]: [httpd] 192.168.10.40 - jeedom_usr [03/Aug/2020:11:14:10 +0200] "POST /write?db=jeedom_db HTTP/1.1" 204 0 "-" "Telegraf/1.14.4" b0a45e65-d569-11ea-872c-005056903494 8261
Aug 03 11:14:20 infsrv001 influxd[862]: [httpd] 192.168.10.40 - jeedom_usr [03/Aug/2020:11:14:20 +0200] "POST /write?db=jeedom_db HTTP/1.1" 204 0 "-" "Telegraf/1.14.4" b69969d6-d569-11ea-872d-005056903494 4109
Aug 03 11:14:30 infsrv001 influxd[862]: [httpd] 192.168.10.40 - jeedom_usr [03/Aug/2020:11:14:30 +0200] "POST /write?db=jeedom_db HTTP/1.1" 204 0 "-" "Telegraf/1.14.4" bc8f402b-d569-11ea-872e-005056903494 6769
Aug 03 11:14:40 infsrv001 influxd[862]: [httpd] 192.168.10.40 - jeedom_usr [03/Aug/2020:11:14:40 +0200] "POST /write?db=jeedom_db HTTP/1.1" 204 0 "-" "Telegraf/1.14.4" c28522e4-d569-11ea-872f-005056903494 2807
Aug 03 11:14:50 infsrv001 influxd[862]: [httpd] 192.168.10.40 - jeedom_usr [03/Aug/2020:11:14:50 +0200] "POST /write?db=jeedom_db HTTP/1.1" 204 0 "-" "Telegraf/1.14.4" c87affc2-d569-11ea-8730-005056903494 3393
Aug 03 11:15:00 infsrv001 influxd[862]: [httpd] 192.168.10.40 - jeedom_usr [03/Aug/2020:11:15:00 +0200] "POST /write?db=jeedom_db HTTP/1.1" 204 0 "-" "Telegraf/1.14.4" ce70e803-d569-11ea-8731-005056903494 6443
Aug 03 11:15:06 infsrv001 influxd[862]: [httpd] 127.0.0.1 - user_sup_netflow [03/Aug/2020:11:15:06 +0200] "POST /write?consistency=&db=netflowDB&precision=ns&rp= HTTP/1.1" 204 0 "-" "InfluxDBClient" d20502d7-d569-11ea-8732-005056903494 132
Aug 03 11:15:06 infsrv001 influxd[862]: [httpd] 127.0.0.1 - user_sup_netflow [03/Aug/2020:11:15:06 +0200] "POST /write?consistency=&db=netflowDB&precision=ns&rp= HTTP/1.1" 204 0 "-" "InfluxDBClient" d2050d81-d569-11ea-8733-005056903494 44
Aug 03 11:15:06 infsrv001 influxd[862]: [httpd] 192.168.10.40 - jeedom_usr [03/Aug/2020:11:15:06 +0200] "POST /write?db=jeedom_db HTTP/1.1" 204 0 "-" "python-requests/2.21.0" d2352150-d569-11ea-8734-005056903494 925
Aug 03 11:15:10 infsrv001 influxd[862]: [httpd] 192.168.10.40 - jeedom_usr [03/Aug/2020:11:15:10 +0200] "POST /write?db=jeedom_db HTTP/1.1" 204 0 "-" "Telegraf/1.14.4" d466c068-d569-11ea-8735-005056903494 3424
Aug 03 11:15:20 infsrv001 influxd[862]: [httpd] 192.168.10.40 - jeedom_usr [03/Aug/2020:11:15:20 +0200] "POST /write?db=jeedom_db HTTP/1.1" 204 0 "-" "Telegraf/1.14.4" da5cab6b-d569-11ea-8736-005056903494 3368
Aug 03 11:15:30 infsrv001 influxd[862]: [httpd] 192.168.10.40 - jeedom_usr [03/Aug/2020:11:15:30 +0200] "POST /write?db=jeedom_db HTTP/1.1" 204 0 "-" "Telegraf/1.14.4" e0529ded-d569-11ea-8737-005056903494 4356
Aug 03 11:15:40 infsrv001 influxd[862]: [httpd] 192.168.10.40 - jeedom_usr [03/Aug/2020:11:15:40 +0200] "POST /write?db=jeedom_db HTTP/1.1" 204 0 "-" "Telegraf/1.14.4" e6486393-d569-11ea-8738-005056903494 3689
Aug 03 11:15:50 infsrv001 influxd[862]: [httpd] 192.168.10.40 - jeedom_usr [03/Aug/2020:11:15:50 +0200] "POST /write?db=jeedom_db HTTP/1.1" 204 0 "-" "Telegraf/1.14.4" ec3e5343-d569-11ea-8739-005056903494 4216
Aug 03 11:16:00 infsrv001 influxd[862]: [httpd] 192.168.10.40 - jeedom_usr [03/Aug/2020:11:16:00 +0200] "POST /write?db=jeedom_db HTTP/1.1" 204 0 "-" "Telegraf/1.14.4" f2345956-d569-11ea-873a-005056903494 18777
Aug 03 11:16:06 infsrv001 influxd[862]: [httpd] 127.0.0.1 - user_sup_netflow [03/Aug/2020:11:16:06 +0200] "POST /write?consistency=&db=netflowDB&precision=ns&rp= HTTP/1.1" 204 0 "-" "InfluxDBClient" f5c85f26-d569-11ea-873b-005056903494 96
Aug 03 11:16:06 infsrv001 influxd[862]: [httpd] 127.0.0.1 - user_sup_netflow [03/Aug/2020:11:16:06 +0200] "POST /write?consistency=&db=netflowDB&precision=ns&rp= HTTP/1.1" 204 0 "-" "InfluxDBClient" f5c868ae-d569-11ea-873c-005056903494 45
Aug 03 11:16:10 infsrv001 influxd[862]: [httpd] 192.168.10.40 - jeedom_usr [03/Aug/2020:11:16:10 +0200] "POST /write?db=jeedom_db HTTP/1.1" 204 0 "-" "Telegraf/1.14.4" f82a0520-d569-11ea-873d-005056903494 3519
Aug 03 11:16:20 infsrv001 influxd[862]: [httpd] 192.168.10.40 - jeedom_usr [03/Aug/2020:11:16:20 +0200] "POST /write?db=jeedom_db HTTP/1.1" 204 0 "-" "Telegraf/1.14.4" fe1ff637-d569-11ea-873e-005056903494 4021
This is the output "SHOW DATABASES"
influx -execute "SHOW DATABASES"
name: databases
name
----
_internal
sup_database
sup_infludb
jeedom_db
sup_netflow
netflowDB
Hi @bvivi57
It seems that there is no problem writing metrics to your InfluxDB database (netflowDB)
Aug 03 11:15:06 infsrv001 influxd[862]: [httpd] 127.0.0.1 - user_sup_netflow [03/Aug/2020:11:15:06 +0200] "POST /write?consistency=&db=netflowDB&precision=ns&rp= HTTP/1.1" 204 0 "-" "InfluxDBClient" d20502d7-d569-11ea-8732-005056903494 132
Aug 03 11:15:06 infsrv001 influxd[862]: [httpd] 127.0.0.1 - user_sup_netflow [03/Aug/2020:11:15:06 +0200] "POST /write?consistency=&db=netflowDB&precision=ns&rp= HTTP/1.1" 204 0 "-" "InfluxDBClient" d2050d81-d569-11ea-8733-005056903494 44
The above line and similar lines show that nfcol is writing metrics well. see 204 HTTP response.
Please run these command in your InfluxDB shell and send back the result:
use netflowDB
show measurements
If you can see sum_proto, then run this command and let me know the result
show series from sum_proto limit 10
Hi,
Thanks for your help.
Here is the return of the command 👍
use netflowDB
show measurements
-> empty result
show series from sum_proto limit 10
-> empty result
I have this same exact issue
Same issue.
same issue here
@dsielert @aaemon @kagareu Next version is under development, all the bugs will fix. Maybe next month ;)
@javadmohebbi first, thank you for building and sharing this cool app. It is really helpful.
I have the same issue reported here... However:
nfcol
but metrics are always empty. My problem is that when running inside Docker, the app wont create the job files on /tmp/nfcol/sum_proto/
nor on /tmp/nfcol/sum_proto_geo/
. I have configured the path to be writable but the process wont write to the folder.
Any clues?
Edit; I noticed that:
2021/04/01 04:35:47 Server is listening on [::]:2055
2021/04/01 04:35:47 Export job started
2021/04/01 04:36:07 decoder error: EOF
2021/04/01 04:36:07 decoder error: EOF
Tcpdump output:
04:35:17.562328 IP 192.168.1.1.41019 > 192.168.1.6.2055: UDP, length 504
04:36:07.659587 IP 192.168.1.1.36516 > 192.168.1.6.2055: UDP, length 504
04:36:07.659665 IP 192.168.1.1.36516 > 192.168.1.6.2055: UDP, length 204
nfcol-dump was running but did not log anything.
Edit2;
About the container, it was a port mapping issue. A typo! I used one port in the configs but exposed another in the container, so no data inbound. Ever. Fixing the port mapping was enough for everything to work.
About the decoder error: EOF, I noticed that they frequent in netflow v9, while in v10 (IPFIX) and v5, they are not. So I have 2 devices sending netflow, one in v10 and another in v5, so far no decoder error: EOF
error.
One thing that took me some time in the troubleshoot was the inability to see in the logs that the application was not receiving any netflow data (due to my port mapping error). Maybe other people have other problems, but they get the same symptom:
success
messages to show in InfluxDB log mentioning the app user login in the correct database. Bad: Temporary directories are empty.
Maybe you could implement a message in the logs for jobs to communicate when no files were found.
So thank you again for this code. I managed to install it on OpenShift, and it is now working just fine :)
Just to say I have got this version working using Opnsense as the netflow source. The only issue is that it will not work with netflow v9, only v5. Opensense only provides v5 or v9 That said, it was pretty easy to get working, compact and efficient compared to all the others I have tried, which tend to be complex and resource hungry. Current testing on a Debian Buster vm, uses 2Gb memory, with 50% cpu (4 threads), hypervisor xcp-ng. Installed using the deb package.
Thank you for your effort in producing this useful tool.
I look forward to trying out the new version once it it ready.
Latest Now getting Netflow V9 from opnsense, this appeared to be fixed in the latest update to opnsense.
Just to say I have got this version working using Opnsense as the netflow source. The only issue is that it will not work with netflow v9, only v5. Opensense only provides v5 or v9 That said, it was pretty easy to get working, compact and efficient compared to all the others I have tried, which tend to be complex and resource hungry. Current testing on a Debian Buster vm, uses 2Gb memory, with 50% cpu (4 threads), hypervisor xcp-ng. Installed using the deb package.
Thank you for your effort in producing this useful tool.
I look forward to trying out the new version once it it ready.
Dear @aimdev,
Thank you for your kind feedback. The new version is almost ready. Next week you can try it.
Hi,
I try to use nfCollector on this plateform
Unfortunately, while I'm encountering no errors, I have no data in InfluxDB. The database remains empty, while the logs indicate that nfcol is sending information.
Jul 31 08:25:04 infsrv001 nfcol[574080]: 2020/07/31 08:25:04 File nfc-20200731-08-23-(sum_proto_geo).metrics.working.done deteled Jul 31 08:25:06 infsrv001 nfcol[574080]: 2020/07/31 08:25:06 Reading directory /tmp/nfcol/sum_proto/ Jul 31 08:25:06 infsrv001 nfcol[574080]: 2020/07/31 08:25:06 Reading metrics nfc-20200731-08-24-(sum_proto).metrics Jul 31 08:25:06 infsrv001 nfcol[574080]: 2020/07/31 08:25:06 Reading directory /tmp/nfcol/sum_proto_geo/ Jul 31 08:25:06 infsrv001 nfcol[574080]: 2020/07/31 08:25:06 Reading metrics nfc-20200731-08-24-(sum_proto_geo).metrics Jul 31 08:25:06 infsrv001 nfcol[574080]: 2020/07/31 08:25:06 Adding .done extension to .metrics in directory: /tmp/nfcol/sum_proto_geo/ Jul 31 08:25:06 infsrv001 nfcol[574080]: 2020/07/31 08:25:06 File nfc-20200731-08-24-(sum_proto_geo).metrics.working renamed to nfc-20200731-08-24-(sum_proto_geo).metrics.working.done Jul 31 08:25:06 infsrv001 nfcol[574080]: 2020/07/31 08:25:06 Job for measurement (sum_proto_geo) Finished! Jul 31 08:25:06 infsrv001 nfcol[574080]: 2020/07/31 08:25:06 Adding .done extension to .metrics in directory: /tmp/nfcol/sum_proto/ Jul 31 08:25:06 infsrv001 nfcol[574080]: 2020/07/31 08:25:06 File nfc-20200731-08-24-(sum_proto).metrics.working renamed to nfc-20200731-08-24-(sum_proto).metrics.working.done Jul 31 08:25:06 infsrv001 nfcol[574080]: 2020/07/31 08:25:06 Job for measurement (sum_proto) Finished! Jul 31 08:26:04 infsrv001 nfcol[574080]: 2020/07/31 08:26:04 Export job configs to send metrics to InfluxDB: localhost:8086 (Database: netflowDB) Jul 31 08:26:04 infsrv001 nfcol[574080]: 2020/07/31 08:26:04 Job for measurement (sum_proto) Started! Jul 31 08:26:04 infsrv001 nfcol[574080]: 2020/07/31 08:26:04 Deleting .done extension & rewrite suspended .working files in directory: /tmp/nfcol/sum_proto/ Jul 31 08:26:04 infsrv001 nfcol[574080]: 2020/07/31 08:26:04 File nfc-20200731-08-24-(sum_proto).metrics.working.done deteled Jul 31 08:26:04 infsrv001 nfcol[574080]: 2020/07/31 08:26:04 Job for measurement (sum_proto_geo) Started! Jul 31 08:26:04 infsrv001 nfcol[574080]: 2020/07/31 08:26:04 Deleting .done extension & rewrite suspended .working files in directory: /tmp/nfcol/sum_proto_geo/ Jul 31 08:26:04 infsrv001 nfcol[574080]: 2020/07/31 08:26:04 File nfc-20200731-08-24-(sum_proto_geo).metrics.working.done deteled Jul 31 08:26:06 infsrv001 nfcol[574080]: 2020/07/31 08:26:06 Reading directory /tmp/nfcol/sum_proto/ Jul 31 08:26:06 infsrv001 nfcol[574080]: 2020/07/31 08:26:06 Reading metrics nfc-20200731-08-25-(sum_proto).metrics Jul 31 08:26:06 infsrv001 nfcol[574080]: 2020/07/31 08:26:06 Reading directory /tmp/nfcol/sum_proto_geo/ Jul 31 08:26:06 infsrv001 nfcol[574080]: 2020/07/31 08:26:06 Reading metrics nfc-20200731-08-25-(sum_proto_geo).metrics Jul 31 08:26:06 infsrv001 nfcol[574080]: 2020/07/31 08:26:06 Adding .done extension to .metrics in directory: /tmp/nfcol/sum_proto/