IP addresses do not come to elasticsearch

[sergkstudio]

Environment:

• OS name: Debian, docker container
• OS version: 12
• Architecture: amd64
• ntopng version/revision: 6.2
• es/kibana version: 8.15.3

What happened: Server end client IP addresses do not come to elasticsearch by sending "-F “es;flows;ntopng-%Y.%m.%d;http://XYZ:9200/_bulk;”" IP address data in ntop is normal

JSON example of incoming data

{
  "_index": "ntopng-2024.10.18",
  "_id": "pd3-n5IBh3vl--E-BFzz",
  "_version": 1,
  "_score": 0,
  "_source": {
    "@timestamp": "2024-10-18T14:17:53.0Z",
    "type": "flows",
    "client": {
      "ip": "0.0.0.0",
      "is_local": false,
      "is_blacklisted": false,
      "has_services": 0,
      "domain": "",
      "geo": {
        "country_name": "",
        "location": {
          "lon": 0,
          "lat": 0
        }
      },
      "tos": 0,
      "port": 58272,
      "packets": 60642,
      "bytes": 3198786,
      "packets_retransmissions": 0,
      "packets_out_of_order": 1,
      "packets_lost": 2,
      "latency": 29.694
    },
    "server": {
      "ip": "0.0.0.0",
      "is_local": false,
      "is_blacklisted": false,
      "has_services": 0,
      "domain": "",
      "geo": {
        "country_name": "NL",
        "location": {
          "lon": 4.904139995574951,
          "lat": 52.36759948730469
        }
      },
      "tos": 0,
      "port": 443,
      "packets": 119413,
      "bytes": 167402933,
      "packets_retransmissions": 3,
      "packets_out_of_order": 1895,
      "packets_lost": 630,
      "latency": 29.694
    },
    "community_id": "1:HmtWa6VRIidny7646FY3uE9PBKw=",
    "network": {
      "iana_number": 6,
      "protocol": "TLS.YouTube",
      "tcp_flags": 26,
      "first_seen": 1729260776,
      "last_seen": 1729261073,
      "category": {
        "name": "Media",
        "id": 1
      },
      "type": "ipv4",
      "info": "rr3---sn-5hnekn7z.googlevideo.com"
    },
    "interface": {
      "id": 0,
      "name": "wg-firezone"
    },
    "observer": {
      "product": "ntopng",
      "vendor": "ntop",
      "name": "firezone"
    },
    "event": {
      "risk_score": 0
    },
    "tls": {
      "server_name": "rr3---sn-5hnekn7z.googlevideo.com"
    }
  },
  "fields": {
    "network.category.id": [
      "1"
    ],
    "network.last_seen": [
      1729261073
    ],
    "server.ip": [
      "0.0.0.0"
    ],
    "server.packets_retransmissions": [
      3
    ],
    "type": [
      "flows"
    ],
    "server.is_blacklisted": [
      false
    ],
    "client.packets_out_of_order": [
      1
    ],
    "client.tos": [
      0
    ],
    "observer.vendor": [
      "ntop"
    ],
    "client.packets_lost": [
      2
    ],
    "client.geo.country_name": [
      ""
    ],
    "tls.server_name": [
      "rr3---sn-5hnekn7z.googlevideo.com"
    ],
    "client.packets_retransmissions": [
      0
    ],
    "server.latency": [
      29.694
    ],
    "client.ip": [
      "0.0.0.0"
    ],
    "event.risk_score": [
      0
    ],
    "server.tos": [
      0
    ],
    "client.port": [
      58272
    ],
    "client.latency": [
      29.694
    ],
    "network.iana_number": [
      "6"
    ],
    "server.bytes": [
      167402933
    ],
    "client.has_services": [
      0
    ],
    "client.is_blacklisted": [
      false
    ],
    "interface.name": [
      "wg-firezone"
    ],
    "client.packets": [
      60642
    ],
    "observer.name": [
      "firezone"
    ],
    "server.packets_lost": [
      630
    ],
    "community_id": [
      "1:HmtWa6VRIidny7646FY3uE9PBKw="
    ],
    "client.geo.location": [
      {
        "coordinates": [
          0,
          0
        ],
        "type": "Point"
      }
    ],
    "network.protocol": [
      "TLS.YouTube"
    ],
    "server.packets_out_of_order": [
      1895
    ],
    "server.port": [
      443
    ],
    "observer.product": [
      "ntopng"
    ],
    "network.type": [
      "ipv4"
    ],
    "client.domain": [
      ""
    ],
    "interface.id": [
      0
    ],
    "server.geo.country_name": [
      "NL"
    ],
    "server.packets": [
      119413
    ],
    "network.tcp_flags": [
      26
    ],
    "server.has_services": [
      0
    ],
    "server.is_local": [
      false
    ],
    "network.info": [
      "rr3---sn-5hnekn7z.googlevideo.com"
    ],
    "@timestamp": [
      "2024-10-18T14:17:53.000Z"
    ],
    "server.domain": [
      ""
    ],
    "client.bytes": [
      3198786
    ],
    "client.is_local": [
      false
    ],
    "network.first_seen": [
      1729260776
    ],
    "network.category.name": [
      "Media"
    ],
    "server.geo.location": [
      {
        "coordinates": [
          4.904139995574951,
          52.36759948730469
        ],
        "type": "Point"
      }
    ]
  }
}

[MatteoBiscosi]

Could you please share your ntopng.conf file?

[sergkstudio]

Could you please share your ntopng.conf file?

This is a docker container, I can share compose.yml File ntopng.conf in ntop directory is empty

version: "3.8"
services:
  ntopng:
    image: ntop/ntopng:stable
    container_name: ntopng
    restart: unless-stopped
    network_mode: host
    privileged: true
    volumes:
      - ./data/ntopng:/var/lib/ntopng
    command: ntopng --community -d "/var/lib/ntopng" -i "wg-firezone" -F
      "es;flows;ntopng-%Y.%m.%d;http://192.168.252.253:9200/_bulk;"
  elasticsearch:
    image: elasticsearch:8.15.3
    container_name: elasticsearch
    restart: unless-stopped
    environment:
      - node.name=elasticsearch
      - discovery.type=single-node
      - bootstrap.memory_lock=true
      - ES_JAVA_OPTS=-Xms256m -Xmx256m
      - xpack.security.enabled=false # Отключение безопасности для простого запуска
    ports:
      - 9200:9200
    #volumes:
    #  - ./esdata:/usr/share/elasticsearch/data
    #      - ./esdata/log:/usr/share/elasticsearch/logs
    networks:
      default:
        ipv4_address: 172.18.2.18
  kibana:
    image: kibana:8.15.3
    restart: unless-stopped
    container_name: kibana
    environment:
      ELASTICSEARCH_URL: http://elasticsearch:9200
      ELASTICSEARCH_HOSTS: '["http://elasticsearch:9200"]'
    ports:
      - 5601:5601
    depends_on:
      - elasticsearch
    networks:
      default:
        ipv4_address: 172.18.2.19
  redis:
    image: redis:alpine
    container_name: redis
    command:
      - redis-server
      - --save 900 1
    ports:
      - 1679:6379
    volumes:
      - ./data/redis:/data
    networks:
      default:
        ipv4_address: 172.18.2.20
networks:
  default:
    ipam:
      config:
        - subnet: 172.18.2.16/29
          gateway: 172.18.2.17

[MatteoBiscosi]

Hi @sergkstudio please capture a small pcap of your traffic and send me that to biscosi@ntop.org In my labs everything is exported correctly, so i'd like to have a small pcap to reproduce your issue

[MatteoBiscosi]

Checking from kibana, same configuration as yours, but on my side the IPs are correctly exported. Please try by moving to the ntopng dev version, if the issue persists or not