search

SHolzhauer / elastic-tip

Elastic TIP is a python tool which automates the process of aggregating Threat Intelligence and ingesting the intelligence into a common format into Elasticsearch with the main goal of being used by the Security solution.
GNU General Public License v3.0
27 stars 5 forks source link

"dataset" : "fwrules/emerging-Block-IPs" #9

Open CyberAbwehr opened 3 years ago

CyberAbwehr commented 3 years ago

Describe the bug Got a small amount of IPs into the Index elastic-tip

To Reproduce Steps to reproduce the behavior:

  1. I am importing the feed with following command python3 tip/elastic_tip_cli.py run -e https://10.11.20.16 -u elastic -p XYZ -m EmergingThreats-Blocklist --ca-cert /etc/elasticsearch/certs/ca/ca.crt

output:

Connection: <Elasticsearch([{'host': '10.11.20.16', 'port': 9200, 'use_ssl': True, 'ca_certs': '/etc/elasticsearch/certs/ca/ca.crt'}])> Connection: <Elasticsearch([{'host': '10.11.20.16', 'port': 9200, 'use_ssl': True, 'ca_certs': '/etc/elasticsearch/certs/ca/ca.crt'}])> Verifying TIP Index elastic-tip exists Running TIP Ingesting 1194 iocs from EmergingThreats-Blocklist into [{'host': '10.11.20.16', 'port': 9200, 'use_ssl': True, 'ca_certs': '/etc/elasticsearch/certs/ca/ca.crt'}] Ingested a total of 1194 IOC's

  1. I made some test and ping IPs from the list, I picked it up directly from the source https://rules.emergingthreats.net/fwrules/emerging-Block-IPs.txt

  2. I wrote a rule that match the IPs with the packetbeat and for some IPs it works and for other it was not working.

  3. I build a index pattern for this index and start to search and figured out, that I only have a small part of the IPs in the elastic.tip index

  4. I make a test in the command line: curl --cacert /etc/elasticsearch/certs/ca/ca.crt -u myadmin:XYZ -X GET https://SRVV-MN1-KIB-CPT.xyu.local:9200/elastic-tip/_search?pretty

Output:

{
  "took" : 1,
  "timed_out" : false,
  "_shards" : {
    "total" : 1,
    "successful" : 1,
    "skipped" : 0,
    "failed" : 0
  },
  "hits" : {
    "total" : {
      "value" : 215,
      "relation" : "eq"
    },
    "max_score" : 1.0,
    "hits" : [
      {
        "_index" : "elastic-tip",
        "_type" : "_doc",
        "_id" : "221d206e4b9e7d04975797e5e22ed790892afadf",
        "_score" : 1.0,
        "_source" : {
          "event" : {
            "kind" : "enrichment",
            "category" : "threat",
            "type" : "indicator",
            "reference" : "https://rules.emergingthreats.net/fwrules/emerging-Block-IPs.txt",
            "provider" : "EmergingThreats",
            "dataset" : "fwrules/emerging-Block-IPs",
            "severity" : 0,
            "risk_score" : 0,
            "original" : "103.146.185.107",
            "hash" : "221d206e4b9e7d04975797e5e22ed790892afadf"
          },
          "ecs" : {
            "version" : "1.8.0"
          },
          "threat" : {
            "indicator" : {
              "first_seen" : null,
              "last_seen" : null,
              "sightings" : 0,
              "type" : [
                "ip_address"
              ],
              "description" : null,
              "ip" : "103.146.185.107"
            },
            "tactic" : { },
            "technique" : { }
          },
          "@timestamp" : [
            "2021-03-11T16:52:01"
          ]
        }
      },
      {
        "_index" : "elastic-tip",
        "_type" : "_doc",
        "_id" : "9b8b4474824f62d883109406f6ee1eeaeb00c4a7",
        "_score" : 1.0,
        "_source" : {
          "event" : {
            "kind" : "enrichment",
            "category" : "threat",
            "type" : "indicator",
            "reference" : "https://rules.emergingthreats.net/fwrules/emerging-Block-IPs.txt",
            "provider" : "EmergingThreats",
            "dataset" : "fwrules/emerging-Block-IPs",
            "severity" : 0,
            "risk_score" : 0,
            "original" : "103.146.2.152",
            "hash" : "9b8b4474824f62d883109406f6ee1eeaeb00c4a7"
          },
          "ecs" : {
            "version" : "1.8.0"
          },
          "threat" : {
            "indicator" : {
              "first_seen" : null,
              "last_seen" : null,
              "sightings" : 0,
              "type" : [
                "ip_address"
              ],
              "description" : null,
              "ip" : "103.146.2.152"
            },
            "tactic" : { },
            "technique" : { }
          },
          "@timestamp" : [
            "2021-03-11T16:52:01"
          ]
        }
      },
      {
        "_index" : "elastic-tip",
        "_type" : "_doc",
        "_id" : "c188b29516f19cedfe73b80fe3c9349127bb86a1",
        "_score" : 1.0,
        "_source" : {
          "event" : {
            "kind" : "enrichment",
            "category" : "threat",
            "type" : "indicator",
            "reference" : "https://rules.emergingthreats.net/fwrules/emerging-Block-IPs.txt",
            "provider" : "EmergingThreats",
            "dataset" : "fwrules/emerging-Block-IPs",
            "severity" : 0,
            "risk_score" : 0,
            "original" : "103.150.68.124",
            "hash" : "c188b29516f19cedfe73b80fe3c9349127bb86a1"
          },
          "ecs" : {
            "version" : "1.8.0"
          },
          "threat" : {
            "indicator" : {
              "first_seen" : null,
              "last_seen" : null,
              "sightings" : 0,
              "type" : [
                "ip_address"
              ],
              "description" : null,
              "ip" : "103.150.68.124"
            },
            "tactic" : { },
            "technique" : { }
          },
          "@timestamp" : [
            "2021-03-11T16:52:01"
          ]
        }
      },
      {
        "_index" : "elastic-tip",
        "_type" : "_doc",
        "_id" : "d79e9d06ee8cca7f47fa3b74444223a88beb3de8",
        "_score" : 1.0,
        "_source" : {
          "event" : {
            "kind" : "enrichment",
            "category" : "threat",
            "type" : "indicator",
            "reference" : "https://rules.emergingthreats.net/fwrules/emerging-Block-IPs.txt",
            "provider" : "EmergingThreats",
            "dataset" : "fwrules/emerging-Block-IPs",
            "severity" : 0,
            "risk_score" : 0,
            "original" : "103.225.138.94",
            "hash" : "d79e9d06ee8cca7f47fa3b74444223a88beb3de8"
          },
          "ecs" : {
            "version" : "1.8.0"
          },
          "threat" : {
            "indicator" : {
              "first_seen" : null,
              "last_seen" : null,
              "sightings" : 0,
              "type" : [
                "ip_address"
              ],
              "description" : null,
              "ip" : "103.225.138.94"
            },
            "tactic" : { },
            "technique" : { }
          },
          "@timestamp" : [
            "2021-03-11T16:52:01"
          ]
        }
      },
      {
        "_index" : "elastic-tip",
        "_type" : "_doc",
        "_id" : "c980a4b1518dccb9c367a2e3e9b25c3ff277892e",
        "_score" : 1.0,
        "_source" : {
          "event" : {
            "kind" : "enrichment",
            "category" : "threat",
            "type" : "indicator",
            "reference" : "https://rules.emergingthreats.net/fwrules/emerging-Block-IPs.txt",
            "provider" : "EmergingThreats",
            "dataset" : "fwrules/emerging-Block-IPs",
            "severity" : 0,
            "risk_score" : 0,
            "original" : "103.239.165.24",
            "hash" : "c980a4b1518dccb9c367a2e3e9b25c3ff277892e"
          },
          "ecs" : {
            "version" : "1.8.0"
          },
          "threat" : {
            "indicator" : {
              "first_seen" : null,
              "last_seen" : null,
              "sightings" : 0,
              "type" : [
                "ip_address"
              ],
              "description" : null,
              "ip" : "103.239.165.24"
            },
            "tactic" : { },
            "technique" : { }
          },
          "@timestamp" : [
            "2021-03-11T16:52:01"
          ]
        }
      },
      {
        "_index" : "elastic-tip",
        "_type" : "_doc",
        "_id" : "a125f8c48e31435fc423eb0e533de017db495ffc",
        "_score" : 1.0,
        "_source" : {
          "event" : {
            "kind" : "enrichment",
            "category" : "threat",
            "type" : "indicator",
            "reference" : "https://rules.emergingthreats.net/fwrules/emerging-Block-IPs.txt",
            "provider" : "EmergingThreats",
            "dataset" : "fwrules/emerging-Block-IPs",
            "severity" : 0,
            "risk_score" : 0,
            "original" : "103.244.206.74",
            "hash" : "a125f8c48e31435fc423eb0e533de017db495ffc"
          },
          "ecs" : {
            "version" : "1.8.0"
          },
          "threat" : {
            "indicator" : {
              "first_seen" : null,
              "last_seen" : null,
              "sightings" : 0,
              "type" : [
                "ip_address"
              ],
              "description" : null,
              "ip" : "103.244.206.74"
            },
            "tactic" : { },
            "technique" : { }
          },
          "@timestamp" : [
            "2021-03-11T16:52:01"
          ]
        }
      },
      {
        "_index" : "elastic-tip",
        "_type" : "_doc",
        "_id" : "788d4b639264c1e7b45634015cbab1e1fc4ac827",
        "_score" : 1.0,
        "_source" : {
          "event" : {
            "kind" : "enrichment",
            "category" : "threat",
            "type" : "indicator",
            "reference" : "https://rules.emergingthreats.net/fwrules/emerging-Block-IPs.txt",
            "provider" : "EmergingThreats",
            "dataset" : "fwrules/emerging-Block-IPs",
            "severity" : 0,
            "risk_score" : 0,
            "original" : "103.40.116.68",
            "hash" : "788d4b639264c1e7b45634015cbab1e1fc4ac827"
          },
          "ecs" : {
            "version" : "1.8.0"
          },
          "threat" : {
            "indicator" : {
              "first_seen" : null,
              "last_seen" : null,
              "sightings" : 0,
              "type" : [
                "ip_address"
              ],
              "description" : null,
              "ip" : "103.40.116.68"
            },
            "tactic" : { },
            "technique" : { }
          },
          "@timestamp" : [
            "2021-03-11T16:52:01"
          ]
        }
      },
      {
        "_index" : "elastic-tip",
        "_type" : "_doc",
        "_id" : "a37837bc5e87495b506078cdd44347bf4dda9200",
        "_score" : 1.0,
        "_source" : {
          "event" : {
            "kind" : "enrichment",
            "category" : "threat",
            "type" : "indicator",
            "reference" : "https://rules.emergingthreats.net/fwrules/emerging-Block-IPs.txt",
            "provider" : "EmergingThreats",
            "dataset" : "fwrules/emerging-Block-IPs",
            "severity" : 0,
            "risk_score" : 0,
            "original" : "103.41.110.115",
            "hash" : "a37837bc5e87495b506078cdd44347bf4dda9200"
          },
          "ecs" : {
            "version" : "1.8.0"
          },
          "threat" : {
            "indicator" : {
              "first_seen" : null,
              "last_seen" : null,
              "sightings" : 0,
              "type" : [
                "ip_address"
              ],
              "description" : null,
              "ip" : "103.41.110.115"
            },
            "tactic" : { },
            "technique" : { }
          },
          "@timestamp" : [
            "2021-03-11T16:52:01"
          ]
        }
      },
      {
        "_index" : "elastic-tip",
        "_type" : "_doc",
        "_id" : "21c25551a5f6439c974637314d9f52915df9aeb2",
        "_score" : 1.0,
        "_source" : {
          "event" : {
            "kind" : "enrichment",
            "category" : "threat",
            "type" : "indicator",
            "reference" : "https://rules.emergingthreats.net/fwrules/emerging-Block-IPs.txt",
            "provider" : "EmergingThreats",
            "dataset" : "fwrules/emerging-Block-IPs",
            "severity" : 0,
            "risk_score" : 0,
            "original" : "103.54.42.218",
            "hash" : "21c25551a5f6439c974637314d9f52915df9aeb2"
          },
          "ecs" : {
            "version" : "1.8.0"
          },
          "threat" : {
            "indicator" : {
              "first_seen" : null,
              "last_seen" : null,
              "sightings" : 0,
              "type" : [
                "ip_address"
              ],
              "description" : null,
              "ip" : "103.54.42.218"
            },
            "tactic" : { },
            "technique" : { }
          },
          "@timestamp" : [
            "2021-03-11T16:52:01"
          ]
        }
      },
      {
        "_index" : "elastic-tip",
        "_type" : "_doc",
        "_id" : "74fa950b85308d4c20c2bd62a04c12fa53f29158",
        "_score" : 1.0,
        "_source" : {
          "event" : {
            "kind" : "enrichment",
            "category" : "threat",
            "type" : "indicator",
            "reference" : "https://rules.emergingthreats.net/fwrules/emerging-Block-IPs.txt",
            "provider" : "EmergingThreats",
            "dataset" : "fwrules/emerging-Block-IPs",
            "severity" : 0,
            "risk_score" : 0,
            "original" : "103.61.101.11",
            "hash" : "74fa950b85308d4c20c2bd62a04c12fa53f29158"
          },
          "ecs" : {
            "version" : "1.8.0"
          },
          "threat" : {
            "indicator" : {
              "first_seen" : null,
              "last_seen" : null,
              "sightings" : 0,
              "type" : [
                "ip_address"
              ],
              "description" : null,
              "ip" : "103.61.101.11"
            },
            "tactic" : { },
            "technique" : { }
          },
          "@timestamp" : [
            "2021-03-11T16:52:01"
          ]
        }
      }
    ]
  }
}

Expected behavior All IPs from the list are in the Index elastic-tip

best regards

SHolzhauer commented 3 years ago

Thank you @CyberAbwehr for reporting this, i'll look into it.

On first thought this is probably mapping conflicts and/or a bulk request not properly being build.

CyberAbwehr commented 3 years ago

Maybe you can integrate a search (index elastic-tip) for each feed to verify if the data is written to the index.

I made a test with the AbuseDB feed yesterday, and made a search after the import and there was nothing. The funny thing is I see that the index was growing.

curl --cacert /etc/elasticsearch/certs/ca/ca.crt -u myadmin:XYZ -X GET https://SRVV-MN1-KIB-CPT.xyz.local:9200/_cat/indices? |grep elastic-tip % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 100 9718 100 9718 0 0 150k 0 --:--:-- --:--:-- --:--:-- 153k green open elastic-tip 0soif9zATPGS1DLMb42XTA 1 1 10215 0 5.5mb 2.7mb

but if I made a search I will get following

curl --cacert /etc/elasticsearch/certs/ca/ca.crt -u myadmin:XYZ -X GET https://SRVV-MN1-KIB-CPT.xyz.local:9200/elastic-tip/_search?pretty { "took" : 1, "timed_out" : false, "_shards" : { "total" : 1, "successful" : 1, "skipped" : 0, "failed" : 0 }, "hits" : { "total" : { "value" : 10000, "relation" : "gte" }, "max_score" : 1.0, "hits" : [ { "_index" : "elastic-tip", "_type" : "_doc", "_id" : "221d206e4b9e7d04975797e5e22ed790892afadf", "_score" : 1.0, "_source" : { "event" : { "kind" : "enrichment", "category" : "threat", "type" : "indicator", "reference" : "https://rules.emergingthreats.net/fwrules/emerging-Block-IPs.txt", "provider" : "EmergingThreats", "dataset" : "fwrules/emerging-Block-IPs", "severity" : 0, "risk_score" : 0, "original" : "103.146.185.107", "hash" : "221d206e4b9e7d04975797e5e22ed790892afadf" }, "ecs" : { "version" : "1.8.0" }, "threat" : { "indicator" : { "first_seen" : null, "last_seen" : null, "sightings" : 0, "type" : [ "ip_address" ], "description" : null, "ip" : "103.146.185.107" }, "tactic" : { }, "technique" : { } }, "@timestamp" : [ "2021-03-11T17:13:41" ] } }, { "_index" : "elastic-tip", "_type" : "_doc", "_id" : "9b8b4474824f62d883109406f6ee1eeaeb00c4a7", "_score" : 1.0, "_source" : { "event" : { "kind" : "enrichment", "category" : "threat", "type" : "indicator", "reference" : "https://rules.emergingthreats.net/fwrules/emerging-Block-IPs.txt", "provider" : "EmergingThreats", "dataset" : "fwrules/emerging-Block-IPs", "severity" : 0, "risk_score" : 0, "original" : "103.146.2.152", "hash" : "9b8b4474824f62d883109406f6ee1eeaeb00c4a7" }, "ecs" : { "version" : "1.8.0" }, "threat" : { "indicator" : { "first_seen" : null, "last_seen" : null, "sightings" : 0, "type" : [ "ip_address" ], "description" : null, "ip" : "103.146.2.152" }, "tactic" : { }, "technique" : { } }, "@timestamp" : [ "2021-03-11T17:13:41" ] } }, { "_index" : "elastic-tip", "_type" : "_doc", "_id" : "c188b29516f19cedfe73b80fe3c9349127bb86a1", "_score" : 1.0, "_source" : { "event" : { "kind" : "enrichment", "category" : "threat", "type" : "indicator", "reference" : "https://rules.emergingthreats.net/fwrules/emerging-Block-IPs.txt", "provider" : "EmergingThreats", "dataset" : "fwrules/emerging-Block-IPs", "severity" : 0, "risk_score" : 0, "original" : "103.150.68.124", "hash" : "c188b29516f19cedfe73b80fe3c9349127bb86a1" }, "ecs" : { "version" : "1.8.0" }, "threat" : { "indicator" : { "first_seen" : null, "last_seen" : null, "sightings" : 0, "type" : [ "ip_address" ], "description" : null, "ip" : "103.150.68.124" }, "tactic" : { }, "technique" : { } }, "@timestamp" : [ "2021-03-11T17:13:41" ] } }, { "_index" : "elastic-tip", "_type" : "_doc", "_id" : "d79e9d06ee8cca7f47fa3b74444223a88beb3de8", "_score" : 1.0, "_source" : { "event" : { "kind" : "enrichment", "category" : "threat", "type" : "indicator", "reference" : "https://rules.emergingthreats.net/fwrules/emerging-Block-IPs.txt", "provider" : "EmergingThreats", "dataset" : "fwrules/emerging-Block-IPs", "severity" : 0, "risk_score" : 0, "original" : "103.225.138.94", "hash" : "d79e9d06ee8cca7f47fa3b74444223a88beb3de8" }, "ecs" : { "version" : "1.8.0" }, "threat" : { "indicator" : { "first_seen" : null, "last_seen" : null, "sightings" : 0, "type" : [ "ip_address" ], "description" : null, "ip" : "103.225.138.94" }, "tactic" : { }, "technique" : { } }, "@timestamp" : [ "2021-03-11T17:13:41" ] } }, { "_index" : "elastic-tip", "_type" : "_doc", "_id" : "c980a4b1518dccb9c367a2e3e9b25c3ff277892e", "_score" : 1.0, "_source" : { "event" : { "kind" : "enrichment", "category" : "threat", "type" : "indicator", "reference" : "https://rules.emergingthreats.net/fwrules/emerging-Block-IPs.txt", "provider" : "EmergingThreats", "dataset" : "fwrules/emerging-Block-IPs", "severity" : 0, "risk_score" : 0, "original" : "103.239.165.24", "hash" : "c980a4b1518dccb9c367a2e3e9b25c3ff277892e" }, "ecs" : { "version" : "1.8.0" }, "threat" : { "indicator" : { "first_seen" : null, "last_seen" : null, "sightings" : 0, "type" : [ "ip_address" ], "description" : null, "ip" : "103.239.165.24" }, "tactic" : { }, "technique" : { } }, "@timestamp" : [ "2021-03-11T17:13:41" ] } }, { "_index" : "elastic-tip", "_type" : "_doc", "_id" : "a125f8c48e31435fc423eb0e533de017db495ffc", "_score" : 1.0, "_source" : { "event" : { "kind" : "enrichment", "category" : "threat", "type" : "indicator", "reference" : "https://rules.emergingthreats.net/fwrules/emerging-Block-IPs.txt", "provider" : "EmergingThreats", "dataset" : "fwrules/emerging-Block-IPs", "severity" : 0, "risk_score" : 0, "original" : "103.244.206.74", "hash" : "a125f8c48e31435fc423eb0e533de017db495ffc" }, "ecs" : { "version" : "1.8.0" }, "threat" : { "indicator" : { "first_seen" : null, "last_seen" : null, "sightings" : 0, "type" : [ "ip_address" ], "description" : null, "ip" : "103.244.206.74" }, "tactic" : { }, "technique" : { } }, "@timestamp" : [ "2021-03-11T17:13:41" ] } }, { "_index" : "elastic-tip", "_type" : "_doc", "_id" : "788d4b639264c1e7b45634015cbab1e1fc4ac827", "_score" : 1.0, "_source" : { "event" : { "kind" : "enrichment", "category" : "threat", "type" : "indicator", "reference" : "https://rules.emergingthreats.net/fwrules/emerging-Block-IPs.txt", "provider" : "EmergingThreats", "dataset" : "fwrules/emerging-Block-IPs", "severity" : 0, "risk_score" : 0, "original" : "103.40.116.68", "hash" : "788d4b639264c1e7b45634015cbab1e1fc4ac827" }, "ecs" : { "version" : "1.8.0" }, "threat" : { "indicator" : { "first_seen" : null, "last_seen" : null, "sightings" : 0, "type" : [ "ip_address" ], "description" : null, "ip" : "103.40.116.68" }, "tactic" : { }, "technique" : { } }, "@timestamp" : [ "2021-03-11T17:13:41" ] } }, { "_index" : "elastic-tip", "_type" : "_doc", "_id" : "a37837bc5e87495b506078cdd44347bf4dda9200", "_score" : 1.0, "_source" : { "event" : { "kind" : "enrichment", "category" : "threat", "type" : "indicator", "reference" : "https://rules.emergingthreats.net/fwrules/emerging-Block-IPs.txt", "provider" : "EmergingThreats", "dataset" : "fwrules/emerging-Block-IPs", "severity" : 0, "risk_score" : 0, "original" : "103.41.110.115", "hash" : "a37837bc5e87495b506078cdd44347bf4dda9200" }, "ecs" : { "version" : "1.8.0" }, "threat" : { "indicator" : { "first_seen" : null, "last_seen" : null, "sightings" : 0, "type" : [ "ip_address" ], "description" : null, "ip" : "103.41.110.115" }, "tactic" : { }, "technique" : { } }, "@timestamp" : [ "2021-03-11T17:13:41" ] } }, { "_index" : "elastic-tip", "_type" : "_doc", "_id" : "21c25551a5f6439c974637314d9f52915df9aeb2", "_score" : 1.0, "_source" : { "event" : { "kind" : "enrichment", "category" : "threat", "type" : "indicator", "reference" : "https://rules.emergingthreats.net/fwrules/emerging-Block-IPs.txt", "provider" : "EmergingThreats", "dataset" : "fwrules/emerging-Block-IPs", "severity" : 0, "risk_score" : 0, "original" : "103.54.42.218", "hash" : "21c25551a5f6439c974637314d9f52915df9aeb2" }, "ecs" : { "version" : "1.8.0" }, "threat" : { "indicator" : { "first_seen" : null, "last_seen" : null, "sightings" : 0, "type" : [ "ip_address" ], "description" : null, "ip" : "103.54.42.218" }, "tactic" : { }, "technique" : { } }, "@timestamp" : [ "2021-03-11T17:13:41" ] } }, { "_index" : "elastic-tip", "_type" : "_doc", "_id" : "74fa950b85308d4c20c2bd62a04c12fa53f29158", "_score" : 1.0, "_source" : { "event" : { "kind" : "enrichment", "category" : "threat", "type" : "indicator", "reference" : "https://rules.emergingthreats.net/fwrules/emerging-Block-IPs.txt", "provider" : "EmergingThreats", "dataset" : "fwrules/emerging-Block-IPs", "severity" : 0, "risk_score" : 0, "original" : "103.61.101.11", "hash" : "74fa950b85308d4c20c2bd62a04c12fa53f29158" }, "ecs" : { "version" : "1.8.0" }, "threat" : { "indicator" : { "first_seen" : null, "last_seen" : null, "sightings" : 0, "type" : [ "ip_address" ], "description" : null, "ip" : "103.61.101.11" }, "tactic" : { }, "technique" : { } }, "@timestamp" : [ "2021-03-11T17:13:41" ] } } ] } }

I looks like that Abusedb Feed is not in the Index (elastic.tip)