ti_opencti integration causes field conflict

[colin-stubbs]

Field event.original is an object within the ti_opencti.indicator datastream, when it should be a keyword to conform to ECS.

This is due to enabling retention of event.original while debugging the integration configuration and connectivity from elastic agent to OpenCTI.

I will not be the last to need to do this.

Field conflicts result in search related issues and likely missed security detections that should occur.

The ti_opencti.indicator ingest pipeline should convert the original OpenCTI object which is being stored in event.original to an escaped JSON text string instead, in order to avoid conflicts with other logs-* indices, and in particular other logs-ti_* indices.

The original object should be stored in an alternative structure instead, for example as opencti.original or similar.

Conflicts with other logs-ti_ indices,

Example document,

{
  "_index": ".ds-logs-ti_opencti.indicator-default-2023.11.01-000001",
  "_id": "v0GviYsB7yFddSXSKrvB",
  "_version": 1,
  "_score": 0,
  "_source": {
    "agent": {
      "name": "elastic-agent",
      "id": "7d3d7879-7cd6-4352-9c2b-186499853bc2",
      "type": "filebeat",
      "ephemeral_id": "4fbed899-b644-4a48-960c-8aa303063f4e",
      "version": "8.10.4"
    },
    "elastic_agent": {
      "id": "7d3d7879-7cd6-4352-9c2b-186499853bc2",
      "version": "8.10.4",
      "snapshot": false
    },
    "cloud": {
      "availability_zone": "",
      "instance": {
        "name": "",
        "id": ""
      },
      "provider": "hetzner",
      "service": {
        "name": "Cloud"
      },
      "region": ""
    },
    "ecs": {
      "version": "8.10.0"
    },
    "related": {
      "ip": [
        "77.97.164.31"
      ]
    },
    "event": {
      "agent_id_status": "verified",
      "ingested": "2023-11-01T07:00:40Z",
      "original": {
        "standard_id": "indicator--44117981-d1dc-58a4-95fc-dd3944b2594b",
        "externalReferences": {
          "edges": []
        },
        "killChainPhases": {
          "edges": []
        },
        "pattern_type": "stix",
        "created": "2023-11-01T05:33:54.025Z",
        "confidence": 70,
        "pattern_version": "2.1",
        "pattern": "[ipv4-addr:value = '77.97.164.31']",
        "valid_from": "2023-11-01T05:33:54.025Z",
        "description": "Malicious SSL connections",
        "revoked": false,
        "objectMarking": {
          "edges": []
        },
        "x_opencti_main_observable_type": "IPv4-Addr",
        "observables": {
          "pageInfo": {
            "globalCount": 1
          },
          "edges": [
            {
              "node": {
                "standard_id": "ipv4-addr--7ade4706-945d-5641-9061-95a8471f8d6f",
                "entity_type": "IPv4-Addr",
                "id": "3096eea9-148f-4419-a407-8d937ea3656b",
                "value": "77.97.164.31",
                "observable_value": "77.97.164.31"
              }
            }
          ]
        },
        "valid_until": "2023-12-31T05:33:54.025Z",
        "is_inferred": false,
        "createdBy": {
          "identity_class": "organization",
          "name": "Abuse.ch ssl blacklist"
        },
        "name": "77.97.164.31",
        "modified": "2023-11-01T05:36:22.028Z",
        "x_opencti_detection": false,
        "id": "8e0ea4c6-3943-4c60-8413-b983217689ad",
        "x_opencti_score": 50,
        "lang": "en",
        "objectLabel": {
          "edges": [
            {
              "node": {
                "value": "osint"
              }
            },
            {
              "node": {
                "value": "ssl-blacklist"
              }
            }
          ]
        }
      },
      "kind": "enrichment",
      "created": "2023-11-01T05:33:54.025Z",
      "id": "8e0ea4c6-3943-4c60-8413-b983217689ad",
      "category": [
        "threat"
      ],
      "type": [
        "indicator"
      ],
      "dataset": "ti_opencti.indicator"
    },
    "tags": [
      "preserve_original_event",
      "forwarded",
      "opencti-indicator",
      "osint",
      "ssl-blacklist",
      "ecs-indicator-detail"
    ],
    "input": {
      "type": "cel"
    },
    "@timestamp": "2023-11-01T07:00:38.018Z",
    "data_stream": {
      "namespace": "default",
      "type": "logs",
      "dataset": "ti_opencti.indicator"
    },
    "threat": {
      "indicator": {
        "reference": "https://opencti/dashboard/observations/indicators/8e0ea4c6-3943-4c60-8413-b983217689ad",
        "provider": "Abuse.ch ssl blacklist",
        "confidence": "High",
        "ip": [
          "77.97.164.31"
        ],
        "name": "77.97.164.31",
        "description": "Malicious SSL connections",
        "modified_at": "2023-11-01T05:36:22.028Z",
        "type": "ipv4-addr"
      },
      "feed": {
        "reference": "https://docs.opencti.io/latest/usage/overview/",
        "name": "OpenCTI",
        "description": "Indicator data from OpenCTI",
        "dashboard_id": "ti_opencti-83b2bef0-591c-11ee-ba5f-49a63bb985cd"
      }
    },
    "opencti": {
      "indicator": {
        "standard_id": "indicator--44117981-d1dc-58a4-95fc-dd3944b2594b",
        "detection": false,
        "pattern_type": "stix",
        "pattern_version": "2.1",
        "pattern": "[ipv4-addr:value = '77.97.164.31']",
        "valid_from": "2023-11-01T05:33:54.025Z",
        "revoked": false,
        "score": 50,
        "valid_until": "2023-12-31T05:33:54.025Z",
        "is_inferred": false,
        "observables_count": 1,
        "lang": "en",
        "creator_identity_class": "organization"
      },
      "observable": {
        "ipv4_addr": {
          "standard_id": "ipv4-addr--7ade4706-945d-5641-9061-95a8471f8d6f",
          "entity_type": "IPv4-Addr",
          "id": "3096eea9-148f-4419-a407-8d937ea3656b",
          "value": "77.97.164.31"
        }
      }
    }
  }
  }
}

[colin-stubbs]

Will create fork and PR to merge @custom based fixes,

PUT _component_template/logs-ti_opencti.indicator@custom
{
  "template": {
    "settings": {},
    "mappings": {
      "properties": {
        "opencti": {
          "type": "object",
          "properties": {
            "original": {
              "type": "object"
            }
          }
        }
      }
    }
  },
  "_meta": {
    "package": {
      "name": "ti_opencti"
    },
    "managed_by": "fleet",
    "managed": true
  }
}

PUT _ingest/pipeline/logs-ti_opencti.indicator@custom
{
  "version": 1,
  "processors": [
    {
      "set": {
        "field": "opencti.original",
        "override": false,
        "ignore_empty_value": true,
        "copy_from": "event.original"
      }
    },
    {
      "convert": {
        "field": "event.original",
        "type": "string",
        "ignore_missing": true
      }
    }
  ]
}

Results in,

{
  "_index": ".ds-logs-ti_opencti.indicator-default-2023.11.08-000001",
  "_id": "vWpBrIsB7yFddSXSrTIm",
  "_version": 1,
  "_score": 0,
  "_ignored": [
    "event.original"
  ],
  "_source": {
    "agent": {
      "name": "elastic-agent",
      "id": "7d3d7879-7cd6-4352-9c2b-186499853bc2",
      "ephemeral_id": "b637692d-fe48-4d2f-99e8-409e3f822849",
      "type": "filebeat",
      "version": "8.10.4"
    },
    "elastic_agent": {
      "id": "7d3d7879-7cd6-4352-9c2b-186499853bc2",
      "version": "8.10.4",
      "snapshot": false
    },
    "cloud": {
      "availability_zone": "",
      "instance": {
        "name": "elastic-agent",
        "id": "fbc5c58500c6da4105900421debc5051"
      },
      "provider": "openstack",
      "machine": {
        "type": ""
      },
      "service": {
        "name": "Nova"
      }
    },
    "ecs": {
      "version": "8.10.0"
    },
    "related": {
      "hosts": [
        "meshdapps.com"
      ]
    },
    "event": {
      "agent_id_status": "verified",
      "ingested": "2023-11-08T00:07:47Z",
      "original": "{standard_id=indicator--21eaa190-20a4-5fe3-918e-a7d443ccc588, externalReferences={edges=[]}, killChainPhases={edges=[]}, pattern_type=stix, created=2023-11-01T09:32:54.782Z, confidence=40, pattern=[url:value = 'https://meshdapps.com/app1/list-1.html'], pattern_version=2.1, valid_from=2023-11-01T09:32:54.782Z, description=Phishunt malicious URL, objectMarking={edges=[]}, revoked=false, x_opencti_main_observable_type=Url, observables={pageInfo={globalCount=1}, edges=[{node={standard_id=url--4497ba4e-dda2-54ca-9387-70902a82d499, entity_type=Url, id=0232b5af-e4ba-49ec-8fb0-cd17547425d9, value=https://meshdapps.com/app1/list-1.html, observable_value=https://meshdapps.com/app1/list-1.html}}]}, valid_until=2024-04-29T09:32:54.782Z, is_inferred=false, createdBy={identity_class=organization, name=Phishunt}, name=https://meshdapps.com/app1/list-1.html, modified=2023-11-01T09:37:38.206Z, x_opencti_detection=false, id=ab2d8d34-ff46-4b3d-b58e-33ac1a1c8d70, lang=en, x_opencti_score=50, objectLabel={edges=[{node={value=osint}}, {node={value=phishing}}]}}",
      "kind": "enrichment",
      "created": "2023-11-01T09:32:54.782Z",
      "id": "ab2d8d34-ff46-4b3d-b58e-33ac1a1c8d70",
      "category": [
        "threat"
      ],
      "type": [
        "indicator"
      ],
      "dataset": "ti_opencti.indicator"
    },
    "tags": [
      "preserve_original_event",
      "forwarded",
      "opencti-indicator",
      "osint",
      "phishing",
      "ecs-indicator-detail"
    ],
    "input": {
      "type": "cel"
    },
    "@timestamp": "2023-11-08T00:07:46.041Z",
    "data_stream": {
      "namespace": "default",
      "type": "logs",
      "dataset": "ti_opencti.indicator"
    },
    "threat": {
      "indicator": {
        "reference": "https://opencti/dashboard/observations/indicators/ab2d8d34-ff46-4b3d-b58e-33ac1a1c8d70",
        "provider": "Phishunt",
        "confidence": "Medium",
        "name": "https://meshdapps.com/app1/list-1.html",
        "description": "Phishunt malicious URL",
        "modified_at": "2023-11-01T09:37:38.206Z",
        "type": "url",
        "url": {
          "path": "/app1/list-1.html",
          "extension": "html",
          "registered_domain": "meshdapps.com",
          "original": "https://meshdapps.com/app1/list-1.html",
          "scheme": "https",
          "top_level_domain": "com",
          "domain": "meshdapps.com",
          "full": "https://meshdapps.com/app1/list-1.html"
        }
      },
      "feed": {
        "reference": "https://docs.opencti.io/latest/usage/overview/",
        "name": "OpenCTI",
        "description": "Indicator data from OpenCTI",
        "dashboard_id": "ti_opencti-83b2bef0-591c-11ee-ba5f-49a63bb985cd"
      }
    },
    "opencti": {
      "indicator": {
        "standard_id": "indicator--21eaa190-20a4-5fe3-918e-a7d443ccc588",
        "detection": false,
        "pattern_type": "stix",
        "pattern_version": "2.1",
        "pattern": "[url:value = 'https://meshdapps.com/app1/list-1.html']",
        "valid_from": "2023-11-01T09:32:54.782Z",
        "revoked": false,
        "score": 50,
        "valid_until": "2024-04-29T09:32:54.782Z",
        "is_inferred": false,
        "observables_count": 1,
        "lang": "en",
        "creator_identity_class": "organization"
      },
      "original": {
        "standard_id": "indicator--21eaa190-20a4-5fe3-918e-a7d443ccc588",
        "killChainPhases": {
          "edges": []
        },
        "pattern": "[url:value = 'https://meshdapps.com/app1/list-1.html']",
        "valid_from": "2023-11-01T09:32:54.782Z",
        "description": "Phishunt malicious URL",
        "objectMarking": {
          "edges": []
        },
        "revoked": false,
        "observables": {
          "pageInfo": {
            "globalCount": 1
          },
          "edges": [
            {
              "node": {
                "standard_id": "url--4497ba4e-dda2-54ca-9387-70902a82d499",
                "entity_type": "Url",
                "id": "0232b5af-e4ba-49ec-8fb0-cd17547425d9",
                "value": "https://meshdapps.com/app1/list-1.html",
                "observable_value": "https://meshdapps.com/app1/list-1.html"
              }
            }
          ]
        },
        "modified": "2023-11-01T09:37:38.206Z",
        "id": "ab2d8d34-ff46-4b3d-b58e-33ac1a1c8d70",
        "lang": "en",
        "x_opencti_score": 50,
        "externalReferences": {
          "edges": []
        },
        "pattern_type": "stix",
        "created": "2023-11-01T09:32:54.782Z",
        "confidence": 40,
        "pattern_version": "2.1",
        "x_opencti_main_observable_type": "Url",
        "valid_until": "2024-04-29T09:32:54.782Z",
        "is_inferred": false,
        "createdBy": {
          "name": "Phishunt",
          "identity_class": "organization"
        },
        "name": "https://meshdapps.com/app1/list-1.html",
        "x_opencti_detection": false,
        "objectLabel": {
          "edges": [
            {
              "node": {
                "value": "osint"
              }
            },
            {
              "node": {
                "value": "phishing"
              }
            }
          ]
        }
      },
      "observable": {
        "url": {
          "standard_id": "url--4497ba4e-dda2-54ca-9387-70902a82d499",
          "entity_type": "Url",
          "id": "0232b5af-e4ba-49ec-8fb0-cd17547425d9",
          "value": "https://meshdapps.com/app1/list-1.html"
        }
      }
    }
  }
}

[colin-stubbs]

Storing full object to opencti.original opens too many cans of worms and creates future potential for field type conflicts again. Plain forced conversion of event.original to text for storage as a keyword is all that is needed and this still permits debugging of the JSON structure received from OpenCTI, albeit outside of Elastic.

[elasticmachine]

Pinging @elastic/security-external-integrations (Team:Security-External Integrations)