search

OpenCTI-Platform / opencti

Open Cyber Threat Intelligence Platform
https://opencti.io
Other
6.14k stars 912 forks source link

STIX Import exception - TypeError: unhashable type: 'dict' #3462

Closed MaxwellDPS closed 1 year ago

MaxwellDPS commented 1 year ago

Description

When importing the exmaple bundle below the import stix breaks with TypeError: unhashable type: 'dict'

Environment

  1. OS (where OpenCTI server runs): CentOS Stream 9
  2. OpenCTI version: 5.7.2
  3. OpenCTI client: Frontend
  4. Other environment details: Kubernetes -- Clustered Deployment

Reproducible Steps

Steps to create the smallest reproducible scenario:

  1. Import bundle
  2. Queue import
  3. 🔥

Expected Output

Proper import of the STIX bundle

Actual Output

Traceback (most recent call last):
  File "/usr/local/lib/python3.10/site-packages/pycti/connector/opencti_connector_helper.py", line 167, in _process_message
    json_data["internal"]["work_id"], message_task.result()
  File "/usr/local/lib/python3.10/concurrent/futures/thread.py", line 58, in run
    result = self.fn(*self.args, **self.kwargs)
  File "/opt/opencti-connector-import-file-stix/import-file-stix.py", line 52, in _process_message
    bundles_sent = self.helper.send_stix2_bundle(
  File "/usr/local/lib/python3.10/site-packages/pycti/connector/opencti_connector_helper.py", line 894, in send_stix2_bundle
    bundles = stix2_splitter.split_bundle(bundle, True, event_version)
  File "/usr/local/lib/python3.10/site-packages/pycti/utils/opencti_stix2_splitter.py", line 90, in split_bundle
    self.enlist_element(item["id"], raw_data)
  File "/usr/local/lib/python3.10/site-packages/pycti/utils/opencti_stix2_splitter.py", line 52, in enlist_element
    nb_deps += self.enlist_element(value, raw_data)
  File "/usr/local/lib/python3.10/site-packages/pycti/utils/opencti_stix2_splitter.py", line 27, in enlist_element
    if item_id not in raw_data:
TypeError: unhashable type: 'dict'

Additional information

Example bundle

{
    "id": "bundle--<REMOVED>",
    "objects": [
        {
            "confidence": 100,
            "created": "<REMOVED>",
            "created_by_ref": "identity--<REMOVED>",
            "description": "brick1",
            "id": "identity--<REMOVED>",
            "identity_class": "system",
            "labels": [
                "intenal:system"
            ],
            "modified": "<REMOVED>",
            "name": "brick1",
            "object_marking_refs": [
                "marking-definition--<REMOVED>"
            ],
            "spec_version": "2.1",
            "type": "identity"
        },
        {
            "confidence": 100,
            "created": "<REMOVED>",
            "created_by_ref": "identity--<REMOVED>",
            "id": "relationship--<REMOVED>",
            "modified": "<REMOVED>",
            "object_marking_refs": [
                "marking-definition--<REMOVED>"
            ],
            "relationship_type": "related-to",
            "source_ref": "identity--<REMOVED>",
            "spec_version": "2.1",
            "target_ref": "identity--<REMOVED>",
            "type": "relationship"
        },
        {
            "confidence": 80,
            "created": "<REMOVED>",
            "createdBy": {
                "contact_information": null,
                "created": "<REMOVED>",
                "description": "",
                "entity_type": "Organization",
                "id": "<REMOVED>",
                "identity_class": "organization",
                "modified": "<REMOVED>",
                "name": "feedxyz",
                "objectLabel": [],
                "objectLabelIds": [],
                "parent_types": [
                    "Basic-Object",
                    "Stix-Object",
                    "Stix-Core-Object",
                    "Stix-Domain-Object",
                    "Identity"
                ],
                "roles": null,
                "spec_version": "2.1",
                "standard_id": "identity--<REMOVED>",
                "x_opencti_aliases": null,
                "x_opencti_organization_type": null,
                "x_opencti_reliability": null
            },
            "createdById": "<REMOVED>",
            "created_at": "<REMOVED>",
            "description": "",
            "entity_type": "Indicator",
            "id": "indicator--<REMOVED>",
            "indicator_types": [
                "malicious-activity"
            ],
            "labels": [
                "port scan",
                "hacking",
                "brute force",
                "clear",
                "sighted"
            ],
            "modified": "<REMOVED>",
            "name": "<REMOVED>",
            "object_marking_refs": [
                "marking-definition--<REMOVED>"
            ],
            "observables": [
                {
                    "createdById": null,
                    "entity_type": "IPv4-Addr",
                    "id": "<REMOVED>",
                    "observable_value": "<REMOVED>"
                }
            ],
            "observablesIds": [
                "<REMOVED>"
            ],
            "parent_types": [
                "Basic-Object",
                "Stix-Object",
                "Stix-Core-Object",
                "Stix-Domain-Object"
            ],
            "pattern": "[ipv4-addr:value = '<REMOVED>']",
            "pattern_type": "stix",
            "pattern_version": "2.1",
            "revoked": true,
            "spec_version": "2.1",
            "type": "indicator",
            "updated_at": "<REMOVED>",
            "valid_from": "<REMOVED>",
            "valid_until": "<REMOVED>",
            "x_opencti_detection": true,
            "x_opencti_id": "<REMOVED>",
            "x_opencti_main_observable_type": "IPv4-Addr",
            "x_opencti_score": 60
        },
        {
            "createdBy": {
                "contact_information": null,
                "created": "<REMOVED>",
                "description": "",
                "entity_type": "Organization",
                "id": "<REMOVED>",
                "identity_class": "organization",
                "modified": "<REMOVED>",
                "name": "feedxyz",
                "objectLabel": [],
                "objectLabelIds": [],
                "parent_types": [
                    "Basic-Object",
                    "Stix-Object",
                    "Stix-Core-Object",
                    "Stix-Domain-Object",
                    "Identity"
                ],
                "roles": null,
                "spec_version": "2.1",
                "standard_id": "identity--<REMOVED>",
                "x_opencti_aliases": null,
                "x_opencti_organization_type": null,
                "x_opencti_reliability": null
            },
            "createdById": "<REMOVED>",
            "created_at": "<REMOVED>",
            "entity_type": "IPv4-Addr",
            "id": "ipv4-addr--<REMOVED>",
            "indicators": [
                {
                    "createdById": null,
                    "id": "<REMOVED>",
                    "pattern": "[ipv4-addr:value = '<REMOVED>']",
                    "pattern_type": "stix"
                }
            ],
            "indicatorsIds": [
                "<REMOVED>"
            ],
            "labels": [
                "exploited host",
                "hacking",
                "web app attack",
                "clear",
                "email spam",
                "brute force",
                "ssh",
                "spoofing",
                "port scan",
                "sighted"
            ],
            "object_marking_refs": [
                "marking-definition--<REMOVED>"
            ],
            "observable_value": "<REMOVED>",
            "parent_types": [
                "Basic-Object",
                "Stix-Object",
                "Stix-Core-Object",
                "Stix-Cyber-Observable"
            ],
            "spec_version": "2.1",
            "type": "ipv4-addr",
            "updated_at": "<REMOVED>",
            "value": "<REMOVED>",
            "x_opencti_id": "<REMOVED>",
            "x_opencti_score": 100
        },
        {
            "contact_information": "Org XYZ",
            "created": "<REMOVED>",
            "description": "Org XYZ",
            "id": "identity--<REMOVED>",
            "identity_class": "organization",
            "modified": "<REMOVED>",
            "name": "Org XYZ",
            "spec_version": "2.1",
            "type": "identity"
        },
        {
            "created_by_ref": {
                "contact_information": "Org XYZ",
                "created": "<REMOVED>",
                "description": "Org XYZ",
                "id": "identity--<REMOVED>",
                "identity_class": "organization",
                "modified": "<REMOVED>",
                "name": "Org XYZ",
                "spec_version": "2.1",
                "type": "identity"
            },
            "dst_port": 3389,
            "end": "<REMOVED>",
            "extensions": {},
            "id": "network-traffic--<REMOVED>",
            "is_active": false,
            "labels": [
                "sighted",
                "sighting:type:network",
                "sighting:direction:in",
                "sighting:action:block",
                "sighting:app:tls"
            ],
            "object_marking_refs": [
                "marking-definition--<REMOVED>"
            ],
            "protocols": [
                "ipv4",
                "tcp"
            ],
            "spec_version": "2.1",
            "src_port": 50479,
            "src_ref": "ipv4-addr--<REMOVED>",
            "start": "<REMOVED>",
            "type": "network-traffic",
            "x_opencti_description": ""
        },
        {
            "confidence": 80,
            "created": "<REMOVED>",
            "createdBy": {
                "contact_information": null,
                "created": "<REMOVED>",
                "description": "",
                "entity_type": "Organization",
                "id": "<REMOVED>",
                "identity_class": "organization",
                "modified": "<REMOVED>",
                "name": "feedxyz",
                "objectLabel": [],
                "objectLabelIds": [],
                "parent_types": [
                    "Basic-Object",
                    "Stix-Object",
                    "Stix-Core-Object",
                    "Stix-Domain-Object",
                    "Identity"
                ],
                "roles": null,
                "spec_version": "2.1",
                "standard_id": "identity--<REMOVED>",
                "x_opencti_aliases": null,
                "x_opencti_organization_type": null,
                "x_opencti_reliability": null
            },
            "createdById": "<REMOVED>",
            "created_at": "<REMOVED>",
            "description": "",
            "entity_type": "Indicator",
            "id": "indicator--<REMOVED>",
            "indicator_types": [
                "malicious-activity"
            ],
            "labels": [
                "port scan",
                "hacking",
                "brute force",
                "clear",
                "sighted"
            ],
            "modified": "<REMOVED>",
            "name": "<REMOVED>",
            "object_marking_refs": [
                "marking-definition--<REMOVED>"
            ],
            "observables": [
                {
                    "createdById": null,
                    "entity_type": "IPv4-Addr",
                    "id": "<REMOVED>",
                    "observable_value": "<REMOVED>"
                }
            ],
            "observablesIds": [
                "<REMOVED>"
            ],
            "parent_types": [
                "Basic-Object",
                "Stix-Object",
                "Stix-Core-Object",
                "Stix-Domain-Object"
            ],
            "pattern": "[ipv4-addr:value = '<REMOVED>']",
            "pattern_type": "stix",
            "pattern_version": "2.1",
            "revoked": true,
            "spec_version": "2.1",
            "type": "indicator",
            "updated_at": "<REMOVED>",
            "valid_from": "<REMOVED>",
            "valid_until": "<REMOVED>",
            "x_opencti_detection": true,
            "x_opencti_id": "<REMOVED>",
            "x_opencti_main_observable_type": "IPv4-Addr",
            "x_opencti_score": 60
        },
        {
            "createdBy": {
                "contact_information": null,
                "created": "<REMOVED>",
                "description": "",
                "entity_type": "Organization",
                "id": "<REMOVED>",
                "identity_class": "organization",
                "modified": "<REMOVED>",
                "name": "feedxyz",
                "objectLabel": [],
                "objectLabelIds": [],
                "parent_types": [
                    "Basic-Object",
                    "Stix-Object",
                    "Stix-Core-Object",
                    "Stix-Domain-Object",
                    "Identity"
                ],
                "roles": null,
                "spec_version": "2.1",
                "standard_id": "identity--<REMOVED>",
                "x_opencti_aliases": null,
                "x_opencti_organization_type": null,
                "x_opencti_reliability": null
            },
            "createdById": "<REMOVED>",
            "created_at": "<REMOVED>",
            "entity_type": "IPv4-Addr",
            "id": "ipv4-addr--<REMOVED>",
            "indicators": [
                {
                    "createdById": null,
                    "id": "<REMOVED>",
                    "pattern": "[ipv4-addr:value = '<REMOVED>']",
                    "pattern_type": "stix"
                }
            ],
            "indicatorsIds": [
                "<REMOVED>"
            ],
            "labels": [
                "exploited host",
                "hacking",
                "web app attack",
                "clear",
                "email spam",
                "brute force",
                "ssh",
                "spoofing",
                "port scan",
                "sighted"
            ],
            "object_marking_refs": [
                "marking-definition--<REMOVED>"
            ],
            "observable_value": "<REMOVED>",
            "parent_types": [
                "Basic-Object",
                "Stix-Object",
                "Stix-Core-Object",
                "Stix-Cyber-Observable"
            ],
            "spec_version": "2.1",
            "type": "ipv4-addr",
            "updated_at": "<REMOVED>",
            "value": "<REMOVED>",
            "x_opencti_id": "<REMOVED>",
            "x_opencti_score": 100
        },
        {
            "confidence": 100,
            "created": "<REMOVED>",
            "created_by_ref": "identity--<REMOVED>",
            "description": "brick2",
            "id": "identity--<REMOVED>",
            "identity_class": "system",
            "labels": [
                "intenal:system"
            ],
            "modified": "<REMOVED>",
            "name": "brick2",
            "object_marking_refs": [
                "marking-definition--<REMOVED>"
            ],
            "spec_version": "2.1",
            "type": "identity"
        },
        {
            "confidence": 100,
            "created": "<REMOVED>",
            "created_by_ref": "identity--<REMOVED>",
            "id": "relationship--<REMOVED>",
            "modified": "<REMOVED>",
            "object_marking_refs": [
                "marking-definition--<REMOVED>"
            ],
            "relationship_type": "based-on",
            "source_ref": "indicator--<REMOVED>",
            "spec_version": "2.1",
            "target_ref": "network-traffic--<REMOVED>",
            "type": "relationship"
        },
        {
            "confidence": 100,
            "created": "<REMOVED>",
            "created_by_ref": "identity--<REMOVED>",
            "first_observed": "<REMOVED>",
            "id": "observed-data--<REMOVED>",
            "labels": [],
            "last_observed": "<REMOVED>",
            "modified": "<REMOVED>",
            "number_observed": 69,
            "object_marking_refs": [
                "marking-definition--<REMOVED>"
            ],
            "object_refs": [
                "identity--<REMOVED>",
                "network-traffic--<REMOVED>",
                "indicator--<REMOVED>",
                "ipv4-addr--<REMOVED>",
                "identity--<REMOVED>",
                "relationship--<REMOVED>"
            ],
            "spec_version": "2.1",
            "type": "observed-data"
        },
        {
            "confidence": 100,
            "count": 1,
            "created": "<REMOVED>",
            "created_by_ref": "identity--<REMOVED>",
            "first_seen": "<REMOVED>",
            "id": "sighting--<REMOVED>",
            "last_seen": "<REMOVED>",
            "modified": "<REMOVED>",
            "observed_data_refs": [
                "observed-data--<REMOVED>"
            ],
            "sighting_of_ref": "indicator--<REMOVED>",
            "spec_version": "2.1",
            "type": "sighting",
            "where_sighted_refs": [
                "identity--<REMOVED>",
                "identity--<REMOVED>"
            ]
        }
    ],
    "type": "bundle"
}

Screenshots (optional)

richard-julien commented 1 year ago

Some invalid data in this bundle.

"created_by_ref": { "contact_information": "Org XYZ", "created": "<REMOVED>", "description": "Org XYZ", "id": "identity--<REMOVED>", "identity_class": "organization", "modified": "<REMOVED>", "name": "Org XYZ", "spec_version": "2.1", "type": "identity" }, Must be an ID instead of complex object.

"indicators": [ { "createdById": null, "id": "<REMOVED>", "pattern": "[ipv4-addr:value = '<REMOVED>']", "pattern_type": "stix" } ], "indicatorsIds": [ "<REMOVED>" ], Invalid attributes in STIX.

And some more :)