search

OpenCTI-Platform / opencti

Open Cyber Threat Intelligence Platform
https://opencti.io
Other
6.32k stars 932 forks source link

[Grakn] DateTime Error #546

Closed ghost closed 4 years ago

ghost commented 4 years ago

Description

The error occurs during the insertion of data in grakn

Environment

  1. OS (where OpenCTI server runs): Ubuntu 18.04
  2. OpenCTI version: 3.0.2
  3. OpenCTI client: python (worker)
  4. Other environment details:

docker-compose.yml: 

version: '3'
services:
  grakn:
    image: graknlabs/grakn:1.6.2
    ports:
      - 48555:48555
    volumes:
      - grakndata:/grakn-core-all-linux/server/db
  redis:
    image: redis:5.0.5
  elasticsearch:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.5.2
    volumes:
      - esdata:/usr/share/elasticsearch/data
    environment:
      - discovery.type=single-node
  minio:
    image: minio/minio:RELEASE.2019-10-12T01-39-57Z
    env_file: .env
    volumes:
      - s3data:/data
    ports:
      - "9000:9000"
    environment:
      MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY}
      MINIO_SECRET_KEY: ${MINIO_SECRET_KEY}
    command: server /data
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
      interval: 30s
      timeout: 20s
      retries: 3
  rabbitmq:
    image: rabbitmq:3.7.17-management
    env_file: .env
    environment:
      - RABBITMQ_DEFAULT_USER=${RABBITMQ_DEFAULT_USER}
      - RABBITMQ_DEFAULT_PASS=${RABBITMQ_DEFAULT_PASS}   
  opencti:
    image: opencti/platform:latest
    env_file: .env
    environment:
      - APP__PORT=8080
      - APP__ADMIN__EMAIL=${OPENCTI_ADMIN_EMAIL}
      - APP__ADMIN__PASSWORD=${OPENCTI_ADMIN_PASSWORD}
      - APP__ADMIN__TOKEN=${OPENCTI_ADMIN_TOKEN}
      - APP__LOGS_LEVEL=error
      - APP__LOGS=./logs
      - APP__REACTIVE=true
      - APP__COOKIE_SECURE=false
      - GRAKN__HOSTNAME=grakn
      - GRAKN__PORT=48555
      - GRAKN__TIMEOUT=30000
      - REDIS__HOSTNAME=redis
      - REDIS__PORT=6379
      - ELASTICSEARCH__URL=http://elasticsearch:9200
      - MINIO__ENDPOINT=minio
      - MINIO__PORT=9000
      - MINIO__USE_SSL=false
      - MINIO__ACCESS_KEY=${MINIO_ACCESS_KEY}
      - MINIO__SECRET_KEY=${MINIO_SECRET_KEY}
      - RABBITMQ__HOSTNAME=rabbitmq
      - RABBITMQ__PORT=5672
      - RABBITMQ__PORT_MANAGEMENT=15672
      - RABBITMQ__MANAGEMENT_SSL=false
      - RABBITMQ__USERNAME=${RABBITMQ_DEFAULT_USER}
      - RABBITMQ__PASSWORD=${RABBITMQ_DEFAULT_PASS}
      - PROVIDERS__LOCAL__STRATEGY=LocalStrategy
    ports:
      - "8080:8080"
    depends_on:
      - grakn
      - redis
      - elasticsearch
      - minio
      - rabbitmq
  worker:
    image: opencti/worker:latest
    env_file: .env
    environment:
      - OPENCTI_URL=${OPENCTI_URL}
      - OPENCTI_TOKEN=${OPENCTI_ADMIN_TOKEN}
      - WORKER_LOG_LEVEL=info
    depends_on:
      - opencti
    deploy:
      mode: replicated
      replicas: 4
  connector-export-file-stix:
    image: opencti/connector-export-file-stix:latest
    env_file: .env
    environment:
      - OPENCTI_URL=${OPENCTI_URL}
      - OPENCTI_TOKEN=${OPENCTI_ADMIN_TOKEN}
      - CONNECTOR_ID=${CONNECTOR_EXPORT_FILE_STIX_ID} # Valid UUDv4
      - CONNECTOR_TYPE=INTERNAL_EXPORT_FILE
      - CONNECTOR_NAME=ExportFileStix2
      - CONNECTOR_SCOPE=application/json
      - CONNECTOR_CONFIDENCE_LEVEL=3
      - CONNECTOR_LOG_LEVEL=info
  connector-export-file-csv:
    image: opencti/connector-export-file-csv:latest
    env_file: .env
    environment:
      - OPENCTI_URL=${OPENCTI_URL}
      - OPENCTI_TOKEN=${OPENCTI_ADMIN_TOKEN}
      - CONNECTOR_ID=${CONNECTOR_EXPORT_FILE_CSV_ID} # Valid UUDv4
      - CONNECTOR_TYPE=INTERNAL_EXPORT_FILE
      - CONNECTOR_NAME=ExportFileCsv
      - CONNECTOR_SCOPE=application/csv
      - CONNECTOR_CONFIDENCE_LEVEL=3
      - CONNECTOR_LOG_LEVEL=info
  connector-import-file-stix:
    image: opencti/connector-import-file-stix:latest
    env_file: .env
    environment:
      - OPENCTI_URL=${OPENCTI_URL}
      - OPENCTI_TOKEN=${OPENCTI_ADMIN_TOKEN}
      - CONNECTOR_ID=${CONNECTOR_IMPORT_FILE_STIX_ID} # Valid UUDv4
      - CONNECTOR_TYPE=INTERNAL_IMPORT_FILE
      - CONNECTOR_NAME=ImportFileStix2
      - CONNECTOR_SCOPE=application/json
      - CONNECTOR_CONFIDENCE_LEVEL=3
      - CONNECTOR_LOG_LEVEL=info
  connector-import-file-pdf-observables:
    image: opencti/connector-import-file-pdf-observables:latest
    env_file: .env
    environment:
      - OPENCTI_URL=${OPENCTI_URL}
      - OPENCTI_TOKEN=${OPENCTI_ADMIN_TOKEN}
      - CONNECTOR_ID=${CONNECTOR_IMPORT_FILE_PDF_OBSERVABLES_ID} # Valid UUDv4
      - CONNECTOR_TYPE=INTERNAL_IMPORT_FILE
      - CONNECTOR_NAME=ImportFilePdfObservables
      - CONNECTOR_SCOPE=application/pdf
      - CONNECTOR_CONFIDENCE_LEVEL=3
      - CONNECTOR_LOG_LEVEL=info
      - PDF_OBSERVABLES_CREATE_INDICATOR=False
  connector-opencti:
    image: opencti/connector-opencti:latest
    env_file: .env
    environment:
      - OPENCTI_URL=${OPENCTI_URL}
      - OPENCTI_TOKEN=${OPENCTI_ADMIN_TOKEN}
      - CONNECTOR_ID=${CONNECTOR_OPENCTI_ID} # Valid UUDv4
      - CONNECTOR_TYPE=EXTERNAL_IMPORT
      - CONNECTOR_NAME=OpenCTI
      - CONNECTOR_SCOPE=identity,sector,region,country,city
      - CONNECTOR_CONFIDENCE_LEVEL=5
      - CONNECTOR_UPDATE_EXISTING_DATA=true
      - CONNECTOR_LOG_LEVEL=info
      - CONFIG_SECTORS_FILE_URL=https://raw.githubusercontent.com/OpenCTI-Platform/datasets/master/data/sectors.json
      - CONFIG_GEOGRAPHY_FILE_URL=https://raw.githubusercontent.com/OpenCTI-Platform/datasets/master/data/geography.json
      - CONFIG_INTERVAL=7 # Days
  connector-mitre:
    image: opencti/connector-mitre:latest
    env_file: .env
    environment:
      - OPENCTI_URL=${OPENCTI_URL}
      - OPENCTI_TOKEN=${OPENCTI_ADMIN_TOKEN}
      - CONNECTOR_ID=${CONNECTOR_MITRE_ID} # Valid UUDv4
      - CONNECTOR_TYPE=EXTERNAL_IMPORT
      - CONNECTOR_NAME=MITRE ATT&CK
      - CONNECTOR_SCOPE=identity,attack-pattern,course-of-action,intrusion-set,malware,tool,report
      - CONNECTOR_CONFIDENCE_LEVEL=3
      - CONNECTOR_UPDATE_EXISTING_DATA=true
      - CONNECTOR_LOG_LEVEL=info
      - MITRE_ENTERPRISE_FILE_URL=https://raw.githubusercontent.com/mitre/cti/master/enterprise-attack/enterprise-attack.json
      - MITRE_PRE_ATTACK_FILE_URL=https://raw.githubusercontent.com/mitre/cti/master/pre-attack/pre-attack.json
      - MITRE_INTERVAL=7 # Days
  connector-cve:
    image: opencti/connector-cve:latest
    env_file: .env
    environment:
      - OPENCTI_URL=${OPENCTI_URL}
      - OPENCTI_TOKEN=${OPENCTI_ADMIN_TOKEN}
      - CONNECTOR_ID=${CONNECTOR_CVE_ID}
      - CONNECTOR_TYPE=EXTERNAL_IMPORT
      - CONNECTOR_NAME=Common Vulnerabilities and Exposures
      - CONNECTOR_SCOPE=identity,vulnerability
      - CONNECTOR_CONFIDENCE_LEVEL=3
      - CONNECTOR_UPDATE_EXISTING_DATA=true
      - CONNECTOR_LOG_LEVEL=info
      - CVE_NVD_DATA_FEED=https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-recent.json.gz
      - CVE_INTERVAL=7 # Days
  connector-ipinfo:
    image: opencti/connector-ipinfo:latest
    env_file: .env
    environment:
      - OPENCTI_URL=${OPENCTI_URL}
      - OPENCTI_TOKEN=${OPENCTI_ADMIN_TOKEN}
      - CONNECTOR_ID=${CONNECTOR_IPINFO_ID}
      - CONNECTOR_TYPE=INTERNAL_ENRICHMENT
      - CONNECTOR_NAME=IpInfo
      - CONNECTOR_SCOPE=ipv4-addr
      - CONNECTOR_CONFIDENCE_LEVEL=3
      - CONNECTOR_LOG_LEVEL=info
      - IPINFO_TOKEN=${CONNECTOR_IPINFO_TOKEN}
  connector-amitt:
    image: opencti/connector-amitt:latest
    env_file: .env
    environment:
      - OPENCTI_URL=${OPENCTI_URL}
      - OPENCTI_TOKEN=${OPENCTI_ADMIN_TOKEN}
      - CONNECTOR_ID=${CONNECTOR_AMITT_ID}
      - CONNECTOR_TYPE=EXTERNAL_IMPORT
      - CONNECTOR_NAME=AM!TT
      - CONNECTOR_SCOPE=identity,attack-pattern,course-of-action,intrusion-set,malware,tool,report
      - CONNECTOR_CONFIDENCE_LEVEL=3
      - CONNECTOR_UPDATE_EXISTING_DATA=true
      - CONNECTOR_LOG_LEVEL=info
      - AMITT_FILE_URL=https://raw.githubusercontent.com/VVX7/cti/master/amitt-attack/amitt-attack.json
      - PRE_AMITT_FILE_URL=https://raw.githubusercontent.com/VVX7/cti/master/amitt-pre-attack/amitt-pre-attack.json
      - AMITT_INTERVAL=7 # Days
  connector-misp:
    image: opencti/connector-misp:latest
    env_file: .env
    environment:
      - OPENCTI_URL=${OPENCTI_URL}
      - OPENCTI_TOKEN=${OPENCTI_ADMIN_TOKEN}
      - CONNECTOR_ID=${CONNECTOR_MISP_ID}
      - CONNECTOR_TYPE=EXTERNAL_IMPORT
      - CONNECTOR_NAME=MISP
      - CONNECTOR_SCOPE=misp
      - CONNECTOR_CONFIDENCE_LEVEL=3
      - CONNECTOR_UPDATE_EXISTING_DATA=false
      - CONNECTOR_LOG_LEVEL=info
      - MISP_URL=${CONNECTOR_MISP_URL} # Required
      - MISP_KEY=${CONNECTOR_MISP_KEY} # Required
      - MISP_SSL_VERIFY=False # Required
      - MISP_CREATE_REPORTS=True # Required, create report for MISP event
      - MISP_REPORT_CLASS=MISP event # Optional, report_class if creating report for event
      - MISP_IMPORT_FROM_DATE=2010-01-01 # Optional, import all event from this date
      - MISP_INTERVAL=1 # Required, in minutes
  connector-virustotal:
    image: opencti/connector-virustotal:latest
    env_file: .env
    environment:
      - OPENCTI_URL=${OPENCTI_URL}
      - OPENCTI_TOKEN=${OPENCTI_ADMIN_TOKEN}
      - CONNECTOR_ID=${CONNECTOR_VIRUSTOTAL_ID}
      - CONNECTOR_TYPE=INTERNAL_ENRICHMENT
      - CONNECTOR_NAME=VirusTotal
      - CONNECTOR_SCOPE=file-md5,file-sha1,file-sha256,file-imphash,file-pehash
      - CONNECTOR_CONFIDENCE_LEVEL=3
      - CONNECTOR_LOG_LEVEL=info
      - VIRUSTOTAL_TOKEN=${CONNECTOR_VIRUSTOTAL_TOKEN}
      - VIRUSTOTAL_MAX_TLP=TLP:AMBER
  connector-alienvault:
    image: opencti/connector-alienvault:latest
    env_file: .env
    environment:
      - OPENCTI_URL=${OPENCTI_URL}
      - OPENCTI_TOKEN=${OPENCTI_ADMIN_TOKEN}
      - CONNECTOR_ID=${CONNECTOR_ALIENVAULT_ID}
      - CONNECTOR_TYPE=EXTERNAL_IMPORT
      - CONNECTOR_NAME=AlienVault OTX
      - CONNECTOR_SCOPE=alienvault
      - CONNECTOR_CONFIDENCE_LEVEL=3
      - CONNECTOR_UPDATE_EXISTING_DATA=false
      - CONNECTOR_LOG_LEVEL=info
      - ALIENVAULT_BASE_URL=https://otx.alienvault.com
      - ALIENVAULT_API_KEY=${CONNECTOR_ALIENVAULT_API_KEY}
      - ALIENVAULT_TLP=White
      - ALIENVAULT_PULSE_START_TIMESTAMP=2020-01-01T00:00:00  # BEWARE! Could be a lot of pulses!
      - ALIENVAULT_REPORT_TYPE=Threat Report
      - ALIENVAULT_REPORT_STATUS=Analyzed
      - ALIENVAULT_INTERVAL_SEC=1800
volumes:
  grakndata:
  esdata:
  s3data:

Reproducible Steps

No steps, this appear after all services are up and workers are consuming MISP events or other events properly.

Expected Output

None.

Actual Output

opencti_opencti.1.fykt3yc1ljzw@ip-10-0-2-7    | error: [OPENCTI] Technical error > Variable "$input" got invalid value "2019-10-17T13:12:54.049499" at "input.modified"; Expected type DateTime. DateTime cannot represent an invalid date-time-string 2019-10-17T13:12:54.049499. 
{
  "locations": [
    {
      "line": 2,
      "column": 38
    }
  ],
  "extensions": {
    "code": "INTERNAL_SERVER_ERROR",
    "exception": {
      "stacktrace": [
        "TypeError: DateTime cannot represent an invalid date-time-string 2019-10-17T13:12:54.049499.",
        "    at GraphQLScalarType.parseValue (/opt/opencti/node_modules/graphql-iso-date/dist/dateTime/index.js:64:11)",
        "    at coerceInputValueImpl (/opt/opencti/node_modules/graphql/utilities/coerceInputValue.js:127:26)",
        "    at coerceInputValueImpl (/opt/opencti/node_modules/graphql/utilities/coerceInputValue.js:105:35)",
        "    at coerceInputValue (/opt/opencti/node_modules/graphql/utilities/coerceInputValue.js:37:10)",
        "    at _loop (/opt/opencti/node_modules/graphql/execution/values.js:107:69)",
        "    at coerceVariableValues (/opt/opencti/node_modules/graphql/execution/values.js:119:16)",
        "    at getVariableValues (/opt/opencti/node_modules/graphql/execution/values.js:48:19)",
        "    at buildExecutionContext (/opt/opencti/node_modules/graphql/execution/execute.js:184:61)",
        "    at executeImpl (/opt/opencti/node_modules/graphql/execution/execute.js:89:20)",
        "    at Object.execute (/opt/opencti/node_modules/graphql/execution/execute.js:64:35)",
        "    at /opt/opencti/node_modules/apollo-server-core/dist/requestPipeline.js:246:46",
        "    at Generator.next (<anonymous>)",
        "    at /opt/opencti/node_modules/apollo-server-core/dist/requestPipeline.js:8:71",
        "    at new Promise (<anonymous>)",
        "    at __awaiter (/opt/opencti/node_modules/apollo-server-core/dist/requestPipeline.js:4:12)",
        "    at execute (/opt/opencti/node_modules/apollo-server-core/dist/requestPipeline.js:225:20)"
      ]
    }
  }
}

opencti_opencti.1.fykt3yc1ljzw@ip-10-0-2-7    | error: [OPENCTI] Technical error > Variable "$input" got invalid value "2019-10-17T13:12:54.049499" at "input.created"; Expected type DateTime. DateTime cannot represent an invalid date-time-string 2019-10-17T13:12:54.049499. 
{
  "locations": [
    {
      "line": 2,
      "column": 47
    }
  ],
  "extensions": {
    "code": "INTERNAL_SERVER_ERROR",
    "exception": {
      "stacktrace": [
        "TypeError: DateTime cannot represent an invalid date-time-string 2019-10-17T13:12:54.049499.",
        "    at GraphQLScalarType.parseValue (/opt/opencti/node_modules/graphql-iso-date/dist/dateTime/index.js:64:11)",
        "    at coerceInputValueImpl (/opt/opencti/node_modules/graphql/utilities/coerceInputValue.js:127:26)",
        "    at coerceInputValueImpl (/opt/opencti/node_modules/graphql/utilities/coerceInputValue.js:105:35)",
        "    at coerceInputValue (/opt/opencti/node_modules/graphql/utilities/coerceInputValue.js:37:10)",
        "    at _loop (/opt/opencti/node_modules/graphql/execution/values.js:107:69)",
        "    at coerceVariableValues (/opt/opencti/node_modules/graphql/execution/values.js:119:16)",
        "    at getVariableValues (/opt/opencti/node_modules/graphql/execution/values.js:48:19)",
        "    at buildExecutionContext (/opt/opencti/node_modules/graphql/execution/execute.js:184:61)",
        "    at executeImpl (/opt/opencti/node_modules/graphql/execution/execute.js:89:20)",
        "    at Object.execute (/opt/opencti/node_modules/graphql/execution/execute.js:64:35)",
        "    at /opt/opencti/node_modules/apollo-server-core/dist/requestPipeline.js:246:46",
        "    at Generator.next (<anonymous>)",
        "    at /opt/opencti/node_modules/apollo-server-core/dist/requestPipeline.js:8:71",
        "    at new Promise (<anonymous>)",
        "    at __awaiter (/opt/opencti/node_modules/apollo-server-core/dist/requestPipeline.js:4:12)",
        "    at execute (/opt/opencti/node_modules/apollo-server-core/dist/requestPipeline.js:225:20)"
      ]
    }
  }
}

Additional information

I'm using docker swarm, and all it's works, but this error probably will be inserting false dates.

richard-julien commented 4 years ago

Hi @EmWhyAitch ,

Looks like data coming from the connector have invalid date. Basically the error 

"TypeError: DateTime cannot represent an invalid date-time-string 2019-10-17T13:12:54.049499.",
        "    at GraphQLScalarType.parseValue (/opt/opencti/node_modules/graphql-iso-date/dist/dateTime/index.js:64:11)",

is directly throw by the API because 2019-10-17T13:12:54.049499 is an invalid date. We expect to receive an ISO format. Because the API reject the date, the complete mutation is rejected, so you will not have any "false" dates, but maybe some missing entities.

We plan to improve the traceability of import (timing, error, partial success) etc but for now you have to dig into the data you fetch from your external providers to find the data that have this kind of bad date definition.