Open anavitgo opened 1 month ago
Hello @anavitgo. Are your connectors deployed in the same docker context as OpenCTI? If your connectors are configured to point to “http://opencti:8080”, they should run in the same docker context as OpenCTI.
I assume they are in the same context, because I can do ping opencti
from the alien vault container and the ping is successful but it keeps giving me the error that Max retries exceeded with url: /graphql
Sorry, I clicked on close by mistake. I tried also deploying it using the helm charts in argo but the alien vault container keeps crashing because of SIGTERM (error code 143). On the other side, while trying to set this up and running locally using docker, I am using docker rootless. What do you think might be the problem for this?
Sorry for the multiple comments. I have other connectors up and running like Mitre and AbuseCH but only alien vault crashes and can't talk to opencti. I will expose my docker compose here:
version: '3'
networks:
opencti-network:
driver: bridge
services:
redis:
networks:
- opencti-network
image: redis:7.2.5
restart: always
volumes:
- redisdata:/data
elasticsearch:
networks:
- opencti-network
image: elasticsearch/elasticsearch:8.13.4
volumes:
- esdata:/usr/share/elasticsearch/data
environment:
# Comment-out the line below for a cluster of multiple nodes
- discovery.type=single-node
# Uncomment the line below below for a cluster of multiple nodes
# - cluster.name=docker-cluster
- xpack.ml.enabled=false
- xpack.security.enabled=false
- thread_pool.search.queue_size=5000
- logger.org.elasticsearch.discovery="ERROR"
- "ES_JAVA_OPTS=-Xms${ELASTIC_MEMORY_SIZE} -Xmx${ELASTIC_MEMORY_SIZE}"
restart: always
# ulimits:
# memlock:
# soft: -1
# hard: -1
# nofile:
# soft: 65536
# hard: 65536
minio:
networks:
- opencti-network
image: minio/minio:RELEASE.2024-05-28T17-19-04Z-cpuv1 # Use "minio/minio:RELEASE.2024-05-28T17-19-04Z-cpuv1" to troubleshoot compatibility issues with CPU
volumes:
- s3data:/data
ports:
- "9000:9000"
environment:
MINIO_ROOT_USER: ${MINIO_ROOT_USER}
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD}
command: server /data
restart: always
rabbitmq:
networks:
- opencti-network
image: rabbitmq:3.13-management
environment:
- RABBITMQ_DEFAULT_USER=${RABBITMQ_DEFAULT_USER}
- RABBITMQ_DEFAULT_PASS=${RABBITMQ_DEFAULT_PASS}
- RABBITMQ_NODENAME=rabbit01@localhost
volumes:
- amqpdata:/var/lib/rabbitmq
restart: always
opencti:
networks:
- opencti-network
image: opencti/platform:6.2.5
environment:
- NODE_OPTIONS=--max-old-space-size=8096
- APP__PORT=8080
- APP__BASE_URL=${OPENCTI_BASE_URL}
- APP__ADMIN__EMAIL=${OPENCTI_ADMIN_EMAIL}
- APP__ADMIN__PASSWORD=${OPENCTI_ADMIN_PASSWORD}
- APP__ADMIN__TOKEN=${OPENCTI_ADMIN_TOKEN}
- APP__APP_LOGS__LOGS_LEVEL=error
- REDIS__HOSTNAME=redis
- REDIS__PORT=6379
- ELASTICSEARCH__URL=http://elasticsearch:9200
- MINIO__ENDPOINT=minio
- MINIO__PORT=9000
- MINIO__USE_SSL=false
- MINIO__ACCESS_KEY=${MINIO_ROOT_USER}
- MINIO__SECRET_KEY=${MINIO_ROOT_PASSWORD}
- RABBITMQ__HOSTNAME=rabbitmq
- RABBITMQ__PORT=5672
- RABBITMQ__PORT_MANAGEMENT=15672
- RABBITMQ__MANAGEMENT_SSL=false
- RABBITMQ__USERNAME=${RABBITMQ_DEFAULT_USER}
- RABBITMQ__PASSWORD=${RABBITMQ_DEFAULT_PASS}
- SMTP__HOSTNAME=${SMTP_HOSTNAME}
- SMTP__PORT=25
- PROVIDERS__LOCAL__STRATEGY=LocalStrategy
ports:
- "8080:8080"
depends_on:
- redis
- elasticsearch
- minio
- rabbitmq
restart: always
worker:
networks:
- opencti-network
image: opencti/worker:6.2.5
environment:
- OPENCTI_URL=http://opencti:8080
- OPENCTI_TOKEN=${OPENCTI_ADMIN_TOKEN}
- WORKER_LOG_LEVEL=info
depends_on:
- opencti
deploy:
mode: replicated
replicas: 3
restart: always
connector-export-file-stix:
networks:
- opencti-network
image: opencti/connector-export-file-stix:6.2.5
environment:
- OPENCTI_URL=http://opencti:8080
- OPENCTI_TOKEN=${OPENCTI_ADMIN_TOKEN}
- CONNECTOR_ID=${CONNECTOR_EXPORT_FILE_STIX_ID} # Valid UUIDv4
- CONNECTOR_TYPE=INTERNAL_EXPORT_FILE
- CONNECTOR_NAME=ExportFileStix2
- CONNECTOR_SCOPE=application/json
- CONNECTOR_LOG_LEVEL=info
restart: always
depends_on:
- opencti
connector-export-file-csv:
networks:
- opencti-network
image: opencti/connector-export-file-csv:6.2.5
environment:
- OPENCTI_URL=http://opencti:8080
- OPENCTI_TOKEN=${OPENCTI_ADMIN_TOKEN}
- CONNECTOR_ID=${CONNECTOR_EXPORT_FILE_CSV_ID} # Valid UUIDv4
- CONNECTOR_TYPE=INTERNAL_EXPORT_FILE
- CONNECTOR_NAME=ExportFileCsv
- CONNECTOR_SCOPE=text/csv
- CONNECTOR_LOG_LEVEL=info
restart: always
depends_on:
- opencti
connector-export-file-txt:
networks:
- opencti-network
image: opencti/connector-export-file-txt:6.2.5
environment:
- OPENCTI_URL=http://opencti:8080
- OPENCTI_TOKEN=${OPENCTI_ADMIN_TOKEN}
- CONNECTOR_ID=${CONNECTOR_EXPORT_FILE_TXT_ID} # Valid UUIDv4
- CONNECTOR_TYPE=INTERNAL_EXPORT_FILE
- CONNECTOR_NAME=ExportFileTxt
- CONNECTOR_SCOPE=text/plain
- CONNECTOR_LOG_LEVEL=info
restart: always
depends_on:
- opencti
connector-import-file-stix:
networks:
- opencti-network
image: opencti/connector-import-file-stix:6.2.5
environment:
- OPENCTI_URL=http://opencti:8080
- OPENCTI_TOKEN=${OPENCTI_ADMIN_TOKEN}
- CONNECTOR_ID=${CONNECTOR_IMPORT_FILE_STIX_ID} # Valid UUIDv4
- CONNECTOR_TYPE=INTERNAL_IMPORT_FILE
- CONNECTOR_NAME=ImportFileStix
- CONNECTOR_VALIDATE_BEFORE_IMPORT=true # Validate any bundle before import
- CONNECTOR_SCOPE=application/json,text/xml
- CONNECTOR_AUTO=true # Enable/disable auto-import of file
- CONNECTOR_LOG_LEVEL=info
restart: always
depends_on:
- opencti
connector-import-document:
networks:
- opencti-network
image: opencti/connector-import-document:6.2.5
environment:
- OPENCTI_URL=http://opencti:8080
- OPENCTI_TOKEN=${OPENCTI_ADMIN_TOKEN}
- CONNECTOR_ID=${CONNECTOR_IMPORT_DOCUMENT_ID} # Valid UUIDv4
- CONNECTOR_TYPE=INTERNAL_IMPORT_FILE
- CONNECTOR_NAME=ImportDocument
- CONNECTOR_VALIDATE_BEFORE_IMPORT=true # Validate any bundle before import
- CONNECTOR_SCOPE=application/pdf,text/plain,text/html
- CONNECTOR_AUTO=true # Enable/disable auto-import of file
- CONNECTOR_ONLY_CONTEXTUAL=false # Only extract data related to an entity (a report, a threat actor, etc.)
- CONNECTOR_CONFIDENCE_LEVEL=15 # From 0 (Unknown) to 100 (Fully trusted)
- CONNECTOR_LOG_LEVEL=info
- IMPORT_DOCUMENT_CREATE_INDICATOR=true
restart: always
depends_on:
- opencti
connector-analysis:
networks:
- opencti-network
image: opencti/connector-import-document:6.2.5
environment:
- OPENCTI_URL=http://opencti:8080
- OPENCTI_TOKEN=${OPENCTI_ADMIN_TOKEN}
- CONNECTOR_ID=${CONNECTOR_ANALYSIS_ID} # Valid UUIDv4
- CONNECTOR_TYPE=INTERNAL_ANALYSIS
- CONNECTOR_NAME=ImportDocumentAnalysis
- CONNECTOR_VALIDATE_BEFORE_IMPORT=false # Validate any bundle before import
- CONNECTOR_SCOPE=application/pdf,text/plain,text/html
- CONNECTOR_AUTO=true # Enable/disable auto-import of file
- CONNECTOR_ONLY_CONTEXTUAL=false # Only extract data related to an entity (a report, a threat actor, etc.)
- CONNECTOR_CONFIDENCE_LEVEL=15 # From 0 (Unknown) to 100 (Fully trusted)
- CONNECTOR_LOG_LEVEL=info
restart: always
depends_on:
- opencti
connector-alienvault:
networks:
- opencti-network
build:
context: .
dockerfile: Dockerfile
environment:
- OPENCTI_URL=http://opencti:8080
- OPENCTI_TOKEN=${OPENCTI_ADMIN_TOKEN}
- CONNECTOR_ID=4fe10786-3b0d-4af0-97e9-549f3040264b
- CONNECTOR_NAME=AlienVault
- CONNECTOR_SCOPE=alienvault
- CONNECTOR_LOG_LEVEL=error
- ALIENVAULT_BASE_URL=https://otx.alienvault.com
- ALIENVAULT_API_KEY=2b3350e1c02b4ca5e61ed081556b6ffb38acb5b246ea27d392878bfc6ad97ff6
- ALIENVAULT_TLP=White
- ALIENVAULT_CREATE_OBSERVABLES=true
- ALIENVAULT_CREATE_INDICATORS=true
- ALIENVAULT_PULSE_START_TIMESTAMP=2024-06-01T00:00:00 # BEWARE! Could be a lot of pulses!
- ALIENVAULT_REPORT_TYPE=threat-report
- ALIENVAULT_REPORT_STATUS=New
- ALIENVAULT_GUESS_MALWARE=false # Use tags to guess malware.
- ALIENVAULT_GUESS_CVE=false # Use tags to guess CVE.
- ALIENVAULT_EXCLUDED_PULSE_INDICATOR_TYPES=FileHash-MD5,FileHash-SHA1 # Excluded Pulse indicator types.
- ALIENVAULT_ENABLE_RELATIONSHIPS=true # Enable/Disable relationship creation between SDOs.
- ALIENVAULT_ENABLE_ATTACK_PATTERNS_INDICATES=false # Enable/Disable "indicates" relationships between indicators and attack patterns
- ALIENVAULT_INTERVAL_SEC=1800
restart: always
depends_on:
- opencti
connector-abuseipdb-ipblacklist:
networks:
- opencti-network
build:
context: .
dockerfile: Dockerfile.abuse
environment:
- OPENCTI_URL=http://opencti:8080
- OPENCTI_TOKEN=${OPENCTI_ADMIN_TOKEN}
- CONNECTOR_ID=3ad51a3b-f549-4f8b-88f9-4486d408d182 # Valid UUIDv4
- "CONNECTOR_NAME=AbuseIPDB IP Blacklist"
- CONNECTOR_SCOPE=abuseipdb
- CONNECTOR_LOG_LEVEL=error
- ABUSEIPDB_URL=https://api.abuseipdb.com/api/v2/blacklist
- ABUSEIPDB_API_KEY=bb31c9330147be0dfc271b2847a33cfbaa4b254d2d93230a156e89ea76daf6d5af07e7d98e5a005c
- ABUSEIPDB_SCORE=100
- ABUSEIPDB_LIMIT=10000
- ABUSEIPDB_INTERVAL=2 #Day
restart: always
connector-mitre:
networks:
- opencti-network
image: opencti/connector-mitre:6.2.7
environment:
- OPENCTI_URL=http://opencti:8080
- OPENCTI_TOKEN=${OPENCTI_ADMIN_TOKEN}
- CONNECTOR_ID=943002bf-c8e6-444f-89d5-2d22ed815031
- "CONNECTOR_NAME=MITRE Datasets"
- CONNECTOR_SCOPE=tool,report,malware,identity,campaign,intrusion-set,attack-pattern,course-of-action,x-mitre-data-source,x-mitre-data-component,x-mitre-matrix,x-mitre-tactic,x-mitre-collection
- CONNECTOR_RUN_AND_TERMINATE=false
- CONNECTOR_LOG_LEVEL=error
- MITRE_REMOVE_STATEMENT_MARKING=true
- MITRE_INTERVAL=1 # In days
restart: always
depends_on:
- opencti
connector-mitre-atlas:
networks:
- opencti-network
image: opencti/connector-mitre-atlas:6.2.7
environment:
- OPENCTI_URL=http://opencti:8080
- OPENCTI_TOKEN=${OPENCTI_ADMIN_TOKEN}
- CONNECTOR_ID=85120fce-f5df-4899-96e6-ab7d7cb99f22
- "CONNECTOR_NAME=MITRE ATLAS"
- CONNECTOR_SCOPE=identity,attack-pattern,course-of-action,relationship,x-mitre-collection,x-mitre-matrix,x-mitre-tactic
- CONNECTOR_CONFIDENCE_LEVEL=75 # From 0 (Unknown) to 100 (Fully trusted)
- CONNECTOR_RUN_AND_TERMINATE=false
- CONNECTOR_LOG_LEVEL=error
- MITRE_ATLAS_URL=https://raw.githubusercontent.com/mitre-atlas/atlas-navigator-data/main/dist/stix-atlas.json
- MITRE_ATLAS_INTERVAL=1 # In days, must be strictly greater than 1
restart: always
connector-virustotal-livehunt-notifications:
networks:
- opencti-network
image: opencti/connector-virustotal-livehunt-notifications:6.2.7
environment:
- OPENCTI_URL=http://opencti:8080
- OPENCTI_TOKEN=${OPENCTI_ADMIN_TOKEN}
- CONNECTOR_ID=Virustotal_Livehunt_Notifications
- "CONNECTOR_NAME=VirusTotal Livehunt Notifications"
- CONNECTOR_SCOPE=StixFile,Indicator,Incident
- CONNECTOR_LOG_LEVEL=error
- VIRUSTOTAL_LIVEHUNT_NOTIFICATIONS_API_KEY=fe3c9cff06b54d52cf25bfbab1721a9408e5c3dde4e7ced52b7539bf07be88db # Private API Key
- VIRUSTOTAL_LIVEHUNT_NOTIFICATIONS_INTERVAL_SEC=300 # Time to wait in seconds between subsequent requests
- VIRUSTOTAL_LIVEHUNT_NOTIFICATIONS_CREATE_ALERT=True # Set to true to create alerts
- VIRUSTOTAL_LIVEHUNT_NOTIFICATIONS_EXTENSIONS='exe,dll' # (Optional) Comma separated filter to only download files matching these extensions
- VIRUSTOTAL_LIVEHUNT_NOTIFICATIONS_MIN_FILE_SIZE=1000 # (Optional) Don't download files smaller than this many bytes
- VIRUSTOTAL_LIVEHUNT_NOTIFICATIONS_MAX_FILE_SIZE=52428800 # (Optional) Don't download files larger than this many bytes
- VIRUSTOTAL_LIVEHUNT_NOTIFICATIONS_MAX_AGE_DAYS=3 # Only create the alert if the first submission of the file is not older than `max_age_days`
- VIRUSTOTAL_LIVEHUNT_NOTIFICATIONS_MIN_POSITIVES=5 # (Optional) Don't download files with less than this many vendors marking malicious
- VIRUSTOTAL_LIVEHUNT_NOTIFICATIONS_CREATE_FILE=True # Set to true to create file object linked to the alerts
- VIRUSTOTAL_LIVEHUNT_NOTIFICATIONS_UPLOAD_ARTIFACT=False # Set to true to upload the file to opencti
- VIRUSTOTAL_LIVEHUNT_NOTIFICATIONS_CREATE_YARA_RULE=True # Set to true to create yara rule linked to the alert and the file
- VIRUSTOTAL_LIVEHUNT_NOTIFICATIONS_DELETE_NOTIFICATION=False # Set to true to remove livehunt notifications
- VIRUSTOTAL_LIVEHUNT_NOTIFICATIONS_FILTER_WITH_TAG="infosec" # Filter livehunt notifications with this tag
restart: always
depends_on:
- opencti
connector-opencti:
networks:
- opencti-network
image: opencti/connector-opencti:6.2.7
environment:
- OPENCTI_URL=http://opencti:8080
- OPENCTI_TOKEN=${OPENCTI_ADMIN_TOKEN}
- CONNECTOR_ID=8d441794-1f49-460e-b176-ea5cb97003c0
- "CONNECTOR_NAME=OpenCTI Datasets"
- CONNECTOR_SCOPE=marking-definition,identity,location
- CONNECTOR_CONFIDENCE_LEVEL=100
- CONNECTOR_UPDATE_EXISTING_DATA=true
- CONNECTOR_RUN_AND_TERMINATE=false
- CONNECTOR_LOG_LEVEL=error
- CONFIG_SECTORS_FILE_URL=https://raw.githubusercontent.com/OpenCTI-Platform/datasets/master/data/sectors.json
- CONFIG_GEOGRAPHY_FILE_URL=https://raw.githubusercontent.com/OpenCTI-Platform/datasets/master/data/geography.json
- CONFIG_COMPANIES_FILE_URL=https://raw.githubusercontent.com/OpenCTI-Platform/datasets/master/data/companies.json
- CONFIG_REMOVE_CREATOR=false
- CONFIG_INTERVAL=1 # In days
restart: always
depends_on:
- opencti
volumes:
esdata:
s3data:
redisdata:
amqpdata:
The connectors that are using the Dockerfile it was me that git cloned the whole repo and build the image locally for troubleshooting, but the problem with alien vault is persistent: I've tried building it locally, I've tried using the already build image referenced in docker-compose.yaml in connector/external-import/alienvault and still doesn't work
Your configuration seems good. Normally, you will see data ingested in your platform. Could you please navigate to your OpenCTI and "Data --> Ingestion --> Connectors". Do you see the Alienvault connector ? Yes if, please click on it, Do you observe any works n progress ?
Yes, I can see the progress bar!
One thing I would like to ask is: I tried to search for this specific CVE-2024-37085 in the top search bar but Opencti couldn't find it. I changed the pulses to fetch data since 2022-05-01T00:00:00
but I wonder how to keep updated constantly with the latest reports out there. Do you have any tips? How to optimize the search in OpenCTI for a specific CVE, threat groups, etc?
Description
ERROR:
{"timestamp": "2024-07-23T09:41:53.447083Z", "level": "ERROR", "name": "api", "message": "HTTPConnectionPool(host='opencti', port=8080): Max retries exceeded with url: /graphql (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x7fb7125615e0>: Failed to establish a new connection: [Errno 111] Connection refused'))", "exc_info": "Traceback (most recent call last):\n File \"/usr/local/lib/python3.9/dist-packages/urllib3/connection.py\", line 196, in _new_conn\n sock = connection.create_connection(\n File \"/usr/local/lib/python3.9/dist-packages/urllib3/util/connection.py\", line 85, in create_connection\n raise err\n File \"/usr/local/lib/python3.9/dist-packages/urllib3/util/connection.py\", line 73, in create_connection\n sock.connect(sa)\nConnectionRefusedError: [Errno 111] Connection refused\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/usr/local/lib/python3.9/dist-packages/urllib3/connectionpool.py\", line 789, in urlopen\n response = self._make_request(\n File \"/usr/local/lib/python3.9/dist-packages/urllib3/connectionpool.py\", line 495, in _make_request\n conn.request(\n File \"/usr/local/lib/python3.9/dist-packages/urllib3/connection.py\", line 398, in request\n self.endheaders()\n File \"/usr/lib/python3.9/http/client.py\", line 1252, in endheaders\n self._send_output(message_body, encode_chunked=encode_chunked)\n File \"/usr/lib/python3.9/http/client.py\", line 1012, in _send_output\n self.send(msg)\n File \"/usr/lib/python3.9/http/client.py\", line 952, in send\n self.connect()\n File \"/usr/local/lib/python3.9/dist-packages/urllib3/connection.py\", line 236, in connect\n self.sock = self._new_conn()\n File \"/usr/local/lib/python3.9/dist-packages/urllib3/connection.py\", line 211, in _new_conn\n raise NewConnectionError(\nurllib3.exceptions.NewConnectionError: <urllib3.connection.HTTPConnection object at 0x7fb7125615e0>: Failed to establish a new connection: [Errno 111] Connection refused\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/usr/local/lib/python3.9/dist-packages/requests/adapters.py\", line 667, in send\n resp = conn.urlopen(\n File \"/usr/local/lib/python3.9/dist-packages/urllib3/connectionpool.py\", line 843, in urlopen\n retries = retries.increment(\n File \"/usr/local/lib/python3.9/dist-packages/urllib3/util/retry.py\", line 519, in increment\n raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type]\nurllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='opencti', port=8080): Max retries exceeded with url: /graphql (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x7fb7125615e0>: Failed to establish a new connection: [Errno 111] Connection refused'))\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n File \"/usr/local/lib/python3.9/dist-packages/pycti/api/opencti_api_client.py\", line 403, in health_check\n test = self.query(\n File \"/usr/local/lib/python3.9/dist-packages/pycti/api/opencti_api_client.py\", line 336, in query\n r = self.session.post(\n File \"/usr/local/lib/python3.9/dist-packages/requests/sessions.py\", line 637, in post\n return self.request(\"POST\", url, data=data, json=json, **kwargs)\n File \"/usr/local/lib/python3.9/dist-packages/requests/sessions.py\", line 589, in request\n resp = self.send(prep, **send_kwargs)\n File \"/usr/local/lib/python3.9/dist-packages/requests/sessions.py\", line 703, in send\n r = adapter.send(request, **kwargs)\n File \"/usr/local/lib/python3.9/dist-packages/requests/adapters.py\", line 700, in send\n raise ConnectionError(e, request=request)\nrequests.exceptions.ConnectionError: HTTPConnectionPool(host='opencti', port=8080): Max retries exceeded with url: /graphql (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x7fb7125615e0>: Failed to establish a new connection: [Errno 111] Connection refused'))"} OpenCTI API is not reachable. Waiting for OpenCTI API to start or check your configuration...
Docker compose file:
Just like it is described on the installation guide and the .env vars are set properly. Both AlienVault and AbuseIPDB gives the same error, I am out of options of what to do
Environment
Reproducible Steps
Steps to create the smallest reproducible scenario:
Expected Output
Actual Output
Additional information
Screenshots (optional)