[ML] ESS Kibana API integration tests fail with `Error: self-signed certificate in certificate chain` when calling `getService('ml').importTrainedModel()` with `ssl:true` configured #193477
Kibana version:main/8.16 3bea483f34e03ea1bbeb836350c650fd06673f10
Elasticsearch version: Same, run via Kibana API Integration Server
Summary
In ESS Kibana API integration tests, when calling getService('ml').importTrainedModel() with ssl:true configured, the test fails with the following Error: self-signed certificate in certificate chain:
└-> "before all" hook: beforeTestSuite.trigger in "@ess Basic Security AI Assistant Knowledge Base Entries"
└-> "before all" hook in "@ess Basic Security AI Assistant Knowledge Base Entries"
│ debg Waiting up to 60000ms for Stats index to exist...
│ debg Creating trained model with id "pt_tiny_elser"
│ debg > Trained model created
│ debg Creating vocabulary for trained model "pt_tiny_elser"
└- ✖ fail: GenAI - Knowledge Base Entries APIs @ess Basic Security AI Assistant Knowledge Base Entries "before all" hook in "@ess Basic Security AI Assistant Knowledge Base Entries"
│ Error: self-signed certificate in certificate chain
│ at TLSSocket.onConnectSecure (node:_tls_wrap:1674:34)
│ at TLSSocket.emit (node:events:519:28)
│ at TLSSocket._finishInit (node:_tls_wrap:1085:8)
│ at TLSWrap.ssl.onhandshakedone (node:_tls_wrap:871:12)
From the │ debg > Trained model created log we can see the createTrainedModel call to /internal/ml/trained_models/${modelId} was successful:
Create an API Integration test that calls getService('ml').importTrainedModel(). This is the configuration I used, which was within the security_solution_api_integration directory, extending from the config.base.trial config:
Disabling SSL via ssl:false in the configuration prevents this issue from occurring, so since the other parts of my tests do not require SSL I've disabled it in my suite for the time being.
Build Details
Kibana version:
main
/8.16
3bea483f34e03ea1bbeb836350c650fd06673f10 Elasticsearch version: Same, run via Kibana API Integration ServerSummary
In ESS Kibana API integration tests, when calling
getService('ml').importTrainedModel()
withssl:true
configured, the test fails with the followingError: self-signed certificate in certificate chain
:From the
│ debg > Trained model created
log we can see thecreateTrainedModel
call to/internal/ml/trained_models/${modelId}
was successful:https://github.com/elastic/kibana/blob/f18224c6869ae52228da3764ca9a427106b872fb/x-pack/test/functional/services/ml/api.ts#L1381-L1391
However the subsequent
createTrainedModelVocabularyES
call to/_ml/trained_models/${modelId}/vocabulary
, surfaces the error:https://github.com/elastic/kibana/blob/f18224c6869ae52228da3764ca9a427106b872fb/x-pack/test/functional/services/ml/api.ts#L1404-L1412
Steps to reproduce
Create an API Integration test that calls
getService('ml').importTrainedModel()
. This is the configuration I used, which was within thesecurity_solution_api_integration
directory, extending from theconfig.base.trial
config:ess.config.ts
``` ts import { FtrConfigProviderContext } from '@kbn/test'; export default async function ({ readConfigFile }: FtrConfigProviderContext) { const functionalConfig = await readConfigFile( require.resolve('../../../../../../config/ess/config.base.trial') ); return { ...functionalConfig.getAll(), kbnTestServer: { ...functionalConfig.get('kbnTestServer'), serverArgs: [ ...functionalConfig.get('kbnTestServer.serverArgs'), `--xpack.securitySolution.enableExperimental=${JSON.stringify([ 'assistantKnowledgeBaseByDefault', ])}`, ], }, testFiles: [require.resolve('..')], junit: { reportName: 'GenAI - Knowledge Base Entries Tests - ESS Env - Trial License', }, }; } ```
ess.config.json
``` json { "testFiles": [ "/Users/garrettspong/dev/kibana-main/x-pack/test/security_solution_api_integration/test_suites/genai/knowledge_base/entries/trial_license_complete_tier/index.ts" ], "servers": { "kibana": { "protocol": "http", "hostname": "localhost", "port": 5620, "auth": "elastic:changeme", "username": "elastic", "password": "changeme" }, "elasticsearch": { "protocol": "https", "hostname": "localhost", "port": 9220, "auth": "system_indices_superuser:changeme", "username": "system_indices_superuser", "password": "changeme" }, "fleetserver": { "protocol": "http", "hostname": "localhost" } }, "services": {}, "junit": { "reportName": "GenAI - Knowledge Base Entries Tests - ESS Env - Trial License" }, "esTestCluster": { "license": "trial", "from": "snapshot", "serverArgs": [ "xpack.license.self_generated.type=trial" ], "ssl": true }, "kbnTestServer": { "buildArgs": [], "sourceArgs": [ "--no-base-path", "--env.name=development" ], "serverArgs": [ "--server.port=5620", "--status.allowAnonymous=true", "--elasticsearch.hosts=http://localhost:9220", "--elasticsearch.username=kibana_system", "--elasticsearch.password=changeme", "--data.search.aggs.shardDelay.enabled=true", "--data.query.timefilter.minRefreshInterval=1000", "--security.showInsecureClusterWarning=false", "--telemetry.banner=false", "--telemetry.optIn=false", "--telemetry.sendUsageTo=staging", "--server.maxPayload=1679958", "--plugin-path=/Users/garrettspong/dev/kibana-main/test/common/plugins/newsfeed", "--plugin-path=/Users/garrettspong/dev/kibana-main/test/common/plugins/otel_metrics", "--newsfeed.service.urlRoot=http://localhost:5620", "--newsfeed.service.pathTemplate=/api/_newsfeed-FTS-external-service-simulators/kibana/v{VERSION}.json", "--logging.appenders.deprecation={\"type\":\"console\",\"layout\":{\"type\":\"json\"}}", "--logging.loggers=[{\"name\":\"elasticsearch.deprecation\",\"level\":\"all\",\"appenders\":[\"deprecation\"]}]", "--logging.appenders.default={\"type\":\"console\",\"layout\":{\"type\":\"pattern\",\"pattern\":\"[%date][%level][%logger] %message %meta\"}}", "--logging.appenders.console={\"type\":\"console\",\"layout\":{\"type\":\"pattern\",\"pattern\":\"[%date][%level][%logger] %message %meta\"}}", "--status.allowAnonymous=true", "--server.uuid=5b2de169-2785-441b-ae8c-186a1936b17d", "--xpack.maps.showMapsInspectorAdapter=true", "--xpack.maps.preserveDrawingBuffer=true", "--xpack.security.encryptionKey=\"wuGNaIhoMpk5sO4UBxgr3NyW1sFcLgIf\"", "--xpack.encryptedSavedObjects.encryptionKey=\"DkdXazszSCYexXqz4YktBGHCRkV6hyNK\"", "--xpack.discoverEnhanced.actions.exploreDataInContextMenu.enabled=true", "--savedObjects.maxImportPayloadBytes=10485760", "--savedObjects.allowHttpApiAccess=false", "--server.restrictInternalApis=false", "--xpack.task_manager.unsafe.exclude_task_types=[\"Fleet-Metrics-Task\"]", "--xpack.security.session.idleTimeout=3600000", "--telemetry.optIn=true", "--xpack.fleet.agents.pollingRequestTimeout=5000", "--xpack.ruleRegistry.write.enabled=true", "--xpack.ruleRegistry.write.enabled=true", "--xpack.ruleRegistry.write.cache.enabled=false", "--monitoring_collection.opentelemetry.metrics.prometheus.enabled=true", "--xpack.actions.allowedHosts=[\"localhost\",\"some.non.existent.com\"]", "--xpack.actions.enabledActionTypes=[\".cases\",\".email\",\".index\",\".pagerduty\",\".swimlane\",\".server-log\",\".servicenow\",\".slack\",\".webhook\",\"test.authorization\",\"test.failing\",\"test.index-record\",\"test.noop\",\"test.rate-limit\"]", "--xpack.eventLog.logEntries=true", "--xpack.securitySolution.alertIgnoreFields=[\"testing_ignored.constant\",\"/testing_regex*/\"]", "--xpack.ruleRegistry.write.enabled=true", "--xpack.ruleRegistry.write.cache.enabled=false", "--xpack.ruleRegistry.unsafe.indexUpgrade.enabled=true", "--xpack.ruleRegistry.unsafe.legacyMultiTenancy.enabled=true", "--xpack.securitySolution.enableExperimental=[\"previewTelemetryUrlEnabled\",\"riskScoringPersistence\",\"riskScoringRoutesEnabled\",\"manualRuleRunEnabled\"]", "--xpack.task_manager.poll_interval=1000", "--xpack.actions.preconfigured={\"my-test-email\":{\"actionTypeId\":\".email\",\"name\":\"TestEmail#xyz\",\"config\":{\"from\":\"me@test.com\",\"service\":\"__json\"},\"secrets\":{\"user\":\"user\",\"password\":\"password\"}}}", "--elasticsearch.hosts=https://localhost:9220", "--elasticsearch.ssl.certificateAuthorities=/Users/garrettspong/dev/kibana-main/packages/kbn-dev-utils/certs/ca.crt", "--xpack.securitySolution.enableExperimental=[\"assistantKnowledgeBaseByDefault\"]" ], "useDedicatedTaskRunner": false, "runOptions": { "wait": {}, "alwaysUseSource": false }, "env": { "ELASTICSEARCH_USERNAME": "elastic" } }, "mochaOpts": { "grep": "/^(?!.*@skipInEss).*@ess.*/", "bail": false, "dryRun": false, "invert": false, "slow": 30000, "timeout": 360000, "ui": "bdd" }, "serverless": false, "suiteFiles": { "include": [], "exclude": [] }, "suiteTags": { "include": [], "exclude": [] }, "servicesRequiredForTestAnalysis": [], "pageObjects": {}, "timeouts": { "find": 10000, "try": 120000, "waitFor": 20000, "esRequestTimeout": 30000, "kibanaReportCompletion": 60000, "kibanaStabilize": 15000, "navigateStatusPageCheck": 250, "waitForExists": 2500 }, "updateBaselines": false, "updateSnapshots": false, "browser": { "type": "chrome", "logPollingMs": 100, "acceptInsecureCerts": false }, "mochaReporter": { "captureLogOutput": false, "sendToCiStats": false }, "esServerlessOptions": { "resources": [] }, "chromedriver": { "url": "http://localhost:9515" }, "firefoxdriver": { "url": "http://localhost:2828" }, "apps": {}, "esArchiver": {}, "kbnArchiver": { "directory": "/Users/garrettspong/dev/kibana-main/x-pack/test/security_solution_api_integration/config/ess/fixtures/kbn_archiver" }, "uiSettings": {}, "screenshots": { "directory": "/Users/garrettspong/dev/kibana-main/x-pack/test/security_solution_api_integration/config/ess/screenshots" }, "snapshots": { "directory": "/Users/garrettspong/dev/kibana-main/x-pack/test/security_solution_api_integration/config/ess/snapshots" }, "failureDebugging": { "htmlDirectory": "/Users/garrettspong/dev/kibana-main/x-pack/test/security_solution_api_integration/config/ess/failure_debug/html" }, "layout": { "fixedHeaderHeight": 100 }, "security": { "roles": {}, "defaultRoles": [ "superuser" ] }, "dockerServers": {} } ```
integration_test.ts
``` ts export default ({ getService }: FtrProviderContext) => { const supertest = getService('supertest'); const ml = getService('ml') as ReturnType;
describe('@ess Repro self-signed certificate error', () => {
before(async () => {
const config = {
...ml.api.getTrainedModelConfig(TINY_ELSER.name),
input: {
field_names: ['text_field'],
},
};
await ml.api.assureMlStatsIndexExists();
await ml.api.importTrainedModel(TINY_ELSER.name, TINY_ELSER.id, config);
});
after(async () => {
await ml.api.stopTrainedModelDeploymentES(TINY_ELSER.id, true);
await ml.api.deleteTrainedModelES(TINY_ELSER.id);
await ml.api.cleanMlIndices();
await ml.testResources.cleanMLSavedObjects();
});
describe('Create Entries', () => {
it('should say hi', async () => {
console.log('hi');
});
});
});
};
```
Workaround
Disabling SSL via
ssl:false
in the configuration prevents this issue from occurring, so since the other parts of my tests do not require SSL I've disabled it in my suite for the time being.