matanolabs / matano

Open source security data lake for threat hunting, detection & response, and cybersecurity analytics at petabyte scale on AWS
https://matano.dev
Apache License 2.0
1.46k stars 100 forks source link

bug: An empty detections directory causes init to fail with stack trace #46

Closed timoguin closed 1 year ago

timoguin commented 1 year ago

What?

If you have an empty detections/ directory, matano init will fail with a stack trace when trying to bundle the (non-existent) detections with maturin. Deleting the empty directory allows the command to succeed.

Full error output with stack trace ``` › Bundling asset DPMainStack/Detections/Function/Code/Stage... › › /snapshot/matano/infra/node_modules/aws-cdk-lib/core/lib/asset-staging.js:2 › `),localBundling=options.local?.tryBundle(bundleDir,options),!localBundling){let user;if(options.user)user=options.user;else{const › userInfo=os.userInfo();user=userInfo.uid!==-1?`${userInfo.uid}:${userInfo.gid}`:"1000:1000"}options.image.run({command:options.command,user,volumes,environment:options.environment,entrypoint:options.entrypoint,workingDirectory:optio › ns.workingDirectory??AssetStaging.BUNDLING_INPUT_DIR,securityOpt:options.securityOpt??"",volumesFrom:options.volumesFrom})}}catch(err){const bundleErrorDir=bundleDir+"-error";throw › fs.existsSync(bundleErrorDir)&&fs.removeSync(bundleErrorDir),fs.renameSync(bundleDir,bundleErrorDir),new Error(`Failed to bundle asset ${this.node.path}, bundle output is located at ${bundleErrorDir}: › ${err}`)}if(fs_1.FileSystem.isEmpty(bundleDir)){const outputDir=localBundling?bundleDir:AssetStaging.BUNDLING_OUTPUT_DIR;throw new Error(`Bundling did not produce any output. Check that content is written to › ${outputDir}.`)}}calculateHash(hashType,bundling,outputDir){if(hashType==assets_1.AssetHashType.CUSTOM||hashType==assets_1.AssetHashType.SOURCE&&bundling){const hash=crypto.createHash("sha256");return › hash.update(this.customSourceFingerprint??fs_1.FileSystem.fingerprint(this.sourcePath,this.fingerprintOptions)),bundling&&hash.update(JSON.stringify(bundling)),hash.digest("hex")}switch(hashType){case › assets_1.AssetHashType.SOURCE:return fs_1.FileSystem.fingerprint(this.sourcePath,this.fingerprintOptions);case assets_1.AssetHashType.BUNDLE:case assets_1.AssetHashType.OUTPUT:if(!outputDir)throw new Error(`Cannot use › \`${hashType}\` hash type when \`bundling\` is not specified.`);return fs_1.FileSystem.fingerprint(outputDir,this.fingerprintOptions);default:throw new Error("Unknown asset hash type.")}}}exports.AssetStaging=AssetStaging,_a=JSII_RT › TI_SYMBOL_1,AssetStaging[_a]={fqn:"aws-cdk-lib.AssetStaging",version:"2.56.0"},AssetStaging.BUNDLING_INPUT_DIR="/asset-input",AssetStaging.BUNDLING_OUTPUT_DIR="/asset-output",AssetStaging.assetCache=new cache_1.Cache;function › renderAssetFilename(assetHash,extension=""){return`asset.${assetHash}${extension}`}function determineHashType(assetHashType,customSourceFingerprint){const › hashType=customSourceFingerprint?assetHashType??assets_1.AssetHashType.CUSTOM:assetHashType??assets_1.AssetHashType.SOURCE;if(customSourceFingerprint&&hashType!==assets_1.AssetHashType.CUSTOM)throw new Error(`Cannot specify › \`${assetHashType}\` for \`assetHashType\` when \`assetHash\` is specified. Use \`CUSTOM\` or leave \`undefined\`.`);if(hashType===assets_1.AssetHashType.CUSTOM&&!customSourceFingerprint)throw new Error("`assetHash` must be › specified when `assetHashType` is set to `AssetHashType.CUSTOM`.");return hashType}function calculateCacheKey(props){return crypto.createHash("sha256").update(JSON.stringify(sortObject(props))).digest("hex")}function › sortObject(object){if(typeof object!="object"||object instanceof Array)return object;const ret={};for(const key of Object.keys(object).sort())ret[key]=sortObject(object[key]);return ret}function › singleArchiveFile(directory){if(!fs.existsSync(directory))throw new Error(`Directory ${directory} does not exist.`);if(!fs.statSync(directory).isDirectory())throw new Error(`${directory} is not a directory.`);const › content=fs.readdirSync(directory);if(content.length===1){const file=path.join(directory,content[0]),extension=getExtension(content[0]).toLowerCase();if(fs.statSync(file).isFile()&&ARCHIVE_EXTENSIONS.includes(extension))return › file}}function determineBundledAsset(bundleDir,outputType){const › archiveFile=singleArchiveFile(bundleDir);switch(outputType===bundling_1.BundlingOutput.AUTO_DISCOVER&&(outputType=archiveFile?bundling_1.BundlingOutput.ARCHIVED:bundling_1.BundlingOutput.NOT_ARCHIVED),outputType){case › bundling_1.BundlingOutput.NOT_ARCHIVED:return{path:bundleDir,packaging:assets_1.FileAssetPackaging.ZIP_DIRECTORY};case bundling_1.BundlingOutput.ARCHIVED:if(!archiveFile)throw new Error("Bundling output directory is expected to › include only a single archive file when `output` is set to `ARCHIVED`");return{path:archiveFile,packaging:assets_1.FileAssetPackaging.FILE,extension:getExtension(archiveFile)}}}function getExtension(source){for(const ext of › ARCHIVE_EXTENSIONS)if(source.toLowerCase().endsWith(ext))return ext;return path.extname(source)} › › › › ^ › Error: Bundling did not produce any output. Check that content is written to /var/folders/sq/yzwl6n255zz2s32y1pwymf400000gq/T/matanocdkoutd2YWDY/asset.874a339ec05d104f21b143404e74723ce7d1e51ad089d6a0183a839a53c207ac. › at AssetStaging.bundle (/snapshot/matano/infra/node_modules/aws-cdk-lib/core/lib/asset-staging.js:2:873) › at AssetStaging.stageByBundling (/snapshot/matano/infra/node_modules/aws-cdk-lib/core/lib/asset-staging.js:1:4506) › at stageThisAsset (/snapshot/matano/infra/node_modules/aws-cdk-lib/core/lib/asset-staging.js:1:1867) › at Cache.obtain (/snapshot/matano/infra/node_modules/aws-cdk-lib/core/lib/private/cache.js:1:242) › at new AssetStaging (/snapshot/matano/infra/node_modules/aws-cdk-lib/core/lib/asset-staging.js:1:2262) › at new Asset (/snapshot/matano/infra/node_modules/aws-cdk-lib/aws-s3-assets/lib/asset.js:1:736) › at AssetCode.bind (/snapshot/matano/infra/node_modules/aws-cdk-lib/aws-lambda/lib/code.js:1:4628) › at new Function (/snapshot/matano/infra/node_modules/aws-cdk-lib/aws-lambda/lib/function.js:1:2803) › at new MatanoDetections (/snapshot/matano/infra/lib/detections.ts:48:30) › at new DPMainStack (/snapshot/matano/infra/src/DPMainStack.ts:76:24) › [07:47:24] Reading cached notices from /Users/REDACTED/.cdk/cache/notices.json › [07:47:24] Failed to get tree.json file: Error: /var/folders/sq/yzwl6n255zz2s32y1pwymf400000gq/T/matanocdkoutd2YWDY/tree.json: ENOENT: no such file or directory, open › '/var/folders/sq/yzwl6n255zz2s32y1pwymf400000gq/T/matanocdkoutd2YWDY/tree.json'. Proceeding with empty tree. › › Subprocess exited with error 1 › [07:47:24] Error: Subprocess exited with error 1 › at ChildProcess. (/snapshot/node_modules/aws-cdk/lib/api/cxapp/exec.ts:153:23) › at ChildProcess.emit (events.js:400:28) › at ChildProcess.emit (domain.js:475:12) › at Process.ChildProcess._handle.onexit (internal/child_process.js:282:12) › Created temporary directory for configuration files: /var/folders/sq/yzwl6n255zz2s32y1pwymf400000gq/T/mtnconfignpydzp/config ```