Closed karl-otto closed 3 years ago
yes me too
Thank you for your report. I am not able to reproduce your findings. I created a test script based on your excerpt above:
async function main() {
bucketName = getUniqueName();
key = 'foo';
body = 'bar';
console.log('createBucket');
createBucketParams = {Bucket: bucketName, CreateBucketConfiguration: {LocationConstraint: ''}};
response = await s3Client.createBucket(createBucketParams).promise();
try {
console.log('upload');
params = {Bucket: bucketName, Key: key, Body: body};
options = {partSize: 10 * 2**20, queueSize: 10};
response = await s3Client.upload(params, options).promise();
response = await s3Client.deleteObject({Bucket: bucketName, Key: key}).promise();
} finally {
response = await s3Client.deleteBucket({Bucket: bucketName}).promise();
}
console.log('Complete!');
}
The output was this:
createBucket
upload
Complete!
I also replaced body
with var readStream = fs.createReadStream(filename);
and a valid filename. That produced the same output.
Could you provide more details about your environment? Also, if you have a completely standalone script that produces the failure, that would be helpful.
Okay… I use the docker image node:latest
node@node:/usr/src/app$ node --version
v15.5.1
Install IBM-COS-SDK v1.0.9
node@node:/usr/src/app$ npm install ibm-cos-sdk@1.9.0
up to date, audited 60 packages in 917ms
3 packages are looking for funding
run `npm fund` for details
found 0 vulnerabilities
Run testcase
node@node:/usr/src/app$ export COS_KEY='<myKey>'
node@node:/usr/src/app$ export COS_CRN='<myCRN>'
node@node:/usr/src/app$ export COS_BUCKET='upload'
node@node:/usr/src/app$ npm list ibm-cos-sdk
app@1.0.0 /usr/src/app
`-- ibm-cos-sdk@1.9.0
node@node:/usr/src/app$ node issue1.mjs
Try…
Upload
Complete!
Update IBM-COS-SDK to version v1.10.0
node@node:/usr/src/app$ npm update ibm-cos-sdk
removed 1 package, changed 1 package, and audited 59 packages in 4s
3 packages are looking for funding
run `npm fund` for details
found 0 vulnerabilities
Run testcase
node@node:/usr/src/app$ npm list ibm-cos-sdk
app@1.0.0 /usr/src/app
`-- ibm-cos-sdk@1.10.0
node@node:/usr/src/app$ node issue1.mjs
Try…
Upload
TypeError [ERR_INVALID_ARG_TYPE]: The "key" argument must be of type string or an instance of ArrayBuffer, Buffer, TypedArray, DataView, KeyObject, or CryptoKey. Received undefined
at new NodeError (node:internal/errors:278:15)
at prepareSecretKey (node:internal/crypto/keys:384:11)
at new Hmac (node:internal/crypto/hash:132:9)
at Object.createHmac (node:crypto:155:10)
at Object.hmac (/usr/src/app/node_modules/ibm-cos-sdk/lib/util.js:423:30)
at Object.getSigningKey (/usr/src/app/node_modules/ibm-cos-sdk/lib/signers/v4_credentials.js:62:8)
at V4.signature (/usr/src/app/node_modules/ibm-cos-sdk/lib/signers/v4.js:98:36)
at V4.authorization (/usr/src/app/node_modules/ibm-cos-sdk/lib/signers/v4.js:93:36)
at V4.addAuthorization (/usr/src/app/node_modules/ibm-cos-sdk/lib/signers/v4.js:35:12)
at /usr/src/app/node_modules/ibm-cos-sdk/lib/event_listeners.js:234:18 {
code: 'ERR_INVALID_ARG_TYPE',
retryDelay: 22.65398033442425
}
Complete!
import ibm from 'ibm-cos-sdk'
async function main() {
const bucketName = process.env.COS_BUCKET
const svcInst = process.env.COS_CRN
const apiId = process.env.COS_KEY
const k = 'dir/file.text'
const cosBucket = new ibm.S3({
endpoint: 's3.eu-de.cloud-object-storage.appdomain.cloud',
apiKeyId: apiId,
ibmAuthEndpoint: 'https://iam.cloud.ibm.com/identity/token',
serviceInstanceId: svcInst,
computeChecksums: true
})
console.log('Try…')
try {
console.log('Upload')
const params = { Bucket: bucketName, Key: k, Body: 'bar' }
const options = { partSize: 10 * 2**20, queueSize: 10 }
const response = await cosBucket.upload(params, options).promise()
} catch (error) {
console.error(error)
}
console.log('Complete!')
}
main()
@huineng Are you using IAM tokens? A different user shared an example that showed the stack trace going through a V4 signature path, but the user was providing an IAM API key. When I added signatureVersion: 'iam',
to the client creation parameter list, the example was successful. Could you check your client declaration code, and if this parameter is not there, add it and report back?
I am able to reproduce the failure message using the test case above. Adding signatureVersion: 'iam'
to the ibm.S3
call resolved the issue for me. Could you verify that works for you?
i'm using
apiKeyId endpoint serviceInstanceId (resource_instance_id)
i can try that in the morning
Adding the given option (signatureVersion: 'iam'
) solves the issue for me.
Thanks!
ok, indeed solved, can i get an explanation ? Maybe some documentation needs to be updated
I'll talk with the documentation team about an update. The short answer is the parameter you set specifically tells the client to look for IAM credentials. The credential handling code had been missing upstream updates going back almost four years. This release fixes a few compatibility bugs going back that far, and this appears to be a side effect of that. The error message is unfortunately confusing: key
is different from Key
but it's hard to notice given the context.
Ok, one more thing, it was only happening for uploads, when doing queries or display the attachments, that error did not happen
@IBMeric This is a breaking change which has been delivered in a minor version change.
It doesn't seem like the change was intentional given that the README still says If the Service Credential contain HMAC keys the client will use those and authenticate using a signature, otherwise the client will use the provided API key to authenticate using bearer tokens.
That statement is still true for most operations but it fails for upload
.
There needs to be a change here which either fixes the upload
code so that it doesn't require signatureVersion
to be set or you need to update your README and docs to say that signatureVersion
is required when using IAM tokens.
Note that in my case I run into the following error but setting signatureVersion: "iam"
also remedies that error.
@jonesphi You're right that this is an unintentional regression. I just looked into this further and am going to re-open this ticket. We will fix this in an upcoming release, hopefully in the next week.
@karl-otto Could you verify that 1.10.2 does not require you to set signatureVersion: "iam"
as in your original test case? If the issue is resolved, please close this ticket.
For @huineng's curiosity, upstream consolidated several signature version checks to all use the same source instead of using values in multiple data structures. The signature version was correctly reported when the signature checker had an actual request passed to it, but if it did not (common now because of the upstream change), it defaulted to a hardcoded value rather than doing detection. The fix does a check on the credentials available in lib/services/s3.js
's configuration. The workaround given above works because the user-provided value in the client overrides any detection.
Closing as resolved based on outside feedback.
This is still an issue for me when using 1.10.2
. If I do not set signatureVersion: 'iam'
it still fails with the same error. The config items I am setting
const cosConfig = {
endpoint: config.cos.DEFAULT_ENDPOINT,
apiKeyId: config.cos.CREDENTIALS_APIKEY,
serviceInstanceId: config.cos.CREDENTIALS_RESOURCE_INSTANCE_ID,
signatureVersion: 'iam', // per github.com/IBM/ibm-cos-sdk-js/issues/83
};
not sure on this but looking at the code if we don't add signatureVersion (regionDefinedVersion) then the default seems to be v4 and not iam
if (isPresigned !== true) {
defaultApiVersion = 'v4';
// If a request was provided, check if that uses IAM.
// Otherwise, check this instance for IAM.
if (request && request.service.config.credentials.tokenManager) {
defaultApiVersion = 'iam';
} else if (this.config.credentials.tokenManager) {
defaultApiVersion = 'iam';
}
} else if (regionDefinedVersion) {
defaultApiVersion = regionDefinedVersion;
}
@jhaaken Please open a new issue with a sample of the failing code.
node version
> node -v
v14.15.4
simple script
const IBMCOS = require('ibm-cos-sdk');
const cosConfig = {
endpoint: 's3.us-east.cloud-object-storage.appdomain.cloud',
apiKeyId: '<api-key>',
serviceInstanceId: '<service-instance-id>',
// signatureVersion: 'iam' // ==> add this in to make it work per github.com/IBM/ibm-cos-sdk-js/issues/83
};
const bucket = '
try {
console.log(file key ${key}
);
const cos = new IBMCOS.S3(cosConfig);
const buffer = Buffer.from('hello world');
const payload = {
Bucket: bucket,
Key: key,
Body: buffer,
Metadata: {
fileExt: .txt
,
},
};
cos.upload(payload, function(err, data) { if (err) { console.log('error', err); } else { console.log('success', data); } });
} catch (error) { console.log(error); }
- output if not providing a `signatureVersion` of IAM
``` bsh
> node ./temp/cos-defect.js
file key some-test/1234/hello-world.txt
error TypeError [ERR_INVALID_ARG_TYPE]: The "key" argument must be of type string or an instance of Buffer, TypedArray, DataView, or KeyObject. Received undefined
at prepareSecretKey (internal/crypto/keys.js:322:11)
at new Hmac (internal/crypto/hash.js:111:9)
at Object.createHmac (crypto.js:147:10)
at Object.hmac (/Users/jhaaken/github-ibm/certauto/ms-data-api/node_modules/ibm-cos-sdk/lib/util.js:427:30)
at Object.getSigningKey (/Users/jhaaken/github-ibm/certauto/ms-data-api/node_modules/ibm-cos-sdk/lib/signers/v4_credentials.js:62:8)
at V4.signature (/Users/jhaaken/github-ibm/certauto/ms-data-api/node_modules/ibm-cos-sdk/lib/signers/v4.js:98:36)
at V4.authorization (/Users/jhaaken/github-ibm/certauto/ms-data-api/node_modules/ibm-cos-sdk/lib/signers/v4.js:93:36)
at V4.addAuthorization (/Users/jhaaken/github-ibm/certauto/ms-data-api/node_modules/ibm-cos-sdk/lib/signers/v4.js:35:12)
at /Users/jhaaken/github-ibm/certauto/ms-data-api/node_modules/ibm-cos-sdk/lib/event_listeners.js:234:18
at finish (/Users/jhaaken/github-ibm/certauto/ms-data-api/node_modules/ibm-cos-sdk/lib/config.js:331:7) {
code: 'ERR_INVALID_ARG_TYPE',
retryDelay: 55.73262088037572
}
The new release (1.10.0) shows following error with our code:
This error is not shown with version 1.9.0.
Code excerpt related to above log: