search

freesoftwarefactory / parse-multipart

A javascript/nodejs multipart/form-data parser which operates on raw data.
MIT License
48 stars 80 forks source link

Boundary not being returned #9

Open jbrosan opened 7 years ago

jbrosan commented 7 years ago

Hi,

I am currently experiencing an issue with uploading a file using multipart/form-data to AWS. I've setup the API Gateway as you have it in your video and added my lambda.

It seems that the getBoundary method is not returning a boundary and when I attempt to parse into parts that its just not working. The parts array is empty.

Any ideas or suggestions on what could be happening and how to resolve it would be most appreciated.

Thanks , John

LAMBDA


'use strict';

const AWS = require('aws-sdk');
const multipart = require('parse-multipart');

module.exports.fileUpload = (event, context, callback) => {

  AWS.config.update({
    accessKeyId: process.env.PUBLIC_AWS_KEY,
    secretAccessKey: process.env.SECRET_AWS_KEY
  });

  try {

    let bodyBuffer = new Buffer(event['body-json'].toString(), 'base64');
    console.log('BUFFER: ', bodyBuffer.toString());

    let boundary = multipart.getBoundary(event.params.header['content-type']);
    console.log('BOUNDARY: ', boundary);

    let parts = multipart.Parse(bodyBuffer, boundary);
    console.log('PARTS: ', parts);
    console.log('PARTS LEN: ', parts.length);

    const response = {
      statusCode: 200,
      headers: {
        "Access-Control-Allow-Origin" : "*", // Required for CORS support to work
        "Access-Control-Allow-Credentials" : false // Required for cookies, authorization headers with HTTPS
      },
      body: JSON.stringify({
        message: 'File upload com1plete'
      }),
    };
    callback(null, response);

  } catch (err) {
    console.log("ERROR: ", err);
    const response = {
      statusCode: 500,
      headers: {
        "Access-Control-Allow-Origin" : "*", // Required for CORS support to work
        "Access-Control-Allow-Credentials" : false // Required for cookies, authorization headers with HTTPS
      },
      body: JSON.stringify({
        message: 'File upload ERR: ' + err.toString()
      }),
    };
    callback(response, null);
  }

};

The event 

Event: { 'body-json': 'LS0tLS0tV2ViS2l0Rm9ybUJvdW5kYXJ5cWlxRUxCQ0lpUlJhU1hpTQ0KQ29udGVudC1EaXNwb3NpdGlvbjogZm9ybS1kYXRhOyBuYW1lPSJmaWxlIjsgZmlsZW5hbWU9ImpvaG5sZWRlczk4YnNhbXBsZS5jc3YiDQpDb250ZW50LVR5cGU6IHRleHQvY3N2DQoNCkxFREVTMTk5OEJbXQpJTlZPSUNFX0RBVEV8SU5WT0lDRV9OVU1CRVJ8Q0xJRU5UX0lEfExBV19GSVJNX01BVFRFUl9JRHxJTlZPSUNFX1RPVEFMfEJJTExJTkdfU1RBUlRfREFURXxCSUxMSU5HX0VORF9EQVRFfElOVk9JQ0VfREVTQ1JJUFRJT058TElORV9JVEVNX05VTUJFUnxFWFAvRkVFL0lOVl9BREpfVFlQRXxMSU5FX0lURU1fTlVNQkVSX09GX1VOSVRTfExJTkVfSVRFTV9BREpVU1RNRU5UX0FNT1VOVHxMSU5FX0lURU1fVE9UQUx8TElORV9JVEVNX0RBVEV8TElORV9JVEVNX1RBU0tfQ09ERXxMSU5FX0lURU1fRVhQRU5TRV9DT0RFfExJTkVfSVRFTV9BQ1RJVklUWV9DT0RFfFRJTUVLRUVQRVJfSUR8TElORV9JVEVNX0RFU0NSSVBUSU9OfExBV19GSVJNX0lEfExJTkVfSVRFTV9VTklUX0NPU1R8VElNRUtFRVBFUl9OQU1FfFRJTUVLRUVQRVJfQ0xBU1NJRklDQVRJT058Q0xJRU5UX01BVFRFUl9JRAoxOTk5MDIyNXw5NjU0MnwwMDcxMXwwNTI4fDE2ODQuNDV8MTk5OTAxMDF8MTk5OTAxMzF8Rm9yIHNlcnZpY2VzIHJlbmRlcmVkfDF8RnwyLjAwfC03MHw2MzB8MTk5OTAxMTV8TDUxMHx8QTEwMnwyMjU0N3xSZXNlYXJjaCBBdHRvcm5leeKAmXMgZmVlcywgU2V0IG9mZiBjbGFpbXwyNC02NDM3MzgxfDM1MHxBcm5zbGV5LCBSb2JlcnR8UEFSVE5SfDQyMy05ODdbXQoxOTk5MDIyNXw5NjU0MnwwMDcxMXwwNTI4fDE2ODQuNDV8MTk5OTAxMDF8MTk5OTAxMzF8Rm9yIHNlcnZpY2VzIHJlbmRlcmVkfDJ8RnwyLjAwfDB8NzAwfDE5OTkwMTE1fEw1MTB8fEExMDJ8MjI1NDd8UmVzZWFyY2ggYXR0b3JuZXkncyBmZWVzLCBUcmlhbCBwbGVhZGluZ3wyNC02NDM3MzgxfDM1MHxBcm5zbGV5LCBSb2JlcnR8UEFSVE5SfDQyMy05ODdbXQoxOTk5MDIyNXw5NjU0MnwwMDcxMXwwNTI4fDE2ODQuNDV8MTk5OTAxMDF8MTk5OTAxMzF8Rm9yIHNlcnZpY2VzIHJlbmRlcmVkfDN8RnwwLjIwMHwwfDQwfDE5OTkwMTE2fEw1MTB8fEExMDd8NDU4NzV8VGVsZXBob25lIGNvbmZlcmVuY2Ugd2l0aCBKb2huIERvZXwyNC02NDM3MzgxfDIwMHxCZWFzdGVyLCBKb2hufEFTU09DfDQyMy05ODdbXQoxOTk5MDIyNXw5NjU0MnwwMDcxMXwwNTI4fDE2ODQuNDV8MTk5OTAxMDF8MTk5OTAxMzF8Rm9yIHNlcnZpY2VzIHJlbmRlcmVkfDR8RXwxfDB8MjQuOTV8MTk5OTAxMTd8fEUxMTF8fHxNZWFsc3wyNC02NDM3MzgxfDI0Ljk1fHx8NDIzLTk4N1tdCjE5OTkwMjI1fDk2NTQyfDAwNzExfDA1Mjh8MTY4NC40NXwxOTk5MDEwMXwxOTk5MDEzMXxGb3Igc2VydmljZXMgcmVuZGVyZWR8NXxFfDF8MHwyODkuNXwxOTk5MDExN3x8RTExMHx8fE91dC1vZl90b3duIHRyYXZlbHwyNC02NDM3MzgxfDI4OS41fHx8NDIzLTk4N1tdCjE5OTkwMjI1fDk2NTQyfDAwNzExfDEzMjZ8MTI1MHwxOTk5MDEwMXwxOTk5MDEzMXxNb250aGx5IFJldGFpbmVyfDZ8SUZ8MXwxMjUwLnwxMjUwfDE5OTkwMTMxfHx8fHxNb250aGx5IFJldGFpbmVyIEZlZXwyNC02NDM3MzgxfHx8fDQyNS05MzZbXQoyMDE2MDkzMHw5NjU1NHwwMDgxNHwwNjI4fDY4NC41MHwyMDE2MDkwMXwyMDE2MDkzMHxGb3Igc2VydmljZXMgcmVuZGVyZWR8MXxGfDMuNTB8LTMwfDY1NnwyMDE2MDExNXxMNTEwfHxBMTAyfDIyNTQ3fFJlc2VhcmNoIEF0dG9ybmV54oCZcyBmZWVzLCBTZXQgb2ZmIGNsYWltfDI0LTY0MzczODF8MzUwfEFybnNsZXksIFJvYmVydHxQQVJUTlJ8NDIzLTk4N1tdCjIwMTYwOTMwfDk2NTU0fDAwODE0fDA2Mjh8Njg0LjUwfDIwMTYwOTAxfDIwMTYwOTMwfEZvciBzZXJ2aWNlcyByZW5kZXJlZHwyfEZ8NC4wMHwwfDcwMXwyMDE2MTExNXxMNTEwfHxBMTAyfDIyNTQ3fFJlc2VhcmNoIGF0dG9ybmV5J3MgZmVlcywgVHJpYWwgcGxlYWRpbmd8MjQtNjQzNzM4MXwzNTB8QXJuc2xleSwgUm9iZXJ0fFBBUlROUnw0MjMtOTg3W10KMjAxNjA5MzB8OTY1NTR8MDA4MTR8MDYyOHw2ODQuNTB8MjAxNjA5MDF8MjAxNjA5MzB8Rm9yIHNlcnZpY2VzIHJlbmRlcmVkfDN8RnwwLjMwMHwwfDQxfDIwMTYwMzE3fEw1MTB8fEExMDd8NDU4NzV8VGVsZXBob25lIGNvbmZlcmVuY2Ugd2l0aCBKb2huIERvZXwyNC02NDM3MzgxfDIwMHxCZWFzdGVyLCBKb2hufEFTU09DfDQyMy05ODdbXQoyMDE3MTEzMHw4NjU0MnwwMDMyMnwwNjI4fDY4NC41MHwyMDE3MTEwMXwyMDE3MTEzMHxGb3Igc2VydmljZXMgcmVuZGVyZWR8NHxFfDF8MHwzNC45NXwyMDE3MDExN3x8RTExMXx8fE1lYWxzfDI0LTY0MzczODF8MjQuOTV8fHw0MjMtOTg3W10KMjAxNzExMzB8ODY1NDJ8MDAzMjJ8MDYyOHw2ODQuNTB8MjAxNzExMDF8MjAxNzExMzB8Rm9yIHNlcnZpY2VzIHJlbmRlcmVkfDV8RXwxfDB8MzAxLjV8MjAxNzAzMjF8fEUxMTB8fHxPdXQtb2ZfdG93biB0cmF2ZWx8MjQtNjQzNzM4MXwyODkuNXx8fDQyMy05ODdbXQoyMDE3MTEzMHw4NjU0MnwwMDMyMnwxNDI2fDEyNTB8MjAxNzExMDF8MjAxNzExMzB8TW9udGhseSBSZXRhaW5lcnw2fElGfDF8MTI1MC58MTI1MHwyMDE3MTAzMXx8fHx8TW9udGhseSBSZXRhaW5lciBGZWV8MjQtNjQzNzM4MXx8fHw0MjUtOTM2W10KDQotLS0tLS1XZWJLaXRGb3JtQm91bmRhcnlxaXFFTEJDSWlSUmFTWGlNLS0NCg==',
params: 
{ path: {},
querystring: {},
header: 
{ Accept: '*/*',
'accept-encoding': 'gzip, deflate, br',
'Accept-Language': 'en-US,en;q=0.8',
'CloudFront-Forwarded-Proto': 'https',
'CloudFront-Is-Desktop-Viewer': 'true',
'CloudFront-Is-Mobile-Viewer': 'false',
'CloudFront-Is-SmartTV-Viewer': 'false',
'CloudFront-Is-Tablet-Viewer': 'false',
'CloudFront-Viewer-Country': 'US',
'content-type': 'multipart/form-data',
Host: 'row9htct2d.execute-api.us-east-1.amazonaws.com',
origin: 'http://localhost:8088',
Referer: 'http://localhost:8088/fileupload',
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36',
Via: '2.0 3ca41706981cad42d8ecaabd29f88efa.cloudfront.net (CloudFront)',
'X-Amz-Cf-Id': 'n9aypJ5g7m02dROkhxA57KOXOYQAA3sQm4VkjqmxA9XTEJvO42yj2g==',
'X-Amzn-Trace-Id': 'Root=1-59cc056f-3b0fff4e2a4dcafc6dd77544',
'X-Forwarded-For': '174.55.194.210, 205.251.250.63',
'X-Forwarded-Port': '443',
'X-Forwarded-Proto': 'https' } },
'stage-variables': {},
context: 
{ 'account-id': '',
'api-id': 'row9htct2d',
'api-key': '',
'authorizer-principal-id': '',
caller: '',
'cognito-authentication-provider': '',
'cognito-authentication-type': '',
'cognito-identity-id': '',
'cognito-identity-pool-id': '',
'http-method': 'POST',
stage: 'dev',
'source-ip': '174.55.194.210',
user: '',
'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36',
'user-arn': '',
'request-id': 'bed12d29-a3bf-11e7-bc09-dd4b4cd0dee2',
'resource-id': 'b70xn9',
'resource-path': '/api/fileupload' } }
christiansalazar commented 7 years ago

hi, sorry for my late response. I see no boundary in your payload.

take a deep look at your "body-json" string, it is a base64 encoded payload (your "post"), this is:

"LS0tLS0tV2V...WGlNLS0NCg=="

when u decode it, you can see:

------WebKitFormBoundaryqiqELBCIiRRaSXiM
Content-Disposition: form-data; name="file"; filename="johnledes98bsample.csv"
Content-Type: text/csv

LEDES1998B[]
INVOICE_DATE|INVOICE_NUMBER|CLIENT_ID|LAW_FIRM_MATTER_ID|INVOICE_TOTAL|BILLING_START_DATE|BILLING_END_DATE|INVOICE_DESCRIPTION|LINE_ITEM_NUMBER|EXP/FEE/INV_ADJ_TYPE|LINE_ITEM_NUMBER_OF_UNITS|LINE_ITEM_ADJUSTMENT_AMOUNT|LINE_ITEM_TOTAL|LINE_ITEM_DATE|LINE_ITEM_TASK_CODE|LINE_ITEM_EXPENSE_CODE|LINE_ITEM_ACTIVITY_CODE|TIMEKEEPER_ID|LINE_ITEM_DESCRIPTION|LAW_FIRM_ID|LINE_ITEM_UNIT_COST|TIMEKEEPER_NAME|TIMEKEEPER_CLASSIFICATION|CLIENT_MATTER_ID
19990225|96542|00711|0528|1684.45|19990101|19990131|For services rendered|1|F|2.00|-70|630|19990115|L510||A102|22547|Research Attorney’s fees, Set off claim|24-6437381|350|Arnsley, Robert|PARTNR|423-987[]
19990225|96542|00711|0528|1684.45|19990101|19990131|For services rendered|2|F|2.00|0|700|19990115|L510||A102|22547|Research attorney's fees, Trial pleading|24-6437381|350|Arnsley, Robert|PARTNR|423-987[]
19990225|96542|00711|0528|1684.45|19990101|19990131|For services rendered|3|F|0.200|0|40|19990116|L510||A107|45875|Telephone conference with John Doe|24-6437381|200|Beaster, John|ASSOC|423-987[]
19990225|96542|00711|0528|1684.45|19990101|19990131|For services rendered|4|E|1|0|24.95|19990117||E111|||Meals|24-6437381|24.95|||423-987[]
19990225|96542|00711|0528|1684.45|19990101|19990131|For services rendered|5|E|1|0|289.5|19990117||E110|||Out-of_town travel|24-6437381|289.5|||423-987[]
19990225|96542|00711|1326|1250|19990101|19990131|Monthly Retainer|6|IF|1|1250.|1250|19990131|||||Monthly Retainer Fee|24-6437381||||425-936[]
20160930|96554|00814|0628|684.50|20160901|20160930|For services rendered|1|F|3.50|-30|656|20160115|L510||A102|22547|Research Attorney’s fees, Set off claim|24-6437381|350|Arnsley, Robert|PARTNR|423-987[]
20160930|96554|00814|0628|684.50|20160901|20160930|For services rendered|2|F|4.00|0|701|20161115|L510||A102|22547|Research attorney's fees, Trial pleading|24-6437381|350|Arnsley, Robert|PARTNR|423-987[]
20160930|96554|00814|0628|684.50|20160901|20160930|For services rendered|3|F|0.300|0|41|20160317|L510||A107|45875|Telephone conference with John Doe|24-6437381|200|Beaster, John|ASSOC|423-987[]
20171130|86542|00322|0628|684.50|20171101|20171130|For services rendered|4|E|1|0|34.95|20170117||E111|||Meals|24-6437381|24.95|||423-987[]
20171130|86542|00322|0628|684.50|20171101|20171130|For services rendered|5|E|1|0|301.5|20170321||E110|||Out-of_town travel|24-6437381|289.5|||423-987[]
20171130|86542|00322|1426|1250|20171101|20171130|Monthly Retainer|6|IF|1|1250.|1250|20171031|||||Monthly Retainer Fee|24-6437381||||425-936[]

------WebKitFormBoundaryqiqELBCIiRRaSXiM--

As you can see in your payload, the boundary is there:

----WebKitFormBoundaryqiqELBCIiRRaSXiM

The problem i see is: The boundary doesnt come in the original parameters, it should.

Todo:

  1. (worst) If you cant find a way to have this boundary in the events data, then hack it..read the base64 encoded payload and extract the string "----WebKitFormBoundaryqiqELBCIiRRaSXiM" (it will change on every post, so find a way to retrieve it). Then, use this boundary instead of a call to getBoundary().

  2. (possible cause) Maybe your html code (the form) does not explicitly set the "multipart/form-data" attribute. So, your apigateway expects a multipart/form-data payload, and because the form doesnt provide it then the boundary is not set (the browser submitting your form is the responsible to provide it, not apigateway.

  3. (best solution) the getBoundary method is designed to read the boundary from your incoming data from "event.params.header", this header is created by the Apigateway.

    var boundary = multipart.getBoundary(event.params.header['content-type']);

In your apigateway, in the specific method (POST or GET), look at the "Integration Request", then "Body Mapping Templates", and ensure you are providing the right values, this values are created on-the-fly by apigateway using a code-template.

Next code fragement is a copy of my own template (created automatically by apigeteway), this templated is executed by apigateway and it copies values from the apigateway to your service (in json format), so maybe you miss something in this part (look at the "params" part, apigateway will build it, and, it should provide it: events.params.header...).

(you can copy and paste this code into your "Body Mapping Template")

#set($allParams = $input.params())
{
"body-json" : $input.json('$'),
"params" : {
#foreach($type in $allParams.keySet())
    #set($params = $allParams.get($type))
"$type" : {
    #foreach($paramName in $params.keySet())
    "$paramName" : "$util.escapeJavaScript($params.get($paramName))"
        #if($foreach.hasNext),#end
    #end
}
    #if($foreach.hasNext),#end
#end
},
"stage-variables" : {
#foreach($key in $stageVariables.keySet())
"$key" : "$util.escapeJavaScript($stageVariables.get($key))"
    #if($foreach.hasNext),#end
#end
},
"context" : {
    "account-id" : "$context.identity.accountId",
    "api-id" : "$context.apiId",
    "api-key" : "$context.identity.apiKey",
    "authorizer-principal-id" : "$context.authorizer.principalId",
    "caller" : "$context.identity.caller",
    "cognito-authentication-provider" : "$context.identity.cognitoAuthenticationProvider",
    "cognito-authentication-type" : "$context.identity.cognitoAuthenticationType",
    "cognito-identity-id" : "$context.identity.cognitoIdentityId",
    "cognito-identity-pool-id" : "$context.identity.cognitoIdentityPoolId",
    "http-method" : "$context.httpMethod",
    "stage" : "$context.stage",
    "source-ip" : "$context.identity.sourceIp",
    "user" : "$context.identity.user",
    "user-agent" : "$context.identity.userAgent",
    "user-arn" : "$context.identity.userArn",
    "request-id" : "$context.requestId",
    "resource-id" : "$context.resourceId",
    "resource-path" : "$context.resourcePath"
    }
}
jbrosan commented 7 years ago

Hi christiansalazar,

Thank you for your response. You were absolutely correct. The boundary was not being set correctly. A header was being set and was overwriting the boundary information. Once I pulled that out, it all worked great!

Thanks again for your help and your response. I really appreciate it!

Cheers! John

christiansalazar commented 7 years ago

:)