Support for AWS JSON-1.0 protocol

[nicklaw5]

AWS SDK JS v3.327.0 has changed the way SQS API calls are serialized. It appears they're now using AWS JSON-1.0 protocol. This is causing my requests to ElasticMQ to fail with errors like:

Unexpected token < in JSON at position 0
  Deserialization error: to see the raw response, inspect the hidden field {error}.$response on this object.

Here's a dump of the returned error response:

 HttpResponse {
  statusCode: 400,
  headers: {
    server: 'akka-http/10.2.10',
    date: 'Fri, 05 May 2023 06:16:48 GMT',
    'content-type': 'text/xml; charset=UTF-8',
    'content-length': '323'
  },
  body: <ref *1> IncomingMessage {
    _readableState: ReadableState {
      objectMode: false,
      highWaterMark: 16384,
      buffer: BufferList { head: null, tail: null, length: 0 },
      length: 0,
      pipes: [],
      flowing: false,
      ended: true,
      endEmitted: true,
      reading: false,
      constructed: true,
      sync: true,
      needReadable: false,
      emittedReadable: false,
      readableListening: false,
      resumeScheduled: false,
      errorEmitted: false,
      emitClose: true,
      autoDestroy: true,
      destroyed: true,
      errored: null,
      closed: true,
      closeEmitted: true,
      defaultEncoding: 'utf8',
      awaitDrainWriters: null,
      multiAwaitDrain: false,
      readingMore: true,
      dataEmitted: true,
      decoder: null,
      encoding: null,
      [Symbol(kPaused)]: true
    },
    _events: [Object: null prototype] {
      end: [Function: responseOnEnd],
      error: [Function (anonymous)]
    },
    _eventsCount: 2,
    _maxListeners: undefined,
    socket: null,
    httpVersionMajor: 1,
    httpVersionMinor: 1,
    httpVersion: '1.1',
    complete: true,
    rawHeaders: [
      'Server',
      'akka-http/10.2.10',
      'Date',
      'Fri, 05 May 2023 06:16:48 GMT',
      'Content-Type',
      'text/xml; charset=UTF-8',
      'Content-Length',
      '323'
    ],
    rawTrailers: [],
    joinDuplicateHeaders: undefined,
    aborted: false,
    upgrade: false,
    url: '',
    method: null,
    statusCode: 400,
    statusMessage: 'Bad Request',
    client: Socket {
      connecting: false,
      _hadError: false,
      _parent: null,
      _host: 'localhost',
      _closeAfterHandlingError: false,
      _readableState: [ReadableState],
      _events: [Object: null prototype],
      _eventsCount: 6,
      _maxListeners: undefined,
      _writableState: [WritableState],
      allowHalfOpen: false,
      _sockname: null,
      _pendingData: null,
      _pendingEncoding: '',
      server: null,
      _server: null,
      parser: null,
      _httpMessage: null,
      timeout: 0,
      [Symbol(async_id_symbol)]: -1,
      [Symbol(kHandle)]: [TCP],
      [Symbol(lastWriteQueueSize)]: 0,
      [Symbol(timeout)]: null,
      [Symbol(kBuffer)]: null,
      [Symbol(kBufferCb)]: null,
      [Symbol(kBufferGen)]: null,
      [Symbol(kCapture)]: false,
      [Symbol(kSetNoDelay)]: true,
      [Symbol(kSetKeepAlive)]: true,
      [Symbol(kSetKeepAliveInitialDelay)]: 1,
      [Symbol(kBytesRead)]: 0,
      [Symbol(kBytesWritten)]: 0
    },
    _consuming: false,
    _dumped: false,
    req: ClientRequest {
      _events: [Object: null prototype],
      _eventsCount: 4,
      _maxListeners: undefined,
      outputData: [],
      outputSize: 0,
      writable: true,
      destroyed: true,
      _last: false,
      chunkedEncoding: false,
      shouldKeepAlive: true,
      maxRequestsOnConnectionReached: false,
      _defaultKeepAlive: true,
      useChunkedEncodingByDefault: true,
      sendDate: false,
      _removedConnection: false,
      _removedContLen: false,
      _removedTE: false,
      strictContentLength: false,
      _contentLength: '71',
      _hasBody: true,
      _trailer: '',
      finished: true,
      _headerSent: true,
      _closed: true,
      socket: [Socket],
      _header: 'POST / HTTP/1.1\r\n' +
        'content-type: application/x-amz-json-1.0\r\n' +
        'x-amz-target: AmazonSQS.SendMessage\r\n' +
        'content-length: 71\r\n' +
        'host: localhost:9324\r\n' +
        'x-amz-user-agent: aws-sdk-js/3.327.0\r\n' +
        'user-agent: aws-sdk-js/3.327.0 os/linux/5.15.90.1-microsoft-standard-WSL2 lang/js md/nodejs/18.16.0 api/sqs/3.327.0\r\n' +
        'amz-sdk-invocation-id: 987f848e-f4b2-458f-b4de-61df956ed260\r\n' +
        'amz-sdk-request: attempt=1; max=3\r\n' +
        'x-amz-date: 20230505T061648Z\r\n' +
        'x-amz-content-sha256: 5eef0e1a2d02a9de4f8740f4a30aadf88f049fd595d8d4d03595445f5b70f113\r\n' +   
        'authorization: AWS4-HMAC-SHA256 Credential=AKIAYUMTPAEO37AMJVUH/20230505/ap-southeast-2/sqs/aws4_request, SignedHeaders=amz-sdk-invocation-id;amz-sdk-request;content-length;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-target;x-amz-user-agent, Signature=cc1057dcb5e12445c70ec22e542c55473da350c89661f35bdc74e31061315313\r\n' +
        'Connection: keep-alive\r\n' +
        '\r\n',
      _keepAliveTimeout: 0,
      _onPendingData: [Function: nop],
      agent: [Agent],
      socketPath: undefined,
      method: 'POST',
      maxHeaderSize: undefined,
      insecureHTTPParser: undefined,
      joinDuplicateHeaders: undefined,
      path: '/',
      _ended: true,
      res: [Circular *1],
      aborted: false,
      timeoutCb: null,
      upgradeOrConnect: false,
      parser: null,
      maxHeadersCount: null,
      reusedSocket: true,
      host: 'localhost',
      protocol: 'http:',
      [Symbol(kCapture)]: false,
      [Symbol(kBytesWritten)]: 0,
      [Symbol(kNeedDrain)]: false,
      [Symbol(corked)]: 0,
      [Symbol(kOutHeaders)]: [Object: null prototype],
      [Symbol(errored)]: null,
      [Symbol(kUniqueHeaders)]: null
    },
    [Symbol(kCapture)]: false,
    [Symbol(kHeaders)]: {
      server: 'akka-http/10.2.10',
      date: 'Fri, 05 May 2023 06:16:48 GMT',
      'content-type': 'text/xml; charset=UTF-8',
      'content-length': '323'
    },
    [Symbol(kHeadersCount)]: 8,
    [Symbol(kTrailers)]: null,
    [Symbol(kTrailersCount)]: 0
  }
}

Once I bumped the AWS SDK JS back to v3.326.0 all worked fine again.

[mikkael131]

Same problem when using the latest awscli v1.27.127 with elasticmq

e.g. creating a new SQS Queue results in:

An error occurred (400) when calling the CreateQueue operation: <ErrorResponse xmlns="http://queue.amazonaws.com/doc/2012-11-05/">
      <Error>
        <Type>Sender</Type>
        <Code>MissingAction</Code>
        <Message>MissingAction; see the SQS docs.</Message>
        <Detail/>
      </Error>
      <RequestId>00000000-0000-0000-0000-000000000000</RequestId>
    </ErrorResponse>

The fix is to downgrade the awscli to v1.27.126: pip install awscli==1.27.126

[eblin]

Same problem as well

botocore 1.29.127 introduced this breaking change, it has been reverted in 1.29.128 see: https://github.com/boto/botocore/pull/2931

elasticmq works fine with 1.29.128 👍

[micossow]

Hey, thanks for reporting the issue. The change would be quite big, so it will take some time to implement.

[micossow]

The json support has been implemented in 1.4.0 thanks to the amazing job by @katlasik and @flsh86 If you find any issue, please report.