Support "file" in Swagger (can't validate real media type 'text/plain' against expected media type 'application/schema+json')

[ignaciolarranaga]

Describe your problem

On running the validation of an endpoint with the following spec:

  swagger: "2.0"
  consumes: 
    - "application/json;charset=UTF-8"
  produces: 
    - "application/json;charset=UTF-8"
  host: "localhost:8010"
  securityDefinitions: 
    bearerAuth: 
      type: "apiKey"
      in: "header"
      name: "Authorization"
  info: 
    title: "Sample API"
    description: "Sample"
    version: "1.0.0"
  paths: 
    /transactions/data-sent-and-received.csv: 
      post: 
        summary: "Data Sent and Received Report CSV"
        description: "The CSV data for the data sent and received report"
        produces: 
          - "text/csv"
        parameters: 
          - 
            in: "query"
            name: "page"
            description: "Page you want to retrieve (0 indexed)"
            type: "integer"
            default: 0
            minimum: 0
            x-example: 0
          - 
            in: "query"
            name: "size"
            description: "Size of the page you want to retrieve"
            type: "integer"
            default: 100
            minimum: 1
            x-example: 100
          - 
            in: "query"
            name: "sort"
            description: "Sort criteria"
            type: "string"
            enum: 
              - "TRADING_PARTNER"
              - "TIMESTAMP_ASCENDING"
              - "TIMESTAMP_DESCENDING"
              - "TRANSACTION_SET"
              - "ACCOUNT_NUMBER"
            required: true
            x-example: "TRADING_PARTNER"
          - 
            in: "body"
            name: "body"
            description: "Search Parameters"
            required: true
            schema: 
              $ref: "#/definitions/DataSentAndReceivedParameters"
              example: 
                businessSegmentId: 1
                partnerIds: 
                  - 4585
                  - 4423
                direction: "SENT"
                from: "2017-05-01T00:00:00Z"
                to: "2017-07-21T17:32:28Z"
                transactionSetIds: 
                  - "248-"
                  - "568-"
                  - "810-RPT"
                transactionStatusIds: 
                  - 16
                  - 13
                  - 8
                  - 7
                  - 14
                  - 12
                  - 0
                  - 2
                  - 17
                  - 18
                  - 4
                  - 1
                accountNumber: "326285102500059"
                transactionNumber: "4282.1"
        responses: 
          200: 
            description: "CVS data"
            schema: 
              type: "file"
          401: 
            description: "Unauthorized access"
          500: 
            description: "Unexpected error (internal error)"
  definitions: 
    DataSentAndReceived: 
      allOf: 
        - 
          type: "object"
          description: "A page of results"
          properties: 
            totalElements: 
              description: "The total amount of elements"
              type: "integer"
              format: "int32"
              example: 6
            totalPages: 
              description: "The number of total pages"
              type: "integer"
              example: 1
            number: 
              description: "The number of the current Page"
              type: "integer"
              example: 0
            numberOfElements: 
              description: "The number of elements in the Page"
              type: "integer"
              example: 6
            size: 
              description: "The size of the Page"
              type: "integer"
              example: 100
            sort: 
              description: "The sort criteria used on the Page"
              type: "string"
              example: "TRADING_PARTNER"
        - 
          properties: 
            content: 
              type: "array"
              items: 
                $ref: "#/definitions/DataSentAndReceivedResult"
    DataSentAndReceivedParameters: 
      type: "object"
      description: "Search parameters for data sent and received report"
      properties: 
        businessSegmentId: 
          description: "Business Segment selected to filter the search"
          type: "integer"
        partnerIds: 
          description: "Partners selected to filter the search"
          type: "array"
          items: 
            type: "integer"
        direction: 
          description: "Data Direction (Sent or Received) to filter the search"
          type: "string"
          enum: 
            - "SENT"
            - "RECEIVED"
        from: 
          description: "From date to filter the search"
          type: "string"
          format: "date-time"
        to: 
          description: "To date to filter the search"
          type: "string"
          format: "date-time"
        transactionSetIds: 
          description: "Transaction Sets selected to filter the search"
          type: "array"
          items: 
            type: "string"
        transactionStatusIds: 
          description: "Transaction status selected to filter the search"
          type: "array"
          items: 
            type: "integer"
        accountNumber: 
          description: "Account Number to filter the search"
          type: "string"
        transactionNumber: 
          description: "Transaction Number to filter the search"
          type: "string"
      required: 
        - "businessSegmentId"
        - "partnerIds"
        - "direction"
        - "from"
        - "to"
        - "transactionSetIds"
        - "transactionStatusIds"
    DataSentAndReceivedResult: 
      type: "object"
      description: "Data Sent and Received row"
      properties: 
        transactionId: 
          type: "integer"
          format: "int32"
          example: 54673416
        transactionSetId: 
          type: "string"
          example: "810"
        transactionNumber: 
          type: "string"
          example: "SC30F420140307"
        transactionMethod: 
          type: "string"
          example: "NBEDM 32323"
        direction: 
          type: "string"
          example: "O"
        partnerName: 
          type: "string"
          example: "Energy"
        accountNumber: 
          type: "string"
          example: "3077777777"
        statusCode: 
          type: "string"
        lifecycleCode: 
          type: "string"
          example: "8672014-03-0710.59.27404419"
        date: 
          type: "string"
          example: "2014-03-07"
        time: 
          type: "string"
          example: "14:27:00"
        crDuns: 
          type: "string"
          example: "037597437"
        udcDuns: 
          type: "string"
          example: "006929509"
        interchangeControlNumber: 
          type: "string"
          example: "000000005"
        interchangeStatus: 
          type: "string"
          example: "FAPending"
        groupControlNumber: 
          type: "string"
          example: "1"
        gisbTransactionId: 
          type: "string"
          example: "902092123156473"
        filename: 
          type: "string"
          example: "ECPNOTXPIPE_SKP_810_F4SC30.DT99999999"

I got this error:

info: Beginning Dredd testing...
info: Found Hookfiles: 0=/Users/ignacio/Workspaces/projects/xchange/swagger/src/dredd-hooks.js
hook: ##teamcity[testSuiteStarted name='DREDD']
hook: ##teamcity[testStarted name='POST (200) /transactions/data-sent-and-received.csv?size=100&sort=TRADING_PARTNER']
hook: Adding auth for: /transactions/data-sent-and-received.csv > Data Sent and Received Report CSV > 200
hook: ##teamcity[testFailed name='POST (200) /transactions/data-sent-and-received.csv?size=100&sort=TRADING_PARTNER' message='body: Cant validate real media type text/plain against expected media type application/schema+json.']
hook: ##teamcity[testFinished name='POST (200) /transactions/data-sent-and-received.csv?size=100&sort=TRADING_PARTNER']
fail: POST (200) /transactions/data-sent-and-received.csv?size=100&sort=TRADING_PARTNER duration: 34ms
hook: ##teamcity[testIgnored name='POST (401) /transactions/data-sent-and-received.csv?size=100&sort=TRADING_PARTNER' message='Ignored by dredd.']
hook: Adding auth for: /transactions/data-sent-and-received.csv > Data Sent and Received Report CSV > 401
skip: POST (401) /transactions/data-sent-and-received.csv?size=100&sort=TRADING_PARTNER
hook: ##teamcity[testIgnored name='POST (500) /transactions/data-sent-and-received.csv?size=100&sort=TRADING_PARTNER' message='Ignored by dredd.']
hook: Adding auth for: /transactions/data-sent-and-received.csv > Data Sent and Received Report CSV > 500
skip: POST (500) /transactions/data-sent-and-received.csv?size=100&sort=TRADING_PARTNER
hook: ##teamcity[testSuiteFinished name='DREDD']
info: Displaying failed tests...
fail: POST (200) /transactions/data-sent-and-received.csv?size=100&sort=TRADING_PARTNER duration: 34ms
fail: body: Can't validate real media type 'text/plain' against expected media type 'application/schema+json'.

request: 
method: POST
uri: /transactions/data-sent-and-received.csv?size=100&sort=TRADING_PARTNER
headers: 
    Content-Type: application/json;charset=UTF-8
    User-Agent: Dredd/4.5.0 (Darwin 16.6.0; x64)
    Authorization: Bearer ey....
    Content-Length: 420

body: 
{
  "businessSegmentId": 1,
  "partnerIds": [
    4585,
    4423
  ],
  "direction": "SENT",
  "from": "2017-05-01T00:00:00Z",
  "to": "2017-07-21T17:32:28Z",
  "transactionSetIds": [
    "248-",
    "568-",
    "810-RPT"
  ],
  "transactionStatusIds": [
    16,
    13,
    8,
    7,
    14,
    12,
    0,
    2,
    17,
    18,
    4,
    1
  ],
  "accountNumber": "326285102500059",
  "transactionNumber": "4282.1"
}

expected: 
headers: 

body: 

statusCode: 200
bodySchema: {"type":"file"}

actual: 
statusCode: 200
headers: 
    x-powered-by: Express
    accept-ranges: bytes
    cache-control: public, max-age=0
    last-modified: Mon, 11 Sep 2017 21:05:12 GMT
    etag: W/"1411-15e72c2a340"
    content-type: text/csv; charset=UTF-8
    content-length: 5137
    date: Tue, 12 Sep 2017 19:23:55 GMT
    connection: close

body: 
Unique Trans ID,Partner name,Set ID,Acct No,Unique Trans No,Reject / Status Cd,Life Cycle No,Time Stamp / Date,Timestamp,Dir,Status,Interchange Control No,Group Control No,File Name
60232622,Energy,...

What command line options do you use?

$ dredd apis.yaml http://localhost:8010

What's your dredd --version output?

dredd v4.5.0 (Darwin 16.6.0; x64)

Does dredd --level=debug uncover something?

Nop.

[honzajavorek]

Hi @ignaciolarranaga, first of all, thank you very much for the report! I was able to identify multiple problems.

1. produces/consumes other than application/json are not supported (yet). That's why in the expected section of the Dredd output, you can't even see the text/csv content-type header. This would be a feature request.
2. I don't think Dredd correctly supports {"type":"file"} in the response (yet). Dredd uses JSON Schema to validate the response and this is a Swagger extension to the spec. Linking relevant docs (also mentioned in the Data Types section). Since this is something specific to a certain format, I think Dredd shouldn't be the one dealing with it. The Swagger adapter should take care of the specifics and pass on something which looks like raw HTTP request-response pair and which Dredd already understands - cc @apiaryio/adt This would be a feature request.
3. The output says Can't validate real media type 'text/plain' against expected media type 'application/schema+json', which doesn't make any sense. Dredd should handle the situation more gracefully and provide you with the information something you use isn't supported and cannot be tested. This is a bug in Dredd's user interface.

[nrktkt]

Just dropped by for a +1

produces/consumes other than application/json are not supported (yet)

This would be really useful to have, since even though most APIs are json, it's not uncommon to see something else at some endpoints (like gpx xml for example). I would think an early version of this feature would be easy to get going with plain text matching as a fallback.

[jpjpjp]

I'll add a plus 1 as well.

What would be cool in the interim is if Dredd CAN'T validate against non application/json return bodies that it simply doesn't TRY TO.

While it would be great to have html, plain text and/or csv validation, it would still be useful if dredd could simply validate that it got a response with the expected content-type header and call that a pass for any content-type other than application/json. Right now I get failures for all my APIs.

Happy to provide an example API blueprint and dredd output if it would be helpful.

[honzajavorek]

@jpjpjp Dredd can validate non-JSON payload when using API Blueprint. If the payload isn't JSON-like, the validation is done by comparing the payloads 1 to 1 as a plaintext.

[MikeRalphson]

@honzajavorek is there the possibility of yaml support for OpenAPI?

[honzajavorek]

@MikeRalphson Which one do you mean?

1. reading YAML OpenAPI document (should be supported)
2. correctly producing transactions with YAML media types (should work soon as plaintext comparison after fixing the bugs like this one)
3. smart validation of payloads with YAML media types in Gavel.js (how should this work? YAML converted to JSON and validated with JSON Schema?)

[MikeRalphson]

I'm not familiar with Gavel, but 3. is what I mean. Yes, I believe the YAML representation should be converted to a JSON object and validated with JSON Schema.

[honzajavorek]

@MikeRalphson Hmm. I think you're first to have such feature request. Would you be willing to file it on https://github.com/apiaryio/gavel.js/issues? Gavel.js is the validation engine Dredd uses under the hood. It is basically a tool, which takes an expected and real HTTP transactions and then provides a validation verdict.

[honzajavorek]

Related: apiaryio/fury-adapter-swagger#143

[honzajavorek]

As follow-up to https://github.com/apiaryio/dredd/issues/883#issuecomment-329151438:

1. Partly resolved in current version of the Swagger adapter, but not completely. I filed https://github.com/apiaryio/fury-adapter-swagger/issues/145 for you. I don't think this is the problem causing the issue you have - basically it's duplicate to https://github.com/apiaryio/dredd/issues/553.
2. I filed https://github.com/apiaryio/fury-adapter-swagger/issues/146 for you. I believe that's the main problem this bug report represents.
3. I'm going to work on fixing that at once.

[sajantha]

I am having a soap API . I tried to validate the soap API having the content type : text/xml (POST method.) I have prepared a swagger file without providing the example request body in the file . I stored my request body which is in xml format in a file and passed the content to the request body using below code var requestBody = fs.readFileSync('RequestBody.xml','utf8'); transaction.body=requestBody; in hook file @before But when i execute it , it is throwing me the the exception "syntaxerror unexpected token in json at position 0 " Is it possible to validate a soap api in dredd?

[honzajavorek]

One thing is API using XML content types in its payloads. That should be definitely testable by Dredd.

Another thing is SOAP API. While in theory SOAP should be testable by Dredd, because it's still just plain old HTTP, I think the nature of SOAP API will make it very difficult with Dredd for you and I would personally advise against it. There are many other tools for working with SOAP APIs and checking their correctness.

From the error you mention it seems Dredd is trying to validate the XML payload as JSON. Could you share the relevant part of your Swagger? If you want to skip the validation, check these docs.

[sajantha]

@honzajavorek I havent passed the payload in the swagger . I have given an empty request body , haven't specified anything other than the basepath , content type and header for the request.

Actually my payload somewhat looks like below <soap:Envelope xmlns:soap="http://schemas.xmlsoap.org/soap/envelope/"

... ... ... I am not sure how to define that in apiary. So to pass the payload , what i did was to store the payload in a file , and pass that as a request body in hook file (hooks.before transaction) using the code below. var requestBody = fs.readFileSync('RequestBody.xml','utf8'); transaction.body=requestBody; I was planing to store the response body inside a variable in the hook file itself (var responseBody=transaction.real.body; )and validate it in hooks.beforeValidation. The response is also an xml , similar to that of my request. But the dredd is throwing the exeption in hooks.before transaction itself as it was reading the content as JSON instead of xml as u mentioned. I am not sure how to resolve this issue ...

[honzajavorek]

@sajantha I tried to reproduce your issue, without success. Moreover, I think it's not related at all to the original issue. Following example is working for me correctly, npm test passes.

app.js

const fs = require('fs');
const express = require('express');

const app = express()

app.get('/foo', (req, res) => {
  res.header('content-type', 'application/xml');
  res.send(`<xml>
    <title>ugly SOAP XML</title>
    <envelope>
        <envelope>
            <envelope>
                <envelope>data</envelope>
            </envelope>
        </envelope>
    </envelope>
</xml>
`);
});

app.listen(3000, () =>
  console.log('Listening on http://localhost:3000')
)

hooks.js

const fs = require('fs');
const hooks = require('hooks');

hooks.before('/foo > GET > 200', (transaction, done) => {
  transaction.expected.body = fs.readFileSync('soap.xml', { encoding: 'utf-8' });
  done();
});

soap.xml

<xml>
    <title>ugly SOAP XML</title>
    <envelope>
        <envelope>
            <envelope>
                <envelope>data</envelope>
            </envelope>
        </envelope>
    </envelope>
</xml>

package.json

{
  "name": "dredd-xml",
  "version": "1.0.0",
  "main": "index.js",
  "scripts": {
    "test": "dredd app.yaml http://localhost:3000 \"--server=node app.js\" --hookfiles=hooks.js"
  },
  "license": "ISC",
  "devDependencies": {
    "dredd": "^5.1.8"
  }
}

app.yaml

swagger: "2.0"
info:
  version: "1.0"
  title: "app.js API"
schemes:
  - http
produces:
  - application/xml; charset=utf-8
paths:
  /foo:
    get:
      responses:
        200:
          description: my SOAP response

---

As I mentioned, I wouldn't recommend testing SOAP API by Dredd. Even the example I posted tests the response as a plaintext, which means even change in the (insignificant) white space would break the build. That's probably not something you want to do. Since gavel.js does not support XML directly, you'd have to implement smarter comparison of the XML documents in the hooks yourself.

In case you want to continue discussing your topic, please file a separate issue. Thanks!

[honzajavorek]

Note: The OpenAPI 2 adapter 0.21.0 introduces better support for file in HTTP requests, but it still doesn't seem to work well for HTTP responses.