sometimes the response is incorrect (malformed or incomplete) when using phantom-pdf recipe

[bjrmatos]

Hi @pofider i'm encountering a weird issue when rendering a report with the jsreport-client

sometimes the buffer returned from the jsreport-client arrives incomplete causing an invalid pdf (seems like a bug with concat-stream)

in order to detect this behaviour, i needed to compare the size of the buffer returned by jsreport and jsreport-client

i have added in jsreport's codebase a log to know the size of the returned buffer

(jsreport/lib/reporter.js line 187)

return q.ninvoke(this.toner, "render", request).then(function (response) {
        self.logger.info("Rendering request " + request.reportCounter + " finished in " + (new Date() - request.startTime) + " ms");
        response.result = response.stream;

        /* NEW CODE TO LOG AND SAVE PDF GENERATION */
        console.log('jsreport response buffer length:', response.content.length);
        // save the pdf to inspect later
        fs.writeFile('C:\\xampp\\htdocs\\facturactiva\\issuer-integration-api\\dist\\from-jsreport.pdf', response.content);
        /* END NEW CODE */

        return response;
    }).catch(function (e) {
        e.message = "Error during rendering report: " + e.message;
        var logFn = e.weak ? self.logger.warn : self.logger.error;
        logFn("Error when processing render request " + e.message + " " + e.stack);
        throw e;
    });

this is my code using jsreport-client (with the log of the returned buffer from jsreport-client):

var reportClient = jsReportClient(REPORT_SERVER_URL, REPORT_SERVER_USER, REPORT_SERVER_PWD);

var asyncPDFRender = new Promise(function(resolve, reject) {
  reportClient.render({
    template: {
      shortid: 'VkNparuL',
      recipe: 'phantom-pdf'
    },
    data: {  ..my data here..  }
  }, function(err, reportResponse) {
    if (err) {
      return reject(new Error('error when trying to render PDF'));
    }

    resolve(reportResponse);
  });
});

asyncPDFRender.then(function(report) {
  return new Promise(function(resolve, reject) {
    report.body(function(reportBody) {
      /** LOG **/
      console.log('jsreport-client response buffer length:', reportBody.length);

      console.log('jsreport-client response is correct:', reportBody.length === 27776);

      /** END LOG **/

      resolve(writeFileAsync(path.join(__dirname, 'dist', 'report.pdf'), reportBody));
    });
  });
}).then(function() {
  console.log('Reporte generado en PDF!');
}).catch(function(err) {
  console.error('Error:', err);
});

test:

as you can see, sometimes the response is different (buffer size: 19590) compared to the original response from jsreport (buffer size: 27776)

i have modified jsreport-client's codebase to return the buffer from request's (the module) callback instead of the buffer returned by concat-stream

(jsreport-client/lib/client.js line 38)

   if (this.username) {
        requestOptions.auth = {
            username: this.username,
            password: this.password
        }
    }

    // var responseStream = request.post(requestOptions);

    // responseStream.on("error", function(err) {
    //     cb(err);
    // });

    // responseStream.on("response", function (response) {
    //     if (response.statusCode !== 200) {
    //         return responseToBuffer(response, function(data) {
    //             try {
    //                 var errorMessage = JSON.parse(data.toString());
    //                 var error = new Error(errorMessage.message);
    //                 error.remoteStack = errorMessage.stack;
    //                 cb(error)
    //             }
    //             catch(e) {
    //                 cb(new Error(data));
    //             }
    //         });
    //     }

    //     response.body = function(cb) { responseToBuffer(response, cb); }
    //     cb(null, response);
    // });

    /* MODIFIED CODE */
    request.post(requestOptions, function(err, httResp, body) {
      if (err) {
        var error = new Error(err.message);
        error.remoteStack = errorMessage.stack;
        return cb(error);
      }

      cb(null, body);
    });
    /* END MODIFIED CODE */

with this change the result is correct, it never gets corrupted i have tested several times (note that in the change i'm not returning a response object, only the buffer, just in order to test quickly):

preview of report.pdf (invalid pdf)

preview of from-jsreport.pdf

i'm using: jsreport = 1.7.2 node = 0.10.40 (jsreport) nodejs-client = 0.3.0 (last version)

[pofider]

Wow, what a bug report!

The problem with your fix is that you are then loading always the whole pdf into the memory. This is unwanted and that is why I don't let request module to load the output into the body. You can easily pipe the response stream to the file without parsing the body. That would probably also solve your issue.

reportClient.render({...}, function(err, resp) {
  var fs = require('fs'); 
  var wstream = fs.createWriteStream('myOutput.pdf');
  resp.pipe(wstream);
}

However it should also work with the buffers, but unfortunately I was not able to reproduce your issue on the same versions with hundreds of reports. Anyway I tried to remove concat-stream and replaced it with the same reading technique as in request module. Could you please try to replace the following method and try it?

//client.js
//install bl module first
function responseToBuffer(response, cb) {
    var buffer = require("bl")();
    response.on("data", function(chunk) {
        buffer.append(chunk)
    });

    response.on("end", function() {
        cb(buffer.slice());
    });
}

[bjrmatos]

i forgot to mention that i'm using windows 7..

i've updated the code with the bl module but now it fails all the time (version of jsreport-client with bl module -> https://github.com/bjrmatos/nodejs-client/tree/using-buffer-list) :(

i have published my project (jsreport server and client) to let you know more about it.. -> https://github.com/bjrmatos/jsreport-client-incomplete-buffer-response

writing the file with streams fails too, it always get corrupted/incomplete

[pofider]

Thanks for sharing repo, it really helped.

It sounds ridiculous but calling bluebird resolve function somehow breaks the stream. If I save the file directly in the first promise, it is saved correctly. I am desperate from it, but cannot figure out the reason why is it happening. Any idea?

[bjrmatos]

wow It is more rare than I thought :/ let me investigate this in the context of bluebird's promises..

I hope to find a solution that brings full suport to promise's users :)

thnks for your time

[bjrmatos]

i have found some time to keep testing this behaviour and confirm that is only happen if i create a promise based on the stream returned from jsreport-client, this is really weird but seems an issue with the stream loses data on the next tick.. i will try without promises and use setTimeout/process.nextTick in order to verify this

[bjrmatos]

resolved.. nothing is wrong with jsreport-client or any dependencies.

a stream must be consumed in the current loop/tick otherwise it will lose data or simply never notifies

to make it work the code i show before, the following should be added:

var asyncPDFRender = new Promise(function(resolve, reject) {
  reportClient.render({
    template: {
      shortid: 'VkNparuL',
      recipe: 'phantom-pdf'
    },
    data: {  ..my data here..  }
  }, function(err, reportResponse) {
    if (err) {
      return reject(new Error('error when trying to render PDF'));
    }

   /* *******NEW CODE******* */
   // pause the stream before going async
   reportResponse.pause();
  /* ***************************** */
    resolve(reportResponse);
  });
});

asyncPDFRender.then(function(report) {
  return new Promise(function(resolve, reject) {
    /* *******NEW CODE******* */
    // resume the stream before consume it
    report.resume();
    /* ***************************** */
    report.body(function(reportBody) {

      console.log('jsreport-client response buffer length:', reportBody.length);

      console.log('jsreport-client response is correct:', reportBody.length === 27776);

      resolve(writeFileAsync(path.join(__dirname, 'dist', 'report.pdf'), reportBody));
    });
  });
}).then(function() {
  console.log('Reporte generado en PDF!');
}).catch(function(err) {
  console.error('Error:', err);
});

this is the way to handle if someone make the same mistakes than i (not using stream.pipe in the first place)