Meteor crashes when trying to access to resized file uploaded to S3.

[Dominis13]

Hello. I getting error listed below when trying to get thumbnail from my collection with S3 storage. Trace:

http.js:732
    throw new Error('Can\'t render headers after they are sent to the client.'
          ^
Error: Can't render headers after they are sent to the client.
  at ServerResponse.OutgoingMessage._renderHeaders (http.js:732:11)
  at ServerResponse.writeHead (http.js:1153:20)
  at ProxyServer.<anonymous> (/home/dominis/.meteor/packages/meteor-tool/.1.1.3.1xnkap++os.linux.x86_64+web.browser+web.cordova/mt-os.linux.x86_64/tools/run-proxy.js:96:21)
  at ProxyServer.emit (/home/dominis/.meteor/packages/meteor-tool/.1.1.3.1xnkap++os.linux.x86_64+web.browser+web.cordova/mt-os.linux.x86_64/dev_bundle/lib/node_modules/http-proxy/node_modules/eventemitter3/index.js:100:27)
  at ClientRequest.proxyError (/home/dominis/.meteor/packages/meteor-tool/.1.1.3.1xnkap++os.linux.x86_64+web.browser+web.cordova/mt-os.linux.x86_64/dev_bundle/lib/node_modules/http-proxy/lib/http-proxy/passes/web-incoming.js:140:16)
  at ClientRequest.emit (events.js:117:20)
  at Socket.socketOnData (http.js:1593:9)
  at TCP.onread (net.js:528:27)

Crash doesn't happens if i disable the resize callback. Here is code for collection initializing

S3opt = {} //settings for S3
photosStore = new FS.Store.S3("photos", S3opt)
transformFunc = (fileObj, readStream, writeStream) ->
  gm(readStream, fileObj.name()).resize('281').stream().pipe(writeStream)
S3opt['transformWrite'] = transformFunc
thumbStore = new FS.Store.S3("thumbs", S3opt)
localThumb = new FS.Store.FileSystem("lthumbs", transformWrite: transformFunc)

@Photos = new FS.Collection 'photos',
  useHttp: yes
  stores: [photosStore, thumbStore, localThumb],
  filter:
    allow:
      contentTypes: ['image/*']

As i said above problem occurs only when transformFunc is turned on, and only for S3. I was add localStorage for test purposes and getting image from there work fine.

I take the url for image like that:

collectionItem.url({store: 'thumbs'})

And when trying to go through this url i got only little part of image (about 10%) before Meteor crashes.

[prik]

I'm having the exact same issue. Any help is appreciated.

[bitomule]

Same issue here, not hapenning before I did a package update (one week ago +-)

[verdavaine]

Same issue

[rhyslbw]

@raix, any ideas on why S3 would be causing the crash?

[mxab]

I had a similar problem I think, can you check if the file on s3 is there as is supposed to be and can you compare the filesize with that is stored in the collection for the store/file with the actual filesize on s3?

[raix]

not sure, is the data for the resized image stored correctly on the S3?

[bitomule]

In my case I'm using S3 as tempstore too, previous to that change I didn't have that problem.

[JohnBernardsson]

I am / was having the same problem. In general seems to be doing some strange things when using different stores with transformations (I didn't check with different stores without transforming the images).

I was having the exact same error: 'Can\'t render headers after they are sent to the client.'

I had a lot of images in my S3 bucket, so I removed all filerecords from the collection, and the bucket folder with the images was cleaned (by the way this is awesome).

After that, I tested again the code. I didn't get the error, but I neither got the expected results. I am trying to generate 3 versions of the image (code below), but in the bucket seems to be storing just one of them (seems to be the last in the stores[] array, I'll do more tests on this).

Now Meteor is not crashing (that scared me a lot, it was simply crashing and stopping), but I am just able to get one of the image versions (because obviously seems that only that one is generated).

I am posting the code I am using for the process (maybe I am posting some unncesary code for the problem, but just in case...):

This is my code in the server:

//=========== global.config.js =============//
AWSImgConfig = {
    'accessKeyId': '#####',
    'secretAccessKey': '####',
    'bucket': '#####',
    'acl': 'public-read',
    'keyStart': 'uploads/'
};

//============ imageStores.js ==============//
ImageStores = {};

var CommonConf = {
    accessKeyId: AWSImgConfig.accessKeyId, 
    secretAccessKey: AWSImgConfig.secretAccessKey, 
    bucket: AWSImgConfig.bucket,
    folder: AWSImgConfig.keyStart,
    ACL: AWSImgConfig.acl
};

//Event image stores
ImageStores.eventImageStore = new FS.Store.S3('eventImage', CommonConf);

ImageStores.eventPreviewImageStore = new FS.Store.S3('eventPreviewImg', 
    _.extend(CommonConf, {
        transformWrite: function(fileObj, readStream, writeStream) {
            gm(readStream, fileObj.name()).resize('550', '550').stream().pipe(writeStream);
        }
    })
);

ImageStores.eventThumbnailStore = new FS.Store.S3('eventThumbnail', 
    _.extend(CommonConf, {
        transformWrite: function(fileObj, readStream, writeStream) {
            gm(readStream, fileObj.name()).resize('150', '150').stream().pipe(writeStream);
        }
    })
);
//END Event image stores

//================ eventImages.js ==================//
Meteor.publish("eventImage", function(imageId){
    return EventImages.find({'_id': imageId});
});

//Define permissions for EventImages collection
EventImages.allow({
    insert: function(userId, file){
        return (
            userId && 
            userId === file.metadata.ownerId &&
            Roles.userIsInRole(userId, ['publisher', 'admin'])
        );
    },
    update: function(userId, file){
        return (
            userId && 
            userId === file.metadata.ownerId &&
            Roles.userIsInRole(userId, ['publisher', 'admin'])
        );
    },
    download:function(){
        return true;
    }
});

This is the shared code between client and server:

//Create "mocks" of ImageStores for client code
if (Meteor.isClient) {
    ImageStores = window.ImageStores || {};
    ImageStores.eventImageStore = new FS.Store.S3("eventImage");
    ImageStores.eventPreviewImageStore = new FS.Store.S3("eventPreviewImg");
    ImageStores.eventThumbnailStore = new FS.Store.S3("eventThumbnail");
}

//Define Collection
EventImages = new FS.Collection("eventimages", {
    stores: [
        ImageStores.eventImageStore,
        ImageStores.eventPreviewImageStore,
        ImageStores.eventThumbnailStore
    ],
    filter: {
        maxSize: 5242880, // in bytes
        allow: {
            contentTypes: ['image/*'],
            extensions: ['png', 'jpg', 'jpeg']
        },
        onInvalid: function (message) {
            console.log(message);
        }
    }
});

This is client code: I am using Meteor-Angular here, and surely there is a better way of doing this, but well, here it goes anyways:

//========== javascript code ========//
//Create new FS File
newImage = new FS.File(image); //Image is web File type
newImage.metadata = {
    ownerId: $rootScope.currentUser._id
};
//Add to images collection and upload it to server
EventImages.insert(newImage, function (err, imageObj) {
    console.log('Inserted new doc with ID ' + imageObj._id + ', and kicked off the data upload using HTTP');
    //If we already have any subscription to event image, stop it
    $scope.imageEventSubs && $scope.imageEventSubs.stop();
    $scope.imageEventSubsHandle && $scope.imageEventSubsHandle.stop();
    //Subscribe so we can check the upload status reactively
    $scope.$meteorSubscribe('eventImage', imageObj._id).then(function(subscriptionHandle){
        $scope.imageEventSubsHandle = subscriptionHandle;
        $scope.imageEventSubs = $meteor.collection(EventImages, false);
    });
    //Check upload state
    $scope.imgUpInterval && $interval.cancel($scope.imgUpInterval);
    $scope.imgUpInterval = $interval($scope.checkUpload, 650);
    //TODO: Do something fancy like showing spinner, loader, whatever
});

$scope.checkUpload = function() {
    if ($scope.imageEventSubs && $scope.imageEventSubs.length && $scope.imageEventSubs[0].isUploaded()){
        //Put image on scope to show it
        $scope.eventImage = $scope.imageEventSubs[0];
        //Set the imageId in the eventData
        $scope.event.mainImageId = $scope.eventImage._id;
        $interval.cancel($scope.imgUpInterval);
        //TODO: Remove fancy thing
    }
};

<!--=========== html / angular template =============-->
...
<div>
    <img ng-if="eventImage" ng-src="{{eventImage.url({store:'eventImage'})}}"/>
    <img ng-if="eventImage" ng-src="{{eventImage.url({store:'eventImgPreview'})}}"/>
    <img ng-if="eventImage" ng-src="{{eventImage.url({store:'eventThumbnail'})}}"/>
</div>
...

I hope it's useful... I know I shouldn't use CollectionFS in production, but actually that's what I am planning todo... :sweat: So the thing that Meteor just crashes and stops scares me a lot.

By the way this is on Meteor 1.1.0.2 in Windows. I'll try to check this in a linux server at Amazon, and see what happens.

Cheers!

[mxab]

@EvilJohnny can you change your transformation code to something like this, sometime the simple stream() will swallow some errors

instead of

.stream().pipe(writeStream);

try

.stream(function(err, stdout, stderr){
 if(err){
  console.error("transform failed", error)
 }
 stderr.pipe(process.stderr)
 stdout.pipe(writeStream)
});

[verdavaine]

@EvilJohnny Just use a different folder for each store. Than u will get all your image versions and u will noot see the error anymore: 'Can\'t render headers after they are sent to the client.

[JohnBernardsson]

@verdavaine You don't know how much love I feel for you right now :smile: It worked! Thanks so much!!!

@mxab I tried changing the code to:

ImageStores.eventImageStore = new FS.Store.S3('eventImage', CommonConf);

ImageStores.eventPreviewImageStore = new FS.Store.S3('eventPreviewImg', 
    _.extend(CommonConf, {
        transformWrite: function(fileObj, readStream, writeStream) {
            gm(readStream, fileObj.name()).resize('550', '550')
            .stream(function(err, stdout, stderr){
             if(err){
              console.error("transform 550 failed", err);
             }
             stderr.pipe(process.stderr);
             stdout.pipe(writeStream);
            });
        }
    })
);

ImageStores.eventThumbnailStore = new FS.Store.S3('eventThumbnail', 
    _.extend(CommonConf, {
        transformWrite: function(fileObj, readStream, writeStream) {
            gm(readStream, fileObj.name()).resize('150', '150')
            .stream(function(err, stdout, stderr){
             if(err){
              console.error("transform 150 failed", err);
             }
             stderr.pipe(process.stderr);
             stdout.pipe(writeStream);
            });
        }
    })
);

But no luck: same error, no more info... This is the complete error:

http.js:732
    throw new Error('Can\'t render headers after they are sent to the client.'
          ^
Error: Can't render headers after they are sent to the client.
    at ServerResponse.OutgoingMessage._renderHeaders (http.js:732:11)
    at ServerResponse.writeHead (http.js:1153:20)
    at ProxyServer.<anonymous> (C:\Users\Juan\AppData\Local\.meteor\packages\meteor-tool\1.1.3\mt-os.windows.x86_32\tools\run-proxy.js:96:21)
    at ProxyServer.emit (C:\Users\Juan\AppData\Local\.meteor\packages\meteor-tool\1.1.3\mt-os.windows.x86_32\dev_bundle\lib\node_modules\eventemitter3\index.js:100:27)
    at ClientRequest.proxyError (C:\Users\Juan\AppData\Local\.meteor\packages\meteor-tool\1.1.3\mt-os.windows.x86_32\dev_bundle\lib\node_modules\http-proxy\lib\http-proxy\passes\web-incoming.js:140:16)
    at ClientRequest.emit (events.js:117:20)
    at Socket.socketOnData (http.js:1593:9)
    at TCP.onread (net.js:528:27)

One question that might be offtopic... There is some "secure" mode of running Meteor to avoid it stopping because of errors like this one? (Maybe I am wrong, but doesn't seem so critic error to stop everything and just exit).

Again thanks @verdavaine . @mxab , if you want me to do some other tests let me know :)