preprocess groupphoto images

clif-os commented 4 years ago

uses s3.getObject instead of ajax to fetch image buffers
preprocesses the gif's middle frame and uploads it to s3 as a JPEG under /public/photos
processing now takes about 20-35s for 1,800 images, instead of 1-2 minutes
use websockets to communicate group photo success/failure (since heroku times out after 30s)

clif-os commented 4 years ago

@julianduque would you like me to write an endpoint that uses your /deleteObj fn() to remove both the gif and the jpeg by their filename?

crcastle commented 3 years ago

Hey @clif-os - These changes are causing an error for me. I'm getting an error when I click "Share with conference" after recording a GIF.

Dev Console:

Heroku app log:

2020-10-16T21:34:24.825721+00:00 app[web.1]: [0] MissingRequiredParameter: Missing required key 'Bucket' in params
2020-10-16T21:34:24.825734+00:00 app[web.1]: [0]     at ParamValidator.fail (/app/node_modules/aws-sdk/lib/param_validator.js:50:37)
2020-10-16T21:34:24.825735+00:00 app[web.1]: [0]     at ParamValidator.validateStructure (/app/node_modules/aws-sdk/lib/param_validator.js:61:14)
2020-10-16T21:34:24.825735+00:00 app[web.1]: [0]     at ParamValidator.validateMember (/app/node_modules/aws-sdk/lib/param_validator.js:88:21)
2020-10-16T21:34:24.825736+00:00 app[web.1]: [0]     at ParamValidator.validate (/app/node_modules/aws-sdk/lib/param_validator.js:34:10)
2020-10-16T21:34:24.825736+00:00 app[web.1]: [0]     at Request.VALIDATE_PARAMETERS (/app/node_modules/aws-sdk/lib/event_listeners.js:132:42)
2020-10-16T21:34:24.825737+00:00 app[web.1]: [0]     at Request.callListeners (/app/node_modules/aws-sdk/lib/sequential_executor.js:106:20)
2020-10-16T21:34:24.825737+00:00 app[web.1]: [0]     at callNextListener (/app/node_modules/aws-sdk/lib/sequential_executor.js:96:12)
2020-10-16T21:34:24.825737+00:00 app[web.1]: [0]     at /app/node_modules/aws-sdk/lib/event_listeners.js:86:9
2020-10-16T21:34:24.825738+00:00 app[web.1]: [0]     at finish (/app/node_modules/aws-sdk/lib/config.js:386:7)
2020-10-16T21:34:24.825738+00:00 app[web.1]: [0]     at /app/node_modules/aws-sdk/lib/config.js:428:9 {
2020-10-16T21:34:24.825739+00:00 app[web.1]: [0]   code: 'MissingRequiredParameter',
2020-10-16T21:34:24.825739+00:00 app[web.1]: [0]   time: 2020-10-16T21:34:24.820Z
2020-10-16T21:34:24.825740+00:00 app[web.1]: [0] }
2020-10-16T21:34:34.625856+00:00 heroku[router]: at=info method=POST path="/uploadBlob" host=murmuring-spire-43170.herokuapp.com request_id=cbf7e20b-b792-4780-84e3-b1a4d8ea38cc fwd="204.14.236.212" dyno=web.1 connect=0ms service=62ms status=200 bytes=448 protocol=https
2020-10-16T21:34:35.213936+00:00 heroku[router]: at=info method=POST path="/video2gif" host=murmuring-spire-43170.herokuapp.com request_id=e27fb0e4-db65-4504-a9cf-da3e2fd09367 fwd="204.14.236.212" dyno=web.1 connect=0ms service=572ms status=200 bytes=282 protocol=https
2020-10-16T21:34:35.261254+00:00 heroku[router]: at=info method=GET path="/img?filename=e74fe432-e123-41d3-9b4e-fc3224c552fb&reload=false" host=murmuring-spire-43170.herokuapp.com request_id=0df0a6f1-af58-4fee-9fa4-a737acfa544c fwd="204.14.236.212" dyno=web.1 connect=0ms service=29ms status=200 bytes=554860 protocol=https
2020-10-16T21:34:46.108916+00:00 heroku[router]: at=info method=POST path="/video2gif" host=murmuring-spire-43170.herokuapp.com request_id=9faed405-25f3-44fa-91b7-6a9fb0f071e1 fwd="204.14.236.212" dyno=web.1 connect=0ms service=378ms status=200 bytes=312 protocol=https
2020-10-16T21:34:46.167140+00:00 heroku[router]: at=info method=GET path="/img?filename=e74fe432-e123-41d3-9b4e-fc3224c552fb&reload=true" host=murmuring-spire-43170.herokuapp.com request_id=52f4e124-bc2f-4a4e-ac51-fef3661bf1e6 fwd="204.14.236.212" dyno=web.1 connect=0ms service=41ms status=200 bytes=548442 protocol=https
2020-10-16T21:34:49.498046+00:00 heroku[router]: at=info method=POST path="/uploadGIF" host=murmuring-spire-43170.herokuapp.com request_id=4dfab14a-ff38-4083-a91b-e75186c1bef4 fwd="204.14.236.212" dyno=web.1 connect=0ms service=2037ms status=500 bytes=348 protocol=https
2020-10-16T21:34:49.495194+00:00 app[web.1]: [0] MissingRequiredParameter: Missing required key 'Bucket' in params
2020-10-16T21:34:49.495219+00:00 app[web.1]: [0]     at ParamValidator.fail (/app/node_modules/aws-sdk/lib/param_validator.js:50:37)
2020-10-16T21:34:49.495228+00:00 app[web.1]: [0]     at ParamValidator.validateStructure (/app/node_modules/aws-sdk/lib/param_validator.js:61:14)
2020-10-16T21:34:49.495228+00:00 app[web.1]: [0]     at ParamValidator.validateMember (/app/node_modules/aws-sdk/lib/param_validator.js:88:21)
2020-10-16T21:34:49.495229+00:00 app[web.1]: [0]     at ParamValidator.validate (/app/node_modules/aws-sdk/lib/param_validator.js:34:10)
2020-10-16T21:34:49.495253+00:00 app[web.1]: [0]     at Request.VALIDATE_PARAMETERS (/app/node_modules/aws-sdk/lib/event_listeners.js:132:42)
2020-10-16T21:34:49.495253+00:00 app[web.1]: [0]     at Request.callListeners (/app/node_modules/aws-sdk/lib/sequential_executor.js:106:20)
2020-10-16T21:34:49.495254+00:00 app[web.1]: [0]     at callNextListener (/app/node_modules/aws-sdk/lib/sequential_executor.js:96:12)
2020-10-16T21:34:49.495254+00:00 app[web.1]: [0]     at /app/node_modules/aws-sdk/lib/event_listeners.js:86:9
2020-10-16T21:34:49.495255+00:00 app[web.1]: [0]     at finish (/app/node_modules/aws-sdk/lib/config.js:386:7)
2020-10-16T21:34:49.495256+00:00 app[web.1]: [0]     at /app/node_modules/aws-sdk/lib/config.js:428:9 {
2020-10-16T21:34:49.495257+00:00 app[web.1]: [0]   code: 'MissingRequiredParameter',
2020-10-16T21:34:49.495257+00:00 app[web.1]: [0]   time: 2020-10-16T21:34:49.493Z
2020-10-16T21:34:49.495257+00:00 app[web.1]: [0] } MissingRequiredParameter: Missing required key 'Bucket' in params
2020-10-16T21:34:49.495258+00:00 app[web.1]: [0]     at ParamValidator.fail (/app/node_modules/aws-sdk/lib/param_validator.js:50:37)
2020-10-16T21:34:49.495258+00:00 app[web.1]: [0]     at ParamValidator.validateStructure (/app/node_modules/aws-sdk/lib/param_validator.js:61:14)
2020-10-16T21:34:49.495258+00:00 app[web.1]: [0]     at ParamValidator.validateMember (/app/node_modules/aws-sdk/lib/param_validator.js:88:21)
2020-10-16T21:34:49.495259+00:00 app[web.1]: [0]     at ParamValidator.validate (/app/node_modules/aws-sdk/lib/param_validator.js:34:10)
2020-10-16T21:34:49.495259+00:00 app[web.1]: [0]     at Request.VALIDATE_PARAMETERS (/app/node_modules/aws-sdk/lib/event_listeners.js:132:42)
2020-10-16T21:34:49.495259+00:00 app[web.1]: [0]     at Request.callListeners (/app/node_modules/aws-sdk/lib/sequential_executor.js:106:20)
2020-10-16T21:34:49.495260+00:00 app[web.1]: [0]     at callNextListener (/app/node_modules/aws-sdk/lib/sequential_executor.js:96:12)
2020-10-16T21:34:49.495260+00:00 app[web.1]: [0]     at /app/node_modules/aws-sdk/lib/event_listeners.js:86:9
2020-10-16T21:34:49.495260+00:00 app[web.1]: [0]     at finish (/app/node_modules/aws-sdk/lib/config.js:386:7)
2020-10-16T21:34:49.495261+00:00 app[web.1]: [0]     at /app/node_modules/aws-sdk/lib/config.js:428:9 temp/e74fe432-e123-41d3-9b4e-fc3224c552fb.gif
2020-10-16T21:34:52.804172+00:00 heroku[router]: at=error code=H12 desc="Request timeout" method=GET path="/listGifs" host=murmuring-spire-43170.herokuapp.com request_id=6b9b50f5-0fde-4fc6-a0c9-3b46ef18f85a fwd="204.14.236.212" dyno=web.1 connect=1ms service=30000ms status=503 bytes=0 protocol=https
2020-10-16T21:34:54.835726+00:00 app[web.1]: [0] MissingRequiredParameter: Missing required key 'Bucket' in params
2020-10-16T21:34:54.835762+00:00 app[web.1]: [0]     at ParamValidator.fail (/app/node_modules/aws-sdk/lib/param_validator.js:50:37)
2020-10-16T21:34:54.835780+00:00 app[web.1]: [0]     at ParamValidator.validateStructure (/app/node_modules/aws-sdk/lib/param_validator.js:61:14)
2020-10-16T21:34:54.835780+00:00 app[web.1]: [0]     at ParamValidator.validateMember (/app/node_modules/aws-sdk/lib/param_validator.js:88:21)
2020-10-16T21:34:54.835781+00:00 app[web.1]: [0]     at ParamValidator.validate (/app/node_modules/aws-sdk/lib/param_validator.js:34:10)
2020-10-16T21:34:54.835781+00:00 app[web.1]: [0]     at Request.VALIDATE_PARAMETERS (/app/node_modules/aws-sdk/lib/event_listeners.js:132:42)
2020-10-16T21:34:54.835781+00:00 app[web.1]: [0]     at Request.callListeners (/app/node_modules/aws-sdk/lib/sequential_executor.js:106:20)
2020-10-16T21:34:54.835782+00:00 app[web.1]: [0]     at callNextListener (/app/node_modules/aws-sdk/lib/sequential_executor.js:96:12)
2020-10-16T21:34:54.835782+00:00 app[web.1]: [0]     at /app/node_modules/aws-sdk/lib/event_listeners.js:86:9
2020-10-16T21:34:54.835783+00:00 app[web.1]: [0]     at finish (/app/node_modules/aws-sdk/lib/config.js:386:7)
2020-10-16T21:34:54.835784+00:00 app[web.1]: [0]     at /app/node_modules/aws-sdk/lib/config.js:428:9 {
2020-10-16T21:34:54.835785+00:00 app[web.1]: [0]   code: 'MissingRequiredParameter',
2020-10-16T21:34:54.835785+00:00 app[web.1]: [0]   time: 2020-10-16T21:34:54.832Z
2020-10-16T21:34:54.835786+00:00 app[web.1]: [0] }
2020-10-16T21:34:57.209761+00:00 heroku[router]: at=info method=GET path="/static/css/main.50440270.chunk.css" host=murmuring-spire-43170.herokuapp.com request_id=df34f859-6ef5-4275-b420-44ba725e3847 fwd="204.14.236.212" dyno=web.1 connect=0ms service=3ms status=304 bytes=238 protocol=https
2020-10-16T21:34:57.211009+00:00 heroku[router]: at=info method=GET path="/manifest.json" host=murmuring-spire-43170.herokuapp.com request_id=4d1774ad-c2c8-49f9-8cd5-b1be83bda91f fwd="204.14.236.212" dyno=web.1 connect=1ms service=2ms status=304 bytes=237 protocol=https

clif-os commented 3 years ago

Hey @crcastle I launched this branch here and am not seeing that issue: https://gif-booth-pr-34.herokuapp.com/new-gif

Is the bucket you're using a bucket that already exists? (This change cannot be placed onto an existing gif booth bucket as it diverges from the file/folder structure).

developerforce / gif-booth

preprocess groupphoto images #34