RequestTimeTooSkewed: The difference between the request time and the current time is too large

[samueleastdev]

Describe the bug When uploading a folder with a large number of files in it the uploader will fail with the error.

RequestTimeTooSkewed: The difference between the request time and the current time is too large

SDK version number "aws-sdk": "^2.894.0",

To Reproduce (observed behavior)

let AWS = require("aws-sdk");

const fs = require('fs');

const path = require('path');

const walk = require('walk');

const s3 = new AWS.S3();

const params = [];

let aws_folder_name = 'hls_folder';

let folder_path = '/Users/dave/Desktop/hls_folder';

// Walker options
let walker  = walk.walk(folder_path, { followLinks: false });

walker.on('file', function(root, stat, next) {

    let file_path = root + '/' + stat.name;

    let aws_data = {
        Bucket: 'example-deliver', 
        Key: aws_folder_name + file_path.replace(folder_path, ""), 
        Body: fs.readFileSync(file_path) 
    };

    params.push(aws_data);

    next();

});

walker.on('end', function() {

    let progress = 1;

    let total_files = params.length;

    const onProgress = async promise => {

        const result = await promise;

        console.log('total_files', total_files);
        console.log('progress', progress);

        if(progress >= total_files){

            console.log('Done');

        }else{

            console.log(progress / total_files * 100 + '%');

        }

        progress++;

        return result;

    };

    // Start function
    const start = async function(a, b) {

        console.time('test');

        const responses = await Promise.all(
            params.map(param => onProgress(s3.upload(param).promise()))
        );

        console.timeEnd('test');

    }

    // Call start
    start();

});

Run the script above on a folder with lots of files in I am testing with a folder of a large movie encoded to hls.

After about 30 minutes sometimes more it fails with this message.

RequestTimeTooSkewed: The difference between the request time and the current time is too large

Full Error

total_files 8312
progress 2165
26.046679499518767%
(node:9665) UnhandledPromiseRejectionWarning: Error: RequestTimeTooSkewed: The difference between the request time and the current time is too large.
    at start (/Users/dave/Documents/uploader/index.js:99:20)
    at runMicrotasks (<anonymous>)
    at processTicksAndRejections (internal/process/task_queues.js:93:5)
(Use `node --trace-warnings ...` to show where the warning was created)
(node:9665) UnhandledPromiseRejectionWarning: Unhandled promise rejection. This error originated either by throwing inside of an async function without a catch block, or by rejecting a promise which was not handled with .catch(). To terminate the node process on unhandled promise rejection, use the CLI flag `--unhandled-rejections=strict` (see https://nodejs.org/api/cli.html#cli_unhandled_rejections_mode). (rejection id: 1)
(node:9665) [DEP0018] DeprecationWarning: Unhandled promise rejections are deprecated. In the future, promise rejections that are not handled will terminate the Node.js process with a non-zero exit code.

[samueleastdev]

I can get it to complete the upload by catching the error and running the uploader again like this.

But still curious why this error keeps happening.

let AWS = require("aws-sdk");

const fs = require('fs');

const path = require('path');

const walk = require('walk');

const s3 = new AWS.S3();

function upload(){

    console.log('uploader running...');

    const params = [];

    let aws_folder_name = 'hls_folder';

    let folder_path = '/Users/dave/Desktop/hls_folder';

    // Walker options
    let walker  = walk.walk(folder_path, { followLinks: false });

    walker.on('file', function(root, stat, next) {

        let file_path = root + '/' + stat.name;

        let aws_data = {
            Bucket: 'example-deliver', 
            Key: aws_folder_name + file_path.replace(folder_path, ""), 
            Body: fs.readFileSync(file_path) 
        };

        params.push(aws_data);

        next();

    });

    walker.on('end', function() {

        let progress = 1;

        let total_files = params.length;

        const onProgress = async promise => {

            const result = await promise;

            console.log(progress / total_files * 100 + '%');

            progress++;

            return result;

        };

        // Start function
        const start = async function(a, b) {

            console.time('test');

            const responses = await Promise.all(
                params.map(param => onProgress(s3.upload(param).promise()))
            ).then(function(arrayOfValuesOrErrors) {

                console.log('Done');

            }).catch(function(err) {

                console.log('ERROR', err.message); // some coding error in handling happened

                setTimeout(function(){

                    console.log('Retrying again...');

                    upload(); // Retry upload

                }, 5000);

            });

            console.timeEnd('test');

        }

        // Call start
        start();

    });

}

upload();

[ajredniwja]

Hi @samueleastdev thanks for opening this issue, yes this is a known behavior of the SDK and you are retrying it is the right approach to work with it. To know more about how clock skew works check this article about clock skew correction for .NET SDK but should be applicable to JS SDK as well. With that being said, you can also use this code found at https://github.com/aws/aws-sdk-js/issues/399#issuecomment-233057244 which is somewhat a similar approach. You can also try to set correctClockSkew to true in your constructor if it defaults to false.

Note: The version 3.x of the AWS SDK for JavaScript is generally available. For more information see the Developer Guide or API Reference.

The V3 SDK always applies a clock skew correction which may solve your problem.

Closing this issue now, please reach out if you have any other questions.