Former2 doesn't do pagination correctly

[jlongman]

When looking for a large number of CloudFront or Lambda resources (>100), the list stops at the first 100 and does not show later resources.

In js/datatables.js in sdkcall there is code to handle pagination but the data returned from the AWS SDK was not in a matching scheme. E.g. it was a layer down like data = {'DistributionList' = [ 'NextMarker'=xxx, 'Lists'= #etc and not a peer to DistributionList like data = {'NextMarker'=xxx, 'DistributionList': # etc

I patched this locally in the browser by changing the function sdkcall as follows:

function sdkcall(svc, method, params, alert_on_errors, backoff) {
    return new Promise(function(resolve, reject) {
        var service = new AWS[svc]({region: region});
        if (svc == "GlobalAccelerator") {
            service = new AWS[svc]({region: 'us-west-2'});
        }

        service[method].call(service, params, async function(err, data) {
            if (err) {
                if (err.code == "TooManyRequestsException" || err.message == "Too Many Requests" || err.code == "ThrottlingException" || err.message == "Rate exceeded" || err.code == "TimeoutError") {
                    if (backoff) {
                        console.log("Too many requests, sleeping for " + backoff + "ms");
                        await new Promise(resolve => setTimeout(resolve, backoff));
                        backoff *= 2;
                    } else {
                        console.log("Too many requests, sleeping for 500ms");
                        await new Promise(resolve => setTimeout(resolve, 500));
                        backoff = 500 + Math.floor(Math.random() * 500);
                    }
                    sdkcall(svc, method, params, alert_on_errors, backoff).then(newdata => {
                        resolve(newdata);
                    }, data => {
                        reject(data);
                    });
                } else {
                    if (err.code == "NetworkingError") {
                        console.log("Skipping " + svc + "." + method + " NetworkingError");
                    } else if (err.code == "AccessDeniedException") {
                        console.log("Skipping " + svc + "." + method + " AccessDeniedException");
                    } else if (err.code == "UnknownError" && svc == "MediaStore") {
                        console.log("Skipping " + svc + "." + method + " UnknownError");
                    } else if (err.code == "ForbiddenException" && svc == "RoboMaker") {
                        console.log("Skipping " + svc + "." + method + " ForbiddenException");
                    } else if (err.code == "AccessDeniedException" && svc == "FSx") {
                        console.log("Skipping " + svc + "." + method + " AccessDeniedException");
                    } else if (alert_on_errors) {
                        console.log("Error calling " + svc + "." + method + ". " + (err.message || JSON.stringify(err)));
                        console.trace(err);
                        $.notify({
                            icon: 'font-icon font-icon-warning',
                            title: '<strong>Error calling ' + svc + '.' + method + '</strong>',
                            message: err.message || JSON.stringify(err)
                        },{
                            type: 'danger'
                        });
                    }

                    reject(data);
                }
            } else {
                 object_keys_zero = Object.keys(data)[0];
                 console.log("xxx: " + object_keys_zero);
                if (data.Marker) {
                    params['Marker'] = data.Marker;
                    sdkcall(svc, method, params, alert_on_errors).then(newdata => {
                        var mergeddata = deepmerge.all([data, newdata]);

                        resolve(mergeddata);
                    }, data => {
                        reject(data);
                    });
                } else if (data.NextPageToken) {
                    params['PageToken'] = data.NextPageToken;
                    sdkcall(svc, method, params, alert_on_errors).then(newdata => {
                        var mergeddata = deepmerge.all([data, newdata]);

                        resolve(mergeddata);
                    }, data => {
                        reject(data);
                    });
                } else if (data.ContinuationToken) {
                    params['ContinuationToken'] = data.ContinuationToken;
                    sdkcall(svc, method, params, alert_on_errors).then(newdata => {
                        var mergeddata = deepmerge.all([data, newdata]);

                        resolve(mergeddata);
                    }, data => {
                        reject(data);
                    });
                } else if (data.NextToken) {
                    params['NextToken'] = data.NextToken;
                    sdkcall(svc, method, params, alert_on_errors).then(newdata => {
                        var mergeddata = deepmerge.all([data, newdata]);

                        resolve(mergeddata);
                    }, data => {
                        reject(data);
                    });
                } else if (data.nextToken) {
                    params['nextToken'] = data.nextToken;
                    sdkcall(svc, method, params, alert_on_errors).then(newdata => {
                        var mergeddata = deepmerge.all([data, newdata]);

                        resolve(mergeddata);
                    }, data => {
                        reject(data);
                    });
                } else if (data[object_keys_zero].NextMarker) {
                    params['Marker'] = data[object_keys_zero].NextMarker;
                    sdkcall(svc, method, params, alert_on_errors).then(newdata => {
                        var mergeddata = deepmerge.all([data, newdata]);

                        resolve(mergeddata);
                    }, data => {
                        reject(data);
                    });

                } else {

                    resolve(data);
                }
            }
        });
    });
}

[jlongman]

(Note, Lambdas may have been something else, but this definitely affected CloudFront).

[iann0036]

Thanks for raising. Looks like the SDK pagination is far more...diverse than I originally catered for.

Doing some work at https://github.com/iann0036/aws-pagination-rules to confirm the correct ruleset, then will update here.

[iann0036]

Updated in codebase and former2.com , cheers!