With a dozen docs IDs in project.json, it's possible to hit the rate limit

[benlk]

$ grunt docs WARN: google-spreadsheets detected googleapis@35.0.0 is installed. This version is unrecognised by this version of google-spreadsheets and may not work correctly. Running "docs" task Warning: User Rate Limit Exceeded. Rate of requests for user exceed configured project quota. You may consider re-evaluating expected per-user traffic to the API and adjust project quota limits accordingly. You may monitor aggregate quota usage and adjust limits in the API Console: https://console.developers.google.com/apis/api/drive.googleapis.com/quotas?project=[redacted] Use --force to continue.

This happened with the seemingly-default limits:

Quota Name	Limit
Queries per day	1,000,000,000
Queries per 100 seconds per user	1,000
Queries per 100 seconds	10,000

But the queries per user per second in the graph never topped 26.666 as seen in the chart in the API console:

Is there a way to debounce or otherwise stall the queries made in the async function in the docs task?

https://github.com/nprapps/interactive-template/blob/369e1c1e6474bc7fd84fa78d8d2f81420cbe15e8/root/tasks/loadDocs.js#L24-L31

[benlk]

Yes, there is a way:

var { google } = require("googleapis");
var async = require("async");
var os = require("os");
var path = require("path");
var { authenticate } = require("./googleauth");

module.exports = function(grunt) {

  grunt.registerTask("docs", "Load Google Docs into the data folder", function() {

    var config = grunt.file.readJSON("project.json");
    var auth = null;
    try {
      auth = authenticate();
    } catch (err) {
      console.log(err);
      return grunt.fail.warn("Couldn't load access token for Docs, try running `grunt google-auth`");
    }

    var done = this.async();

    var drive = google.drive({ auth, version: "v3" });

    /*
     * limit 2 concurrent operations, https://caolan.github.io/async/docs.html#eachLimit
     * thanks to https://stackoverflow.com/a/34865245
     *
     * Because of https://github.com/nprapps/interactive-template/issues/12
     */
    async.eachLimit(
      config.docs,
      2, 
      async function(fileId) {
        var meta = await drive.files.get({ fileId });
        var name = meta.data.name.replace(/\s+/g, "_") + ".docs.txt";
        var body = await drive.files.export({ fileId, mimeType: "text/plain" });
        var text = body.data.replace(/\r\n/g, "\n");
        console.log(`Writing document as data/${name}`);
        grunt.file.write(path.join("data", name), text);
      },
      done
    );

  });
}

[thomaswilburn]

Should this be a PR? Seems like a reasonable default.