Question: how to function-rate-limit in a module?

[marcolino]

Just a bit weird question: I'm building a module ("network-request"), which places http requests. I'd like to integrate "function-rate-limit" functionality in it, but I can't understand how to do it... I did try:

exports.myRequestLimited = rateLimit(1, 3 * 1000, function(url, provider, error, success) {
  exports.myRequest(url, provider, error, success);
});

exports.myRequest = function(url, provider, error, success) {
  // my logic here...
});

Which works, but I don't like it, since, for example, I can't pass it the limits (1 request every 3 seconds, in this example...).

I'd prefer something like this:

exports.MyRequestLimited = function(url, provider, error, success) {
  return rateLimit(1, 3 * 1000, function(url, provider, error, success) {
    exports.myRequest(url, provider, error, success);
  });
});

But this doesn't work (it freezes...). Any clues?

[wankdanker]

In your preferred case, you'd need to call it like:

var limited = require('module').MyRequestLimited;
var provider = whatever;

limited('http://www.google.com/', provider, fnerr, fnsuccess)('http://www.google.com/', provider, fnerr, fnsuccess);

But the arguments passed to limited (MyRequestLimited) are actually not used for anything because all that function does is return a rate limited anonymous function.

I'd probably do something like:

var rateLimit = require('function-rate-limit');

exports.MyRequestLimited = function (limitCount, limitInterval) {
    return rateLimit(limitCount, limitInterval,  exports.myRequest);
};

exports.myRequest = function(url, provider, error, success) {
  // my logic here...
});

So then you could call it like:

var limited = require('module').MyRequestLimited;
var provider = whatever;

var limitedReq = limited(1, 3*1000);

limitedReq('http://www.google.com/', provider, fnerr, fnsuccess);

Not sure if that's what you're aiming for.

[marcolino]

Thanks for the quick answer!

Not really what I was looking forward: in provider object I have the specific limits for that provider (I'm scraping data from public sites ("providers") without an API...). So I'd like to someway call a limitedReq (or something like that) method, passing the limit options as one of the parameters (for example, as a field of provider object)...

[wankdanker]

Hmm. You could set the limiting values based on the provider like so:

exports.MyRequestLimited = function(provider) {
 return rateLimit(provider.limitCount, provider.limitInterval,  exports.myRequest);
});

In order for the rate limiting to work, you'd probably need to create rate limited functions for each provider...

function ProviderRequest(provider) {
 return rateLimit(provider.limitCount, provider.limitInterval,  exports.myRequest);
}

exports.myRequestLimited = function (url, provider, error, success) {
    if (!provider.limitedReq) {
        provider.limitedReq = ProviderRequest(provider);
    }

    provider.limitedReq(url, provider, error, success);
}

exports.myRequest = function(url, provider, error, success) {
  // my logic here...
});

Not sure though. I don't have a full understanding of what you've got going on.

[marcolino]

Yes... I don't have provider.limitedReq in provider object, so i did eliminate if (!provider.limitedReq) from your suggested, simlpifying this way:

// ...
provider.limitCount = 1;
provider.limitInterval = 3 * 1000;
url = 'www.example.com';

exports.myRequestLimited(
  url,
  provider,
  function(err) { console.error(err); },
  function(result) { console.ingo(result); }
);
// ...

exports.myRequestLimited = function (url, provider, error, success) {
  rateLimit(provider.limitCount, provider.limitInterval, exports.myRequest)
                (url, provider, error, success);
}

exports.myRequest = function(url, provider, error, success) {
  // my logic here...
});

myRequest is called correctly, but it is not rate limited... :-(

P.S.: you were very kind, until now... Please don't feel obliged to answer me, I don't want to waste more of your time... :-(

[wankdanker]

Yeah, you have to keep the instance of the rate limited function around and call it repeatedly. Otherwise you're creating a new set of rate limiting "accounting books" each time you initialize a new one. If you can't append the limited function to the provider object, then hopefully each provider has a name or something and you can have a local hash containing the rate limited functions...

var providerLimitedReq = {};

function ProviderRequest(provider) {
 return rateLimit(provider.limitCount, provider.limitInterval,  exports.myRequest);
}
exports.myRequestLimited = function (url, provider, error, success) {
    var limitedReq = providerLimitedReq[provider.name] = providerLimitedReq[provider.name] || ProviderRequest(provider);

   limitedReq(url, provider, error, success);
}

Hope this helps!

[marcolino]

Finally I think I understand your points... Unfortunately I made probably some mistake porting it to my code base, so it doesn't work yet (requests are still unlimited... :-1: :-) I'll try again tomorrow, after some sleep... :-)