parse multiple urls?

[AbdelhamidAhmed]

//

[mooyoul]

Yup, You can.

Wrap feedparser into Promise, and mix it Promise.map.

I'm using bluebird to sync 500+ RSS feeds with concurrency limiting.

[mooyoul]

@AbdelHamidAhmed Sure. this is an example of reading multiple rss feeds.

'use strict';

/**
 * Module dependencies.
 */

const
  Promise     = require('bluebird'),
  request     = require('request'),
  FeedParser  = require('feedparser');

const fetch = (url) => {
  return new Promise((resolve, reject) => {
    if (!url) { return reject(new Error(`Bad URL (url: ${url}`)); }

    const
      feedparser = new FeedParser(),
      items     = [];

    feedparser.on('error', (e) => {
      return reject(e);
    }).on('readable', () => {
      // This is where the action is!
      var item;

      while (item = feedparser.read()) {
        items.push(item)
      }
    }).on('end', () => {
      resolve({
        meta: feedparser.meta,
        records: items
      });
    });

    request({
      method: 'GET',
      url: url
    }, (e, res, body) => {
      if (e) {
        return reject(e);
      }

      if (res.statusCode != 200) {
        return reject(new Error(`Bad status code (status: ${res.statusCode}, url: ${url})`));
      }

      feedparser.end(body);
    });
  });
};

Promise.map([
  'http://example.com/rss1',
  'http://example.com/rss2',
  ''http://example.com/rss3',
  ....
  ''http://example.com/rssN'
], (url) => fetch(url), {concurrency: 4}) // note that concurrency limit
.then((feeds) => {
 // do something with your feeds...
});

There are many techniques to handle tasks with concurrency support.

If you interested, See these links: https://github.com/visionmedia/batch https://github.com/Automattic/kue