matthewmueller / x-ray

The next web scraper. See through the <html> noise.
MIT License
5.88k stars 350 forks source link

promisified x-ray functions, return null response #174

Closed madibalive closed 5 years ago

madibalive commented 8 years ago

WRAP XRAY IN PROMISE

i promise two x-ray function , and running them via promise.all statement , i only get one return data and undefined on the second call, ideally the plan is execute about 5 of these function. ps using version 2.0.2 the crawling breaks using the lasted build as 20 april

Your environment

version x-ray :2.0.2

var x = require("x-ray")(); // 2.0.2
var sentiment = require('sentiment');
var _ = require('lodash');
var Promise = require('bluebird');

 function topJoy() {
        return new Promise(function (resolve, reject) {
            x('http://www.myjoyonline.com', '.latest_headlines ul li', [{
                title: 'a',
                url: 'a@href',
                image: x('a@href', '.main-article-section .side1 img@src'),
                full: x('a@href', ['.main-article-section .storypane p'])
            }])(function (err, obj) {

                if (err) {
                    reject(err)
                } else {
                    _(obj).forEach(function (story) {
                        var a = story.fullStory;
                        story.category = 'Top story';
                        story.mood = sentiment(a.join()).score;
                    });
                    resolve(obj);
                }
            })
        });
    }
    function joyEnt() {
        return new Promise(function (resolve, reject) {
            x('http://www.myjoyonline.com/entertainment.php', 'ul.opinion-listings li', [{
                title: '.head .title a',
                desc: '.info',
                date: '.head',
                img: 'div.image-inner > a img@src',
                url: '.head .title a@href',
                fullStory: x('.head .title a@href', ['.main-article-section .storypane p'])
            }])(function (err, obj) {
                if (err) {
                    reject(err)
                } else {
                    _(obj).forEach(function (story) {
                        var a = story.fullStory;
                        story.category = 'entertainment';
                        story.mood = sentiment(a.join()).score;
                    });
                    resolve(obj);
                }
            })
        })
    }
function cityEnt() {
        return new Promise(function (resolve, reject) {
            x('http://showbiz.citifmonline.com', '.widget-container ul li', [{
                title: 'a',
                url: 'a@href',
                imgfull: x('a@href', 'article .single-post-thumb img@src'),
                fullStory: x('a@href', ['article .post-inner p'])
            }])(function (err, obj) {
                if (err) {
                    reject(err);
                }
                _(obj).forEach(function (story) {
                    var a = story.fullStory;
                    story.category = 'entertainment';
                    story.mood = sentiment(a.join()).score;
                });
                resolve(obj)

            })
        })
    }
    function execute() {
        return Promise.all([joyEnt(), joyPolitics(),cityEnt()]);

    }

and call

execute().then(function(data) {
                console(data)
                res.json([data]);
            }).catch(function(err) {
                res.send(err);
                console.log('failed');
            })

### Expected behaviour

console.log(data.lenght > 2) //should be true show that it return all three success

### Actual behaviour
console.log(data.lenght > 2)   - false//should be true show that it return all three if success
console.log(data.length) 1 // return only one and null for the rest
Tell us what happens instead.
lathropd commented 5 years ago

I think this has subsequently been addressed.