chill117 / proxy-verifier

NodeJS module to check proxies: if functional, anonymity level, tunneling, supported protocols.
MIT License
76 stars 16 forks source link

Cannot verify proxies #11

Closed rafis closed 8 years ago

rafis commented 8 years ago

proxies-fetch.js (seems works ok)

var fs = require('fs');

var ProxyLists = require('proxy-lists');

var filename = 'proxies-unchecked.txt';

// Read current list of proxies
var proxiesAntidup = {};
if (fs.existsSync(filename)) {

    fs
        .readFileSync(filename, 'utf8')
        .split("\n")
        .forEach(function(proxy) {
            proxiesAntidup[proxy] = true;
        });
}

function parseProxies(protocol) {

    return new Promise(function(resolve, reject) {

        var options = {
            anonymityLevels: 'https' == protocol ? ['elite'] : null,
            protocols: [protocol],
            sourcesBlackList: ['bitproxies', 'kingproxies']
        };

        // `gettingProxies` is an event emitter object.
        var gettingProxies = ProxyLists.getProxies(options);

        gettingProxies.on('data', function(proxies) {

            // Received some proxies.
            proxies.forEach(function(proxy) {

                // Add proxy to the list
                proxy = proxy.ipAddress + "\t" + proxy.port + "\t" + protocol;
                if ( ! proxiesAntidup[proxy]) {

                    fs.appendFileSync(filename, proxy + "\n");
                    proxiesAntidup[proxy] = true;

                }

            });

        });

        gettingProxies.on('error', function(error) {
            // Some error has occurred.
            console.error(error);
        });

        gettingProxies.once('end', function() {
            // Done getting proxies.
            resolve();
        });

    });

}

Promise.all(['https', 'socks5'].map(function(protocol) {

    return parseProxies(protocol);

})).then(function() {

    process.exit(0);

});

proxies-check.js (has problems)

var fs = require('fs');

var ProxyVerifier = require('proxy-verifier');

var sourceFilename = 'proxies-unchecked.txt';
var outputFilename = 'proxies.txt';

if ( ! fs.existsSync(sourceFilename)) {

    process.exit(0);

}

// Read current list of proxies
var promises = fs
    .readFileSync(sourceFilename, 'utf8')
    .split("\n")
    .map(function(proxy) {

        proxy = proxy.split("\t");
        if ( ! proxy[0] || ! proxy[1] || ! proxy[2] ) {
            return;
        }

        var proxy = {
            ipAddress: proxy[0],
            port: proxy[1],
            protocol: proxy[2],
            protocols: ['https', 'socks5']
        };

        return new Promise(function(resolve, reject) {
            ProxyVerifier.testAll(proxy, function(error, result) {

                console.log(result);

                if (error) {
                    reject(error);
                    return;
                }

                if ('elite' == result.anonymityLevel && (result.protocols.https && result.protocols.https.ok || result.protocols.socks5 && result.protocols.socks5.ok)) {

                    proxy.protocol = result.protocols.socks5 && result.protocols.socks5.ok ? 'socks5' : 'https';
                    proxy.country = result.country;
                    resolve(proxy);
                    return;

                }

                reject();
            });
        });
    });

fs.unlinkSync(sourceFilename);

Promise
    .all(promises)
    .then(function(proxies) {

        proxies.forEach(function(proxy) {

            if (!proxy) {
                return;
            }

            fs.appendFileSync(outputFilename, JSON.stringify(proxy) + "\n");

        });

    }).then(function() {

        process.exit(0);

    }).catch(function(error) {

        if (error) {
            console.error(error);
        }

    });

I don't get any proxy. In console.log(result) all proxies are failed and process also hangs. Forgot how to use node-inspector or IDE debugger :(

chill117 commented 8 years ago

There's a lot of code in your snippet. I'm not sure if the issue lies with this module or with the rest of your code. Can you isolate the part where you are using this module?