Proxy. Error: tunneling socket could not be established, statusCode=502

[vladgovor77771]

Hey, I am not really sure, is it a bug report or question about the module or coding in general because I am trying to a bit rework SteamCommunity.prototype.marketSearch implementation. The idea is to switch proxy between each requests for increasing speed. I experimentally discovered that if request to Steam every 3 second I won't get 429 error. So with 10 proxy I can request to market every 0.3~ sec.

I have proxy pool, which I can use like this:

let proxies = [];

config.proxies.map(_proxy => {
  proxies.push({ lastUse: Date.now() - 3000, ..._proxy  });
})

const reloadProxy = () => {
  let res = proxies.shift();
  console.log(`http://${res.username}:${res.password}@${res.ip}:${res.port}`);
  if (Date.now() - res.lastUse < 3000) {
    proxies.unshift(res);   
    return reloadProxy();
  } else {
    proxies.push(res);
    return `http://${res.username}:${res.password}@${res.ip}:${res.port}`;
  }
}

So, I wanted to create externed instance of SteamCommunity for scanning market.

const sleep = require('sleep-promise');
const Cheerio = require('cheerio');
const config = require('../../config');
const Request = require('request');
const reloadProxy = require('./reloadProxy');

const SteamCommunity = require('steamcommunity');
let community = new SteamCommunity();

community.marketSearch = function(options, stream) {
  logger.info('Started parsing...')
  var qs = {};

  if(typeof options === 'string') {
    qs.query = options;
  } else {
    qs.query = options.query || '';
    qs.appid = options.appid;
    qs.search_descriptions = options.searchDescriptions ? 1 : 0;

    if(qs.appid) {
      for(var i in options) {
        if(['query', 'appid', 'searchDescriptions'].indexOf(i) != -1) {
          continue;
        }

        // This is a tag
        qs['category_' + qs.appid + '_' + i + '[]'] = 'tag_' + options[i];
      }
    }
  }

  qs.start = 0;
  qs.count = 100;
  qs.sort_column = 'price';
  qs.sort_dir = 'asc';

  var self = this;
  performSearch();

  function performSearch() {
    let USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36";
    let defaults = {  
      proxy: reloadProxy(),
      jar: this._jar,
      timeout: options.timeout || 50000,
      gzip: true,
      headers: {
        'User-Agent': USER_AGENT
      }
    };

    self.request = Request.defaults(defaults);

    self.httpRequest({
      // uri: 'https://iplogger.org/1oxzx', // here i can change uri to another, and all starts work
      uri: "https://steamcommunity.com/market/search/render/", // but when we requesting to steam, 502 error
      qs: qs,
      headers: {
        "referer": "https://steamcommunity.com/market/search"
      },
      json: true
    }, async function(err, response, body) {
      if (err) {
        if (err.code == 429) {
          logger.error('429 error! Sleeping 5 sec')
          await sleep(5000);
          performSearch();
          return;
        }
        stream.emit('error', err);
        return;
      }

      if(!body.success) {
        // return performSearch() // this is for testing proxy pool working
        stream.emit('error', new Error("Success is not true"));
        return;
      }

      if(!body.results_html) {
        stream.emit('error', new Error("No results_html in response"));
        return;
      }

      var $ = Cheerio.load(body.results_html);
      var $errorMsg = $('.market_listing_table_message');
      if($errorMsg.length > 0) {
        stream.emit('error', new Error($errorMsg.text()));
        return;
      }

      var rows = $('.market_listing_row_link');
      for(var i = 0; i < rows.length; i++) {
        stream.emit('newItem', (new CMarketSearchResult($(rows[i]))).market_hash_name);
      }

      if(body.start + body.pagesize >= body.total_count) {
        stream.emit('end', null);
        return;
      } else {
        console.log(qs.start);
        qs.start += body.pagesize;
        await sleep(3000/proxies.length);
        performSearch(); 
      }
    }, "steamcommunity");
  }
};

function CMarketSearchResult(row) {
  var match = row.attr('href').match(/\/market\/listings\/(\d+)\/([^\?\/]+)/);

  this.appid = parseInt(match[1], 10);
  this.market_hash_name = decodeURIComponent(match[2]);
  this.image = ((row.find('.market_listing_item_img').attr('src') || "").match(/^https?:\/\/[^\/]+\/economy\/image\/[^\/]+\//) || [])[0];
  this.price = parseInt(row.find('.market_listing_their_price .market_table_value span.normal_price').text().replace(/[^\d]+/g, ''), 10);
  this.quantity = parseInt(row.find('.market_listing_num_listings_qty').text().replace(/[^\d]+/g, ''), 10);
}

module.exports = community;

This is like the same as your implementation, but here I reload proxy each request and use stream to push elements back. So I can use this ljust like this:

const createAllItemStream = async () => {
  const itemsStream = new Readable();
  itemsStream._read = function () {};
  community.marketSearch({
    // query: 'desert', 
    appid: 730
  }, itemsStream);

  return itemsStream;
}

module.exports = async ({ mod }) => {
  if (mod == 'stream')
    return createAllItemStream();
  if (mod == 'silent' || !mod) {
    let count = 0;
    const itemsStream = await createAllItemStream();
    itemsStream.on('error', (err) => {
      throw err;
    });
    itemsStream.on('newItem', async (itemName) => {
      let item = await Item.findOrCreate({ where: { marketHashName: itemName }, defaults: { status: 'new' } });
      if (item[1]) count++;
    })
    itemsStream.on('end', () => {
      logger.info(`${count} new items.`);
      return count;
    });
  }
}

And when I requesting to Steam I get this:

(node:14916) UnhandledPromiseRejectionWarning: Error: tunneling socket could not be established, statusCode=502
    at ClientRequest.onConnect (D:\Desktop\work\projects\steam-market-bot\node_modules\tunnel-agent\index.js:166:19)
    at Object.onceWrapper (events.js:273:13)
    at ClientRequest.emit (events.js:182:13)
    at Socket.socketOnData (_http_client.js:482:11)
    at Socket.emit (events.js:182:13)
    at addChunk (_stream_readable.js:283:12)
    at readableAddChunk (_stream_readable.js:264:11)
    at Socket.Readable.push (_stream_readable.js:219:10)
    at TCP.onStreamRead [as onread] (internal/stream_base_commons.js:94:17)
(node:14916) UnhandledPromiseRejectionWarning: Unhandled promise rejection. This error originated either by throwing inside of an async function withoutout a catch block, or by rejecting a promise which was not handled with .catch(). (rejection id: 1)
(node:14916) [DEP0018] DeprecationWarning: Unhandled promise rejections are deprecated. In the future, promise rejections that are not handled will termerminate the Node.js process with a non-zero exit code.

Do you know what does it mean, or how to fix it?

[vladgovor77771]

It seem, that the problem might be with proxy. Can Steam ban my proxy or something?

[DoctorMcKay]

Steam could have banned your proxy, but I kinda doubt it given that error message. I'm pretty sure that message indicates that the proxy couldn't connect to Steam. Either way, it's on your end.

That said, I'd suggest not modifying the module and instead creating a separate SteamCommunity instance for each proxy.

[vladgovor77771]

Actually yes, it turned out that my proxy provider blocked requests to Steam themself, so this strange error. I changed my proxies and all work. Anyway thank you for answer here.