ccxt / ccxt

A JavaScript / TypeScript / Python / C# / PHP cryptocurrency trading API with support for more than 100 bitcoin/altcoin exchanges
https://docs.ccxt.com
MIT License
33.38k stars 7.57k forks source link

load all market ohlcv #15445

Open Zourvan opened 2 years ago

Zourvan commented 2 years ago

I want load all OHLCV with multi connection, I did it in other framework but in CCXT I got error "throw new Error ('throttle queue is over maxCapacity (' + this.config['maxCapacity'].toString () + '), see https://github.com/ccxt/ccxt/issues/11645#issuecomment-1195695526');" how can I make multi WS connection??

"use strict";

const ccxt = require("ccxt");
let candles_list = new Array();
function handle(exchange, symbol, timeframe, candles) {
  const lastCandle = candles[candles.length - 1];
  const datetime = exchange.iso8601(lastCandle[0]);

  // console.log(lastCandle[0]);
  // console.log(parseInt(lastCandle[0]) / durationInMs);
  // console.log(currentMinute);
  if (lastCandle[6] === "true") {
    let table_data = {
      View: new Date(),
      Candel_time: datetime,
      exchange_id: exchange.id,
      timeframe: timeframe,
      symbol: symbol,
      open: lastCandle[1],
      high: lastCandle[2],
      low: lastCandle[3],
      close: lastCandle[4],
      volume: lastCandle[5],
      state: lastCandle[6],
    };
    if (candles_list.length >= 20) {
      candles_list.shift();
    }

    candles_list.push(table_data);
    console.clear();
    console.table(candles_list); ///new Date(), " for time :", datetime, exchange.id, timeframe, symbol, "\t", lastClosingPrice);
  }
}

async function watchSymbol(exchange, symbol, timeframe) {
  while (true) {
    try {
      const method = "watchOHLCV";
      const candles = await exchange[method](symbol, timeframe);
      handle(exchange, symbol, timeframe, candles);
    } catch (e) {
      console.log(symbol, e);
    }
  }
}

async function watchTimeframe(exchange, symbol, timeframes) {
  await Promise.all(timeframes.map((timeframe) => watchSymbol(exchange, symbol, timeframe)));
}

async function watchExchange(exchangeId, symbols, timeframes) {
  const exchange = new ccxt.pro[exchangeId]();
  await exchange.loadMarkets();
  await Promise.all(symbols.map((symbol) => watchTimeframe(exchange, symbol, timeframes)));
}

async function main() {
  console.log("CCXT Version:", ccxt.version);

  const exchange = "binance";
  const package_ = 10;
  const binance = new ccxt.pro[exchange]();
  await binance.loadMarkets();
  const symbols_list = binance.symbols.sort();
  const streams = {};
  console.log(symbols_list.length);
  console.log(parseInt(symbols_list.length / package_));
  console.log(symbols_list.length - parseInt(symbols_list.length / package_) * package_);
  for (let i = 0; i < package_; i++) {
    streams["symbols_list_" + i] = symbols_list.slice(
      parseInt(symbols_list.length / package_) * i,
      parseInt(symbols_list.length / package_) * (i + 1)
    );
  }
  streams["symbols_list_" + package_] = symbols_list.slice(parseInt(symbols_list.length / package_) * package_, symbols_list.length);

  const timeframes = ["1m", "3m", "5m", "15m", "30m", "1h"]; //, "2h", "4h", "6h", "8h", "12h", "1d", "3d", "1w", "1M"];
  // const streams = {
  //   symbol_List1: ["BTC/USDT", "ETH/BTC"],
  // };

  const entries = Object.entries(streams);
  await Promise.all(entries.map(([exchangeId, symbols]) => watchExchange(exchange, symbols, timeframes)));
}

main();
throw new Error ('throttle queue is over maxCapacity (' + this.config['maxCapacity'].toString () + '), see https://github.com/ccxt/ccxt/issues/11645#issuecomment-1195695526');
                  ^

Error: throttle queue is over maxCapacity (1000), see https://github.com/ccxt/ccxt/issues/11645#issuecomment-1195695526
    at Throttle.inner (D:\MD\Project\Pro\node_modules\ccxt\js\base\functions\throttle.js:56:19)
    at D:\MD\Project\Pro\node_modules\ccxt\js\pro\base\Exchange.js:126:32
    at process.processTicksAndRejections (node:internal/process/task_queues:95:5)
vladpuz commented 1 year ago

To set the maxCapacity value in JavaScript, I did the following:

const exchange = new ccxt.binance()
exchange.throttler = new exchange.throttler.constructor({ ...exchange.tokenBucket, maxCapacity: 10_000 })

In my use case, I parallelize ~1940 exchange.fetchOrderBook() calls via Promise.all(). I would like to be able to somehow set the maxCapacity through a constructor and not a property.