tonkeeper / opentonapi

Opentonapi simplifies development of TON-based applications and provides an API centered around high-level concepts like Jettons, NFTs and so on keeping a way to access low-level details.
MIT License
224 stars 68 forks source link

/v2/events/{event_id} not working for the large amount of the ACCOUNTS environment variable #356

Closed tccorp closed 5 months ago

tccorp commented 5 months ago

If i leave ACCOUNTS="comma-separated-list-of-raw-account-addresses" make run

list empty

then i get:

{ "error": "not found tx 97264395bd65a255a429b11326c84128b7d70ffed7949abae3036d506ba38621" }

{"level":"info","ts":1714550257.265961,"caller":"api/middlewares.go:23","msg":"Handling request","operation":"GetEvent","path":"/v2/events/97264395BD65A255A429B11326C84128B7D70FFED7949ABAE3036D506BA38621"}

{"level":"error","ts":1714550257.266055,"caller":"api/middlewares.go:27","msg":"Fail","operation":"GetEvent","path":"/v2/events/97264395BD65A255A429B11326C84128B7D70FFED7949ABAE3036D506BA38621","error":"code 500: {Error:not found tx 97264395bd65a255a429b11326c84128b7d70ffed7949abae3036d506ba38621}","stacktrace":"github.com/tonkeeper/opentonapi/pkg/api.NewServer.ogenLoggingMiddleware.func1\n\t/home/user0/opentonapi/pkg/api/middlewares.go:27\ngithub.com/ogen-go/ogen/middleware.ChainMiddlewares.func2\n\t/home/user0/go/pkg/mod/github.com/ogen-go/ogen@v1.0.0/middleware/middleware.go:92\ngithub.com/ogen-go/ogen/middleware.HookMiddleware[...]\n\t/home/user0/go/pkg/mod/github.com/ogen-go/ogen@v1.0.0/middleware/internal.go:29\ngithub.com/tonkeeper/opentonapi/pkg/oas.(Server).handleGetEventRequest\n\t/home/user0/opentonapi/pkg/oas/oas_handlers_gen.go:5964\ngithub.com/tonkeeper/opentonapi/pkg/oas.(Server).ServeHTTP\n\t/home/user0/opentonapi/pkg/oas/oas_router_gen.go:1379\nnet/http.(ServeMux).ServeHTTP\n\t/usr/local/go/src/net/http/server.go:2683\nnet/http.serverHandler.ServeHTTP\n\t/usr/local/go/src/net/http/server.go:3137\nnet/http.(conn).serve\n\t/usr/local/go/src/net/http/server.go:2039"}

but there is no way to supply file with accounts list, in case i need a bulk monitoring (100 000 accounts in my case)

tccorp commented 5 months ago

solved by directly passing large list (>100k) of ACCOUNTS from the file modifying pkg/config/config.go, it takes around 20 minutes after for the opentonapi to become responsive

tccorp commented 5 months ago

package config

import ( "bufio" "fmt" "log" "os" "reflect"

"github.com/caarlos0/env/v6"
"github.com/tonkeeper/tongo"
"github.com/tonkeeper/tongo/config"

)

type Config struct { API struct { Port int env:"PORT" envDefault:"8081" } App struct { LogLevel string env:"LOG_LEVEL" envDefault:"INFO" MetricsPort int env:"METRICS_PORT" envDefault:"9010" Accounts accountsList env:"ACCOUNTS" LiteServers []config.LiteServer env:"LITE_SERVERS" SendingLiteservers []config.LiteServer env:"SENDING_LITE_SERVERS" } TonConnect struct { Secret string env:"TON_CONNECT_SECRET" } }

type accountsList []tongo.AccountID

const ( AddressPath = "https://raw.githubusercontent.com/tonkeeper/ton-assets/main/accounts.json" CollectionPath = "https://raw.githubusercontent.com/tonkeeper/ton-assets/main/collections.json" JettonPath = "https://raw.githubusercontent.com/tonkeeper/ton-assets/main/jettons.json" )

func Load() Config { var c Config if err := env.ParseWithFuncs(&c, map[reflect.Type]env.ParserFunc{ reflect.TypeOf([]config.LiteServer{}): func(v string) (interface{}, error) { servers, err := config.ParseLiteServersEnvVar(v) if err != nil { return nil, err } if len(servers) == 0 { return nil, fmt.Errorf("empty liteservers list") } fmt.Print("SERVERS: ",servers) return servers, nil }, reflect.TypeOf(accountsList{}): parseAccountsFromFile, }); err != nil { log.Panicf("[‼️ Config parsing failed] %+v\n", err) }

return c

}

func parseAccountsFromFile(v string) (interface{}, error) { var accs accountsList file, err := os.Open("accounts.txt") if err != nil { return nil, err } defer file.Close()

counter := 0 // Initialize counter

scanner := bufio.NewScanner(file)
for scanner.Scan() {
    account, err := tongo.ParseAddress(scanner.Text())
    if err != nil {
        return nil, err
    }
    accs = append(accs, account.ID)
    //fmt.Print(".")
    counter ++
}

if err := scanner.Err(); err != nil {
    return nil, err
}

fmt.Printf("Loaded %d accounts\n", counter) //Print the total number of the accounts loaded

return accs, nil

}