Support complex type such as map(string, array(map(string, float64)))

[gangtao]

following query can be successfully running on proton but failed to run in neutron.

with t1 as (
SELECT 
  '1' as granularity, to_float64(_tp_time) as ts , product_id as symbol, cast((['ts', 'price'], [ts, price]), 'map(string, float64)') AS value
FROM 
  btc_usd_coinbase
),
t2 as (
  select symbol, granularity, group_array(value) AS value_array from t1 group by symbol, granularity order by symbol, granularity
),
t3 as (
  select symbol, group_array(granularity) as granularities, group_array(value_array) as values from t2 group by symbol
)
select symbol, cast((granularities, values), 'map(string, array(map(string, float64)))') from t3

neutron error is

proton: unsupported column type "map(string, array(map(string, float64)))".

I think it is because go-driver does not support this complex type as of now, which is required .

[ye11ow]

The error is from https://github.com/timeplus-io/proton-go-driver/blob/v2.0.15/lib/column/column.go#L67

[leo-cai-timeplus]

clickhouse support this

package main

import (
    "context"
    "fmt"
    "github.com/ClickHouse/clickhouse-go/v2"
    "log"
    "time"
)

func example() error {
    var (
        ctx       = context.Background()
        conn, err = clickhouse.Open(&clickhouse.Options{
            Addr: []string{"127.0.0.1:9000"},
            Auth: clickhouse.Auth{
                Database: "default",
                Username: "default",
                Password: "",
            },
            //Debug:           true,
            DialTimeout:     time.Second,
            MaxOpenConns:    10,
            MaxIdleConns:    5,
            ConnMaxLifetime: time.Hour,
        })
    )
    if err != nil {
        return err
    }
    if err := conn.Exec(ctx, "set allow_experimental_map_type = 1;"); err != nil {
        return err
    }
    if err := conn.Exec(ctx, `DROP TABLE IF EXISTS table_map;`); err != nil {
        return err
    }
    err = conn.Exec(ctx, `
        CREATE TABLE table_map (a Map(String, Array(Map(String, Float64)))) ENGINE = Memory;
    `)
    if err != nil {
        return err
    }

    batch, err := conn.PrepareBatch(ctx, "INSERT INTO table_map")
    if err != nil {
        return err
    }
    for i := 0; i < 100; i++ {
        err := batch.Append(
            map[string][]map[string]float64{
                "key1": {
                    {"sub_key1": 10.9, "sub_key2": float64(i)},
                    {"sub_key3": float64(i) + 0.1},
                },
            },
        )
        if err != nil {
            return err
        }
    }
    err = batch.Send()
    if err != nil {
        return err
    }
    rows, err := conn.Query(ctx, "SELECT * FROM table_map")
    for rows.Next() {
        a := &map[string][]map[string]float64{}
        err := rows.Scan(a)
        if err != nil {
            return err
        }
        fmt.Println(a)
    }
    if err != nil {
        return err
    }
    return err
}

func main() {
    start := time.Now()
    if err := example(); err != nil {
        log.Fatal(err)
    }
    fmt.Println(time.Since(start))
}

[ye11ow]

Yeah, we were too behind the offical clickhouse go driver. It has been years since we rebased. This can be an issue

[jovezhong]

(Jove Github Bot) added it to the current sprint.

[jovezhong]

(Jove Github Bot) moved this ticket out of the GitHub project(up to 1200 tickets for one project).