[Bug]: [GOSDK] The bf16 vectors output by the query is slightly different from the inserted vectors

[ThreadDao]

Is there an existing issue for this?

• [X] I have searched the existing issues

Environment

- Milvus version: master-20240613-512ea6be-amd64
- Deployment mode(standalone or cluster): cluster
- MQ type(rocksmq, pulsar or kafka):    
- SDK version(e.g. pymilvus v2.0.0rc2): go-sdk v2
- OS(Ubuntu or CentOS): 
- CPU/Memory: 
- GPU: 
- Others:

Current Behavior

The dim of bf16 field is 128. A bf16 vector type is slice []byte. The len of a inserted vector is 256 (128dim * 2, fp16 has 16bits 2 bytes). But the query output vector len is 128

• query output

  ([]uint8) (len=256) {
                            00000000  45 3f 66 3e 63 3f 9b 3d  6b 3f 95 3d 8d 3d ed 3e  |E?f>c?.=k?.=.=.>|
                            00000010  eb 3e bb 3d 50 3f 0b 3e  1e 3f 3d 3f c8 3e 42 3e  |.>.=P?.>.?=?.>B>|
                            00000020  0a 3e e2 3e 5d 3e 83 3d  20 3f 1a 3f 1c 3e 75 3f  |.>.>]>.= ?.?.>u?|
                            00000030  02 3f 8c 3d 26 3f 0d 3d  54 3e 6e 3d 0b 3f a3 3e  |.?.=&?.=T>n=.?.>|
                            00000040  f7 3e 60 3f 78 3e 6e 3f  47 3f 52 3f 66 3f 3b 3f  |.>`?x>n?G?R?f?;?|
                            00000050  5a 3f 70 3d ae 3e 30 3f  94 3d 3d 3f 52 3f 3e 3f  |Z?p=.>0?.==?R?>?|
                            00000060  68 3f 4e 3e 9a 3e 43 3d  39 3f c4 3e 06 3c 64 3e  |h?N>.>C=9?.>.<d>|
                            00000070  7b 3e 16 3e 59 3f 27 3f  a0 3d 3b 3f 66 3f 4e 3f  |{>.>Y?'?.=;?f?N?|
                            00000080  6a 3f 2a 3f 88 3e 17 3f  47 3d 1f 3f 68 3c 59 3e  |j?*?.>.?G=.?h<Y>|
                            00000090  18 3f 04 3f 44 3f 2d 3f  42 3f 24 3f 5a 3f d3 3d  |.?.?D?-?B?$?Z?.=|
                            000000a0  24 3e 4e 3f 26 3f 3a 3f  58 3f 48 3f 43 3f 10 3f  |$>N?&?:?X?H?C?.?|
                            000000b0  5f 3f 6d 3f 28 3f a9 3e  24 3f d4 3e 5b 3f 5d 3e  |_?m?(?.>$?.>[?]>|
                            000000c0  d1 3e 0e 3f 5c 3f 48 3f  52 3e 00 3f d9 3e 35 3e  |.>.?\?H?R>.?.>5>|
                            000000d0  5e 3f 35 3f ee 3e 86 3e  d1 3e 20 3f eb 3e a4 3e  |^?5?.>.>.> ?.>.>|
                            000000e0  b8 3e 88 3e 0e 3e 03 3f  38 3f 2d 3f ec 3e 38 3e  |.>.>.>.?8?-?.>8>|
                            000000f0  91 3e 98 3e 43 3f 3b 3f  87 3d 5a 3e e0 3e 19 3f  |.>.>C?;?.=Z>.>.?|
                           },

• insert vector:

  ([][]uint8) (len=10) {
                           ([]uint8) (len=128) {
                            00000000  45 3f 66 3e 63 3f 9b 3d  6b 3f 95 3d 8d 3d ed 3e  |E?f>c?.=k?.=.=.>|
                            00000010  eb 3e bb 3d 50 3f 0b 3e  1e 3f 3d 3f c8 3e 42 3e  |.>.=P?.>.?=?.>B>|
                            00000020  0a 3e e2 3e 5d 3e 83 3d  20 3f 1a 3f 1c 3e 75 3f  |.>.>]>.= ?.?.>u?|
                            00000030  02 3f 8c 3d 26 3f 0d 3d  54 3e 6e 3d 0b 3f a3 3e  |.?.=&?.=T>n=.?.>|
                            00000040  f7 3e 60 3f 78 3e 6e 3f  47 3f 52 3f 66 3f 3b 3f  |.>`?x>n?G?R?f?;?|
                            00000050  5a 3f 70 3d ae 3e 30 3f  94 3d 3d 3f 52 3f 3e 3f  |Z?p=.>0?.==?R?>?|
                            00000060  68 3f 4e 3e 9a 3e 43 3d  39 3f c4 3e 06 3c 64 3e  |h?N>.>C=9?.>.<d>|
                            00000070  7b 3e 16 3e 59 3f 27 3f  a0 3d 3b 3f 66 3f 4e 3f  |{>.>Y?'?.=;?f?N?|
                           },

• case:

  // test query output all fields and verify data
  func TestOutputAllFieldsColumn(t *testing.T) {
  ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
  mc := createDefaultMilvusClient(ctx, t)
  
  // create collection
  for _, isDynamic := range [2]bool{true, false} {
      prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.AllFields), hp.TNewFieldsOption(), hp.TNewSchemaOption().TWithEnableDynamicField(isDynamic))
      prepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema))
      prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName))
  
      // insert
      columns := make([]column.Column, 0, len(schema.Fields)+1)
      dynamicColumns := make([]column.Column, 0, len(schema.Fields)+1)
      genDataOpt := hp.TNewDataOption().TWithMaxCapacity(common.TestCapacity)
      for _, field := range schema.Fields {
          if field.IsDynamic {
              dynamicColumns = append(dynamicColumns, hp.GenDynamicColumnData(0, common.DefaultNb)...)
          } else {
              if field.DataType == entity.FieldTypeArray{
                  genDataOpt.TWithElementType(field.ElementType)
              }
              columns = append(columns, hp.GenColumnData(common.DefaultNb, field.DataType, *genDataOpt.TWithDim(common.DefaultDim)))
          }
      }
      ids, err := mc.Insert(ctx, clientv2.NewColumnBasedInsertOption(schema.CollectionName, columns...).WithColumns(dynamicColumns...))
      common.CheckErr(t, err, true)
      require.Equal(t, int64(common.DefaultNb), ids.InsertCount)
      prepare.FlushData(ctx, t, mc, schema.CollectionName)
  
      // query output all fields -> output all fields, includes vector and $meta field
      pos := 10
      allFieldsName := make([]string, 0, len(schema.Fields))
      for _, field := range schema.Fields {
          allFieldsName = append(allFieldsName, field.Name)
      }
      if isDynamic {
          allFieldsName = append(allFieldsName, common.DefaultDynamicFieldName)
      }
      queryResultAll, errQuery := mc.Query(ctx, clientv2.NewQueryOption(schema.CollectionName).WithConsistencyLevel(entity.ClStrong).
                                          WithFilter(fmt.Sprintf("%s < %d", common.DefaultInt64FieldName, pos)).WithOutputFields([]string{"*"}))
      common.CheckErr(t, errQuery, true)
      common.CheckOutputFields(t, allFieldsName, queryResultAll.Fields)
  
      expColumns := make([]column.Column, 0, len(columns)+1)
      for _, _column := range columns {
          expColumns = append(expColumns, _column.Slice(0, pos))
          if _column.Type() == entity.FieldTypeBFloat16Vector {
              //log.Debug("_column", zap.ByteStrings("data", _column.(*column.ColumnBFloat16Vector).Data()))
              log.Debug("_column", zap.Int("len", len(_column.(*column.ColumnBFloat16Vector).Data()[0])))
          }
      }
      if isDynamic {
          expColumns = append(expColumns, hp.MergeColumnsToDynamic(pos, dynamicColumns))
      }
      common.CheckQueryResult(t, expColumns, queryResultAll.Fields)
  }
  }

Expected Behavior

No response

Steps To Reproduce

No response

Milvus Log

No response

Anything else?

No response

[ThreadDao]

/assign @congqixia

[yanliang567]

/unassign

[congqixia]

/assign @ThreadDao shall be fixed in #33684, could you please verify?

[ThreadDao]

@congqixia not fixed yet

[congqixia]

@congqixia not fixed yet

there is still some problem in bf16, tested with fp16 missing bf16 path

[adetunjii]

Hi @congqixia has this issue been resolved? It looks like you've fixed it, yet the issue remains open

[congqixia]

@adetunjii i think this issue is under verification by @ThreadDao

[ThreadDao]

@adetunjii @congqixia fixed.