search

gonum / hdf5

hdf5 is a wrapper for the HDF5 library
BSD 3-Clause "New" or "Revised" License
132 stars 33 forks source link

Example of writing a dataset #80

Open dcu opened 3 years ago

dcu commented 3 years ago

What are you trying to do?

I'm trying to write an array in a dataset, what I do is reading it, applying some operations to the data and then I want to write it back. At the moment of writing it back I've tried:

        dataToWrite := ts.Data().([]float32)
        err = ds.WriteSubset(dataToWrite, nil, nil)
        dataToWrite := ts.Data().([]float32)
        err = ds.WriteSubset(&dataToWrite, nil, nil)
        dataToWrite := ts.Data().([]float32)
        err = ds.WriteSubset(dataToWrite, dspace, nil)

etc...

but all of them fail with either:

panic: reflect.Value.UnsafeAddr of unaddressable value

or

panic: code -1

so I'm not sure what I'm missing here

thanks in advance

dcu commented 3 years ago

I just tried:

        towrite := reflect.New(reflect.SliceOf(dt.GoType())).Elem()
        towrite.Set(reflect.MakeSlice(towrite.Type(), n, n))

        dataToWrite := ts.Data().([]float32)
        for i, v := range dataToWrite {
            towrite.Index(i).Set(reflect.ValueOf(v))
        }

no luck either

kortschak commented 3 years ago

Sorry for the delay in responding. Can you provide a minimal complete reproducer?

dcu commented 3 years ago

I'm using the same example @sbinet posted in https://github.com/gonum/hdf5/issues/76#issuecomment-787090588 + the part to write the dataset after reading it

kortschak commented 3 years ago

Please paste the code you are using here.

dcu commented 3 years ago 
package main

import (
    "log"
    "os"

    "gonum.org/v1/hdf5"
)

func main() {
    if len(os.Args) < 2 {
        log.Printf("Usage: %v <h5-file>", os.Args[0])
        os.Exit(1)
    }

    f, err := hdf5.OpenFile(os.Args[1], 0)
    if err != nil {
        panic(err)
    }
    defer f.Close()

    objects, err := f.NumObjects()
    if err != nil {
        panic(err)
    }

    for i := uint(0); i < objects; i++ {
        groupName, err := f.ObjectNameByIndex(i)
        if err != nil {
            panic(err)
        }

        log.Printf("group: %v", groupName)

        processGroup(f, groupName)
    }
}

func processGroup(f *hdf5.File, name string) {
    g, err := f.OpenGroup(name)
    if err != nil {
        panic(err)
    }
    defer g.Close()

    objects, err := g.NumObjects()
    if err != nil {
        panic(err)
    }

    for i := uint(0); i < objects; i++ {
        dsName, err := g.ObjectNameByIndex(i)
        if err != nil {
            panic(err)
        }

        log.Printf("dataset: %v", dsName)

        processDataSet(g, dsName)
    }
}

func processDataSet(g *hdf5.Group, name string) {
    ds, err := g.OpenDataset(name)
    if err != nil {
        panic(err)
    }
    defer ds.Close()

    dt, err := ds.Datatype()
    if err != nil {
        log.Panicf("could not retrieve data-type from data-set %q: %+v",
            name, err,
        )
    }
    defer dt.Close()

    dspace := ds.Space()

    n := dspace.SimpleExtentNPoints()
    dims, max, err := dspace.SimpleExtentDims()
    if err != nil {
        log.Panicf("could not retrieve data-space shape %q: %+v", name, err)
    }
    log.Printf("dtype(%q): %v (%d, dims=%v, max=%v)", name, dt.GoType(), n, dims, max)

    //d := reflect.New(reflect.SliceOf(dt.GoType())).Elem()
    //d.Set(reflect.MakeSlice(d.Type(), n, n))

    d := make([]float32, n) // we know all data is of type float32. we could make this more general with reflect like above.

    err = ds.Read(&d)
    if err != nil {
        log.Printf("error reading dataset: %v", err)
        return
    }

    log.Printf("read: %v", d[len(d)-10:]) // print out the last 10 elements

    d[0] = 1.0

    err = ds.WriteSubset(d, nil, nil)
    if err != nil {
        log.Printf("error writing dataset: %v", err)
        return
    }
}

file to test: https://github.com/fchollet/deep-learning-models/releases/download/v0.1/vgg16_weights_th_dim_ordering_th_kernels_notop.h5

kortschak commented 3 years ago

The issue is that d is passed in as a non-addressable type. The full panic for reference is

panic: reflect.Value.UnsafeAddr of unaddressable value

goroutine 1 [running]:
reflect.Value.UnsafeAddr(...)
    /home/user/go/src/reflect/value.go:1972
gonum.org/v1/hdf5.(*Dataset).WriteSubset(0xc000098380, 0x4c64e0, 0xc0000a80a8, 0x0, 0x0, 0x0, 0x0)
    /home/user/src/gonum.org/v1/hdf5/h5d_dataset.go:126 +0x3eb
main.processDataSet(0xc0000ba780, 0xc0000ba790, 0x10)
    /home/user/hdf5bug/main.go:102 +0x5d4
main.processGroup(0xc0000ba700, 0xc0000ba720, 0xc)
    /home/user/hdf5bug/main.go:59 +0x172
main.main()
    /home/user/hdf5bug/main.go:35 +0x212
exit status 2

If instead &d is passed in we get no panic, but hdf5 errors.

2021/04/08 12:22:36 group: block1_conv1
2021/04/08 12:22:36 dataset: block1_conv1_W:0
2021/04/08 12:22:36 dtype("block1_conv1_W:0"): float32 (1728, dims=[64 3 3 3], max=[64 3 3 3])
2021/04/08 12:22:36 read: [0.037494343 -0.26969463 0.094652526 0.08827017 -0.23283976 0.42376447 0.3925612 -0.32398528 -0.03907965 0.034896567]
2021/04/08 12:22:36 error writing dataset: code -1
2021/04/08 12:22:36 dataset: block1_conv1_b:0
2021/04/08 12:22:36 dtype("block1_conv1_b:0"): float32 (64, dims=[64], max=[64])
2021/04/08 12:22:36 read: [0.4662595 0.09850298 0.3803252 0.66880196 0.4015123 0.90510356 0.43166816 1.302014 0.5306885 0.48993504]
2021/04/08 12:22:36 error writing dataset: code -1
2021/04/08 12:22:36 group: block1_conv2
2021/04/08 12:22:36 dataset: block1_conv2_W:0
2021/04/08 12:22:36 dtype("block1_conv2_W:0"): float32 (36864, dims=[64 64 3 3], max=[64 64 3 3])
2021/04/08 12:22:36 read: [-0.04239257 0.016082443 0.019441988 0.016724234 0.011314725 -0.010651959 -0.020697923 0.0045214365 -0.025818452 -0.021707503]
2021/04/08 12:22:36 error writing dataset: code -1
2021/04/08 12:22:36 dataset: block1_conv2_b:0
2021/04/08 12:22:36 dtype("block1_conv2_b:0"): float32 (64, dims=[64], max=[64])
2021/04/08 12:22:36 read: [0.12590808 0.16480374 0.2403943 0.23401979 -0.19334187 0.016635647 0.27144948 0.033857856 0.060762916 -0.5825159]
2021/04/08 12:22:36 error writing dataset: code -1
2021/04/08 12:22:36 group: block1_pool
2021/04/08 12:22:36 group: block2_conv1
2021/04/08 12:22:36 dataset: block2_conv1_W:0
2021/04/08 12:22:36 dtype("block2_conv1_W:0"): float32 (73728, dims=[128 64 3 3], max=[128 64 3 3])
2021/04/08 12:22:36 read: [-0.03356791 -0.008927183 -0.024192244 -0.022514213 0.033750452 0.013049415 -0.047272913 0.044430662 0.011701834 -0.05851893]
2021/04/08 12:22:36 error writing dataset: code -1
2021/04/08 12:22:36 dataset: block2_conv1_b:0
2021/04/08 12:22:36 dtype("block2_conv1_b:0"): float32 (128, dims=[128], max=[128])
2021/04/08 12:22:36 read: [0.03289016 0.2673606 -0.10560317 0.0037151142 0.2806813 -0.08938356 0.06679128 0.3166792 0.09598574 0.13152032]
2021/04/08 12:22:36 error writing dataset: code -1
2021/04/08 12:22:36 group: block2_conv2
2021/04/08 12:22:36 dataset: block2_conv2_W:0
2021/04/08 12:22:36 dtype("block2_conv2_W:0"): float32 (147456, dims=[128 128 3 3], max=[128 128 3 3])
2021/04/08 12:22:36 read: [0.024663102 0.006217653 -0.03943136 -0.011366554 -0.0071595274 -0.07429281 -0.029608304 -0.0028715055 -0.0645612 -0.021931699]
2021/04/08 12:22:36 error writing dataset: code -1
2021/04/08 12:22:36 dataset: block2_conv2_b:0
2021/04/08 12:22:36 dtype("block2_conv2_b:0"): float32 (128, dims=[128], max=[128])
2021/04/08 12:22:36 read: [-0.106785804 0.018788522 -0.083175175 0.066695474 0.010670638 0.37834755 0.117178276 0.20528455 0.2403423 0.27507326]
2021/04/08 12:22:36 error writing dataset: code -1
2021/04/08 12:22:36 group: block2_pool
2021/04/08 12:22:36 group: block3_conv1
2021/04/08 12:22:36 dataset: block3_conv1_W:0
2021/04/08 12:22:36 dtype("block3_conv1_W:0"): float32 (294912, dims=[256 128 3 3], max=[256 128 3 3])
2021/04/08 12:22:36 read: [-0.00069804006 -0.013151026 -0.028749265 -0.018696973 -0.020024896 -0.03709273 -0.023235539 -0.016867789 -0.030263098 -0.01557073]
2021/04/08 12:22:36 error writing dataset: code -1
2021/04/08 12:22:36 dataset: block3_conv1_b:0
2021/04/08 12:22:36 dtype("block3_conv1_b:0"): float32 (256, dims=[256], max=[256])
2021/04/08 12:22:36 read: [0.054253206 0.031449527 0.0039394326 0.06625993 0.08037452 -0.029116355 -0.019869104 0.052503522 -0.04464494 -0.027863992]
2021/04/08 12:22:36 error writing dataset: code -1
2021/04/08 12:22:36 group: block3_conv2
2021/04/08 12:22:36 dataset: block3_conv2_W:0
2021/04/08 12:22:36 dtype("block3_conv2_W:0"): float32 (589824, dims=[256 256 3 3], max=[256 256 3 3])
2021/04/08 12:22:36 read: [0.010546397 -0.0075213206 -0.012308732 -0.00813317 -0.0011238871 -0.0068346476 -0.0052868654 0.003602401 0.00017348543 0.0011733016]
2021/04/08 12:22:36 error writing dataset: code -1
2021/04/08 12:22:36 dataset: block3_conv2_b:0
2021/04/08 12:22:36 dtype("block3_conv2_b:0"): float32 (256, dims=[256], max=[256])
2021/04/08 12:22:36 read: [0.14706013 0.031156866 -0.08825309 0.20430823 0.046666417 0.028138021 -0.008833468 0.078824416 0.09819423 0.16311687]
2021/04/08 12:22:36 error writing dataset: code -1
2021/04/08 12:22:36 group: block3_conv3
2021/04/08 12:22:36 dataset: block3_conv3_W:0
2021/04/08 12:22:36 dtype("block3_conv3_W:0"): float32 (589824, dims=[256 256 3 3], max=[256 256 3 3])
2021/04/08 12:22:36 read: [-0.0019292701 0.021894354 0.020934677 0.009575244 0.017014114 0.023410853 0.010203712 -0.011256691 -0.0043449774 -0.0020875747]
2021/04/08 12:22:36 error writing dataset: code -1
2021/04/08 12:22:36 dataset: block3_conv3_b:0
2021/04/08 12:22:36 dtype("block3_conv3_b:0"): float32 (256, dims=[256], max=[256])
2021/04/08 12:22:36 read: [0.59477174 0.06861225 0.120992266 0.04660773 -0.036298074 -0.014072049 0.1710038 -0.0060511944 0.0059940466 -0.026757497]
2021/04/08 12:22:36 error writing dataset: code -1
2021/04/08 12:22:36 group: block3_pool
2021/04/08 12:22:36 group: block4_conv1
2021/04/08 12:22:36 dataset: block4_conv1_W:0
2021/04/08 12:22:36 dtype("block4_conv1_W:0"): float32 (1179648, dims=[512 256 3 3], max=[512 256 3 3])
2021/04/08 12:22:36 read: [0.014004302 0.004575403 -0.0029756208 0.00032509465 0.0012312339 -0.0063083465 -0.008074714 0.0023351565 -0.0074020172 -0.013586803]
2021/04/08 12:22:36 error writing dataset: code -1
2021/04/08 12:22:36 dataset: block4_conv1_b:0
2021/04/08 12:22:36 dtype("block4_conv1_b:0"): float32 (512, dims=[512], max=[512])
2021/04/08 12:22:36 read: [0.07294629 0.042588152 0.017346933 0.036423076 0.05075784 -0.008754369 0.061357588 0.03979362 -0.004130272 0.0101658]
2021/04/08 12:22:36 error writing dataset: code -1
2021/04/08 12:22:36 group: block4_conv2
2021/04/08 12:22:36 dataset: block4_conv2_W:0
2021/04/08 12:22:36 dtype("block4_conv2_W:0"): float32 (2359296, dims=[512 512 3 3], max=[512 512 3 3])
2021/04/08 12:22:36 read: [-0.0024421078 0.0077318195 0.003848577 0.0019056331 0.0070800385 0.006598049 0.0026297206 -0.0041080536 0.006198359 0.0041070357]
HDF5-DIAG: Error detected in HDF5 (1.10.0-patch1) thread 140628033795840:
  #000: ../../../src/H5Dio.c line 268 in H5Dwrite(): can't prepare for writing data
    major: Dataset
    minor: Write failed
  #001: ../../../src/H5Dio.c line 344 in H5D__pre_write(): can't write data
    major: Dataset
    minor: Write failed
  #002: ../../../src/H5Dio.c line 635 in H5D__write(): no write intent on file
    major: Dataset
    minor: Write failed
2021/04/08 12:22:36 error writing dataset: code -1
2021/04/08 12:22:36 dataset: block4_conv2_b:0
2021/04/08 12:22:36 dtype("block4_conv2_b:0"): float32 (512, dims=[512], max=[512])
2021/04/08 12:22:36 read: [0.14858922 0.13076046 0.05624543 0.0143612195 0.06421687 0.0483797 0.1259111 0.12565154 0.074530825 -0.017192408]
HDF5-DIAG: Error detected in HDF5 (1.10.0-patch1) thread 140628033795840:
  #000: ../../../src/H5Dio.c line 268 in H5Dwrite(): can't prepare for writing data
    major: Dataset
    minor: Write failed
  #001: ../../../src/H5Dio.c line 344 in H5D__pre_write(): can't write data
    major: Dataset
    minor: Write failed
  #002: ../../../src/H5Dio.c line 635 in H5D__write(): no write intent on file
    major: Dataset
    minor: Write failed
2021/04/08 12:22:36 error writing dataset: code -1
2021/04/08 12:22:36 group: block4_conv3
2021/04/08 12:22:36 dataset: block4_conv3_W:0
2021/04/08 12:22:36 dtype("block4_conv3_W:0"): float32 (2359296, dims=[512 512 3 3], max=[512 512 3 3])
2021/04/08 12:22:36 read: [-0.0023793117 0.014877313 0.012657574 0.0011535814 0.011865209 0.007362641 -0.00077057845 0.006034485 0.0054911408 0.0021901021]
HDF5-DIAG: Error detected in HDF5 (1.10.0-patch1) thread 140628033795840:
  #000: ../../../src/H5Dio.c line 268 in H5Dwrite(): can't prepare for writing data
    major: Dataset
    minor: Write failed
  #001: ../../../src/H5Dio.c line 344 in H5D__pre_write(): can't write data
    major: Dataset
    minor: Write failed
  #002: ../../../src/H5Dio.c line 635 in H5D__write(): no write intent on file
    major: Dataset
    minor: Write failed
2021/04/08 12:22:36 error writing dataset: code -1
2021/04/08 12:22:36 dataset: block4_conv3_b:0
2021/04/08 12:22:36 dtype("block4_conv3_b:0"): float32 (512, dims=[512], max=[512])
2021/04/08 12:22:36 read: [0.031873517 -0.011650325 0.16348428 0.05404903 -0.009967377 0.0026050396 -0.07919379 0.07228316 -0.12903117 0.05372257]
HDF5-DIAG: Error detected in HDF5 (1.10.0-patch1) thread 140628033795840:
  #000: ../../../src/H5Dio.c line 268 in H5Dwrite(): can't prepare for writing data
    major: Dataset
    minor: Write failed
  #001: ../../../src/H5Dio.c line 344 in H5D__pre_write(): can't write data
    major: Dataset
    minor: Write failed
  #002: ../../../src/H5Dio.c line 635 in H5D__write(): no write intent on file
    major: Dataset
    minor: Write failed
2021/04/08 12:22:36 error writing dataset: code -1
2021/04/08 12:22:36 group: block4_pool
2021/04/08 12:22:36 group: block5_conv1
2021/04/08 12:22:36 dataset: block5_conv1_W:0
2021/04/08 12:22:36 dtype("block5_conv1_W:0"): float32 (2359296, dims=[512 512 3 3], max=[512 512 3 3])
2021/04/08 12:22:36 read: [-0.009863088 0.004848376 0.005122031 0.004686328 0.010079567 0.012100901 0.0062693246 0.007307631 0.0052126674 0.0035778836]
HDF5-DIAG: Error detected in HDF5 (1.10.0-patch1) thread 140628033795840:
  #000: ../../../src/H5Dio.c line 268 in H5Dwrite(): can't prepare for writing data
    major: Dataset
    minor: Write failed
  #001: ../../../src/H5Dio.c line 344 in H5D__pre_write(): can't write data
    major: Dataset
    minor: Write failed
  #002: ../../../src/H5Dio.c line 635 in H5D__write(): no write intent on file
    major: Dataset
    minor: Write failed
2021/04/08 12:22:36 error writing dataset: code -1
2021/04/08 12:22:36 dataset: block5_conv1_b:0
2021/04/08 12:22:36 dtype("block5_conv1_b:0"): float32 (512, dims=[512], max=[512])
2021/04/08 12:22:36 read: [0.042981084 0.35259366 -0.11554319 0.20432594 0.12964748 -0.08826873 -0.04206413 0.030895568 0.12093141 0.6397485]
HDF5-DIAG: Error detected in HDF5 (1.10.0-patch1) thread 140628033795840:
  #000: ../../../src/H5Dio.c line 268 in H5Dwrite(): can't prepare for writing data
    major: Dataset
    minor: Write failed
  #001: ../../../src/H5Dio.c line 344 in H5D__pre_write(): can't write data
    major: Dataset
    minor: Write failed
  #002: ../../../src/H5Dio.c line 635 in H5D__write(): no write intent on file
    major: Dataset
    minor: Write failed
2021/04/08 12:22:36 error writing dataset: code -1
2021/04/08 12:22:36 group: block5_conv2
2021/04/08 12:22:36 dataset: block5_conv2_W:0
2021/04/08 12:22:36 dtype("block5_conv2_W:0"): float32 (2359296, dims=[512 512 3 3], max=[512 512 3 3])
2021/04/08 12:22:36 read: [0.008477414 0.0024646511 0.0032366544 0.0060234377 0.0044697914 0.004924782 0.0037874947 0.009905161 0.010246712 0.0059190476]
HDF5-DIAG: Error detected in HDF5 (1.10.0-patch1) thread 140628033795840:
  #000: ../../../src/H5Dio.c line 268 in H5Dwrite(): can't prepare for writing data
    major: Dataset
    minor: Write failed
  #001: ../../../src/H5Dio.c line 344 in H5D__pre_write(): can't write data
    major: Dataset
    minor: Write failed
  #002: ../../../src/H5Dio.c line 635 in H5D__write(): no write intent on file
    major: Dataset
    minor: Write failed
2021/04/08 12:22:36 error writing dataset: code -1
2021/04/08 12:22:36 dataset: block5_conv2_b:0
2021/04/08 12:22:36 dtype("block5_conv2_b:0"): float32 (512, dims=[512], max=[512])
2021/04/08 12:22:36 read: [0.100513615 0.06527971 0.3569455 0.0058571887 -0.15308182 -0.010849963 -0.4603464 0.16436121 0.23517226 -0.17672488]
HDF5-DIAG: Error detected in HDF5 (1.10.0-patch1) thread 140628033795840:
  #000: ../../../src/H5Dio.c line 268 in H5Dwrite(): can't prepare for writing data
    major: Dataset
    minor: Write failed
  #001: ../../../src/H5Dio.c line 344 in H5D__pre_write(): can't write data
    major: Dataset
    minor: Write failed
  #002: ../../../src/H5Dio.c line 635 in H5D__write(): no write intent on file
    major: Dataset
    minor: Write failed
2021/04/08 12:22:36 error writing dataset: code -1
2021/04/08 12:22:36 group: block5_conv3
2021/04/08 12:22:36 dataset: block5_conv3_W:0
2021/04/08 12:22:36 dtype("block5_conv3_W:0"): float32 (2359296, dims=[512 512 3 3], max=[512 512 3 3])
2021/04/08 12:22:36 read: [-0.0030180835 -0.009201038 -0.01421334 -0.004793251 -0.00021134656 -0.004520924 0.00307678 0.009854216 -0.001796579 0.0009858251]
HDF5-DIAG: Error detected in HDF5 (1.10.0-patch1) thread 140628033795840:
  #000: ../../../src/H5Dio.c line 268 in H5Dwrite(): can't prepare for writing data
    major: Dataset
    minor: Write failed
  #001: ../../../src/H5Dio.c line 344 in H5D__pre_write(): can't write data
    major: Dataset
    minor: Write failed
  #002: ../../../src/H5Dio.c line 635 in H5D__write(): no write intent on file
    major: Dataset
    minor: Write failed
2021/04/08 12:22:36 error writing dataset: code -1
2021/04/08 12:22:36 dataset: block5_conv3_b:0
2021/04/08 12:22:36 dtype("block5_conv3_b:0"): float32 (512, dims=[512], max=[512])
2021/04/08 12:22:36 read: [0.80902797 0.3094042 0.34415254 0.16621841 0.13615513 0.23337358 0.008004058 0.103328384 0.6381871 -0.026539654]
HDF5-DIAG: Error detected in HDF5 (1.10.0-patch1) thread 140628033795840:
  #000: ../../../src/H5Dio.c line 268 in H5Dwrite(): can't prepare for writing data
    major: Dataset
    minor: Write failed
  #001: ../../../src/H5Dio.c line 344 in H5D__pre_write(): can't write data
    major: Dataset
    minor: Write failed
  #002: ../../../src/H5Dio.c line 635 in H5D__write(): no write intent on file
    major: Dataset
    minor: Write failed
2021/04/08 12:22:36 error writing dataset: code -1
2021/04/08 12:22:36 group: block5_pool