Open migel opened 9 years ago
It sounds like this is probably not specific to Scala module, but rather something related to YAML module. If so, it would be great to have a Java equivalent of the test so that test could be aded to YAML module (which does not depend on Scala).
java version:
import java.io.IOException;
import org.junit.Test;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.annotation.JsonTypeInfo;
import com.fasterxml.jackson.databind.JsonSerializer;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializerProvider;
import com.fasterxml.jackson.databind.jsontype.TypeSerializer;
import com.fasterxml.jackson.databind.module.SimpleModule;
import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
public class TestYAML {
// Dummy INDArray
static class INDArray {
DataBuffer _data = new DataBuffer();
public int[] shape() {
return new int[] { 1, 3 };
}
public DataBuffer data() {
return _data;
}
public char ordering() {
return 'f';
}
}
// Dummy DataBuffer
static class DataBuffer {
public int dataType() {
return 1;
}
public byte[] asBytes() {
return new byte[] { 1, 2, 3 };
}
}
// Custom serializer for INDArray
class NDArraySerializer extends JsonSerializer<INDArray> {
public void serialize(INDArray value, JsonGenerator jgen,
SerializerProvider provider) throws IOException {
jgen.writeStartObject();
jgen.writeArrayFieldStart("shape");
for (int i : value.shape())
jgen.writeNumber(i);
jgen.writeEndArray();
String dtype;
if (value.data().dataType() == 1 /* DataBuffer.FLOAT */)
dtype = "float";
else
dtype = "double";
jgen.writeStringField("dtype", dtype);
jgen.writeStringField("order", Character.toString(value.ordering()));
jgen.writeBinaryField("data", value.data().asBytes());
jgen.writeEndObject();
}
public void serializeWithType(INDArray value, JsonGenerator jgen,
SerializerProvider provider, TypeSerializer typeSer)
throws IOException {
typeSer.writeTypePrefixForObject(value, jgen);
serialize(value, jgen, provider);
typeSer.writeTypeSuffixForObject(value, jgen);
}
}
@Test
public void test() throws JsonProcessingException {
SimpleModule nd4jModule = new SimpleModule();
nd4jModule.addSerializer(INDArray.class, new NDArraySerializer());
ObjectMapper jsonMapper = new ObjectMapper();
jsonMapper.registerModule(nd4jModule);
jsonMapper.enableDefaultTyping(ObjectMapper.DefaultTyping.NON_FINAL,
JsonTypeInfo.As.PROPERTY);
ObjectMapper yamlMapper = new ObjectMapper(new YAMLFactory());
yamlMapper.registerModule(nd4jModule);
yamlMapper.enableDefaultTyping(ObjectMapper.DefaultTyping.NON_FINAL,
JsonTypeInfo.As.PROPERTY);
INDArray array = new INDArray();
jsonMapper.writeValueAsString(array);
// FAIL HERE:
yamlMapper.writeValueAsString(array);
}
}
I realized that I've created an additional nesting when serializing with type information. I've changed it so it wouldn't and I don't get an exception in this case. I think the above case is still a bug. Changed code:
import java.io.IOException;
import org.junit.Test;
import com.fasterxml.jackson.annotation.JsonTypeInfo;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonSerializer;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializerProvider;
import com.fasterxml.jackson.databind.jsontype.TypeSerializer;
import com.fasterxml.jackson.databind.module.SimpleModule;
import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
public class TestYAML {
// Dummy INDArray
static class INDArray {
DataBuffer _data = new DataBuffer();
public int[] shape() {
return new int[] { 1, 3 };
}
public DataBuffer data() {
return _data;
}
public char ordering() {
return 'f';
}
}
// Dummy DataBuffer
static class DataBuffer {
public int dataType() {
return 1;
}
public byte[] asBytes() {
return new byte[] { 1, 2, 3 };
}
}
// Custom serializer for INDArray
class NDArraySerializer extends JsonSerializer<INDArray> {
public void serialize(INDArray value, JsonGenerator jgen,
SerializerProvider provider) throws IOException {
jgen.writeStartObject();
serializeFields(value, jgen);
jgen.writeEndObject();
}
public void serializeWithType(INDArray value, JsonGenerator jgen,
SerializerProvider provider, TypeSerializer typeSer)
throws IOException {
typeSer.writeTypePrefixForObject(value, jgen);
// AVOID CREATING NESTED OBJECT
serializeFields(value, jgen);
typeSer.writeTypeSuffixForObject(value, jgen);
}
void serializeFields(INDArray value, JsonGenerator jgen)
throws IOException {
jgen.writeArrayFieldStart("shape");
for (int i : value.shape())
jgen.writeNumber(i);
jgen.writeEndArray();
String dtype;
if (value.data().dataType() == 1 /* DataBuffer.FLOAT */)
dtype = "float";
else
dtype = "double";
jgen.writeStringField("dtype", dtype);
jgen.writeStringField("order", Character.toString(value.ordering()));
jgen.writeBinaryField("data", value.data().asBytes());
}
}
@Test
public void test() throws JsonProcessingException {
SimpleModule nd4jModule = new SimpleModule();
nd4jModule.addSerializer(INDArray.class, new NDArraySerializer());
ObjectMapper jsonMapper = new ObjectMapper();
jsonMapper.registerModule(nd4jModule);
jsonMapper.enableDefaultTyping(ObjectMapper.DefaultTyping.NON_FINAL,
JsonTypeInfo.As.PROPERTY);
ObjectMapper yamlMapper = new ObjectMapper(new YAMLFactory());
yamlMapper.registerModule(nd4jModule);
yamlMapper.enableDefaultTyping(ObjectMapper.DefaultTyping.NON_FINAL,
JsonTypeInfo.As.PROPERTY);
INDArray array = new INDArray();
jsonMapper.writeValueAsString(array);
yamlMapper.writeValueAsString(array);
}
}
Hi, I'm trying to write a custom serializer for an Nd4j array (see http://nd4j.org). I was about to get it working with json but when I try yaml output I get an exception:
In order to simplify the test I created dummy classes for
Nd4j
so I know the issue is not specific to that package. This test case:If I comment out the
enableDefaultTyping
call it works.