search

bedatadriven / renjin

JVM-based interpreter for the R language for the statistical analysis.
https://www.renjin.org
GNU General Public License v2.0
515 stars 82 forks source link

Conversion of RDS to JSON fails #518

Open varontron opened 4 years ago

varontron commented 4 years ago

Attempting to convert the objects, of various datatypes, found in an RDS file, to a JSON string via java. Both examples below work fine using the R interpreter directly on the CL.

jsonlite and jsonify result in different errors. In both cases, the exceptions are thrown when evaluating the value jcmd. The problem appears to be related to datatype conversion. Is that an accurate read, and if so, where to begin resolving the issue?

RenjinScriptEngineFactory factory = new RenjinScriptEngineFactory();
ScriptEngine engine = factory.getScriptEngine();
String cmd = String.format("robj <- readRDS('%s')", path);
engine.eval(cmd);

// for jsonlite:
engine.eval("library('org.renjin.cran:jsonlite')");
String jcmd = "jobj <- toJSON(robj)";

// for jsonify:
// engine.eval("library('org.renjin.cran:jsonify')");
// String jcmd = "jobj <- to_json(robj)";

engine.eval(jcmd);

StringWriter outputWriter = new StringWriter();
engine.getContext().setWriter(outputWriter);   
String pcmd = "print(jobj)";     
engine.eval(pcmd);
result = outputWriter.toString();

jsonlite errors:

org.renjin.eval.EvalException: values must be type 'character',
but FUN(list(staple = character(0), res1 = integer(0), res1.delta.mass = numeric(0), res2 = integer(0), res2.delta.mass = numeric(0), info = character(0))[[1L]], collapse = FALSE, complex = complex, na = na, oldna = oldna, rownames = rownames, dataframe = dataframe, indent = indent + 2L, no_dots = no_dots, auto_unbox = auto_unbox, Date = Date, POSIXt = POSIXt, factor = factor, raw = raw, matrix = matrix, digits = digits, null = null, force = force) result is type 'character'
at org.renjin.primitives.Evaluation.vapply(Evaluation.java:181)

jsonify errors:

Caused by: java.lang.IllegalStateException: Attempt to modify a shared SEXP
at org.renjin.gnur.api.Rinternals.SET_STRING_ELT(Rinternals.java:716)

Working R statements:

# jsonlite
toJSON(readRDS('/path/to/file.RDS'))
# jsonify
to_json(readRDS('/path/to/file.RDS'))
perNyfelt commented 4 years ago

Tried to replicate this on Renjin 3.5-beta76: rjson and jsonlite works (output differs though), jsonify has issues:

saveRDS(mtcars, "mtcars.rds")

robj <- readRDS("mtcars.rds")

library('org.renjin.cran:rjson')
jobj <- toJSON(robj)
print(jobj)
# [1] "{\"mpg\":[21,21,22.8,21.4,18.7,18.1,14.3,24.4,22.8,19.2,17.8,16.4,17.3,15.2,10.4,10.4,14.7,32.4,30.4,33.9,21.5,15.5,15.2,13.3,19.2,27.3,26,30.4,15.8,19.7,15,21.4],\"cyl\":[6,6,4,6,8,6,8,4,4,6,6,8,8,8,8,8,8,4,4,4,4,8,8,8,8,4,4,4,8,6,8,4],\"disp\":[160,160,108,258,360,225,360,146.7,140.8,167.6,167.6,275.8,275.8,275.8,472,460,440,78.7,75.7,71.1,120.1,318,304,350,400,79,120.3,95.1,351,145,301,121],\"hp\":[110,110,93,110,175,105,245,62,95,123,123,180,180,180,205,215,230,66,52,65,97,150,150,245,175,66,91,113,264,175,335,109],\"drat\":[3.9,3.9,3.85,3.08,3.15,2.76,3.21,3.69,3.92,3.92,3.92,3.07,3.07,3.07,2.93,3,3.23,4.08,4.93,4.22,3.7,2.76,3.15,3.73,3.08,4.08,4.43,3.77,4.22,3.62,3.54,4.11],\"wt\":[2.62,2.875,2.32,3.215,3.44,3.46,3.57,3.19,3.15,3.44,3.44,4.07,3.73,3.78,5.25,5.424,5.345,2.2,1.615,1.835,2.465,3.52,3.435,3.84,3.845,1.935,2.14,1.513,3.17,2.77,3.57,2.78],\"qsec\":[16.46,17.02,18.61,19.44,17.02,20.22,15.84,20,22.9,18.3,18.9,17.4,17.6,18,17.98,17.82,17.42,19.47,18.52,19.9,20.01,16.87,17.3,15.41,17.05,18.9,16.7,16.9,14.5,15.5,14.6,18.6],\"vs\":[0,0,1,1,0,1,0,1,1,1,1,0,0,0,0,0,0,1,1,1,1,0,0,0,0,1,0,1,0,0,0,1],\"am\":[1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,0,0,0,0,0,1,1,1,1,1,1,1],\"gear\":[4,4,4,3,3,3,3,4,4,4,4,3,3,3,3,3,3,4,4,4,3,3,3,3,3,4,5,5,5,5,5,4],\"carb\":[4,4,1,1,2,1,4,2,2,4,4,3,3,3,4,4,4,1,2,1,1,2,2,4,2,1,2,2,4,6,8,2]}"
library('org.renjin.cran:jsonlite')
jobj <- toJSON(robj)
print(jobj)
# [{"mpg":21,"cyl":6,"disp":160,"hp":110,"drat":3.9,"wt":2.62,"qsec":16.46,"vs":0,"am":1,"gear":4,"carb":4,"_row":"Mazda RX4"},{"mpg":21,"cyl":6,"disp":160,"hp":110,"drat":3.9,"wt":2.875,"qsec":17.02,"vs":0,"am":1,"gear":4,"carb":4,"_row":"Mazda RX4 Wag"},{"mpg":22.8,"cyl":4,"disp":108,"hp":93,"drat":3.85,"wt":2.32,"qsec":18.61,"vs":1,"am":1,"gear":4,"carb":1,"_row":"Datsun 710"},{"mpg":21.4,"cyl":6,"disp":258,"hp":110,"drat":3.08,"wt":3.215,"qsec":19.44,"vs":1,"am":0,"gear":3,"carb":1,"_row":"Hornet 4 Drive"},{"mpg":18.7,"cyl":8,"disp":360,"hp":175,"drat":3.15,"wt":3.44,"qsec":17.02,"vs":0,"am":0,"gear":3,"carb":2,"_row":"Hornet Sportabout"},{"mpg":18.1,"cyl":6,"disp":225,"hp":105,"drat":2.76,"wt":3.46,"qsec":20.22,"vs":1,"am":0,"gear":3,"carb":1,"_row":"Valiant"},{"mpg":14.3,"cyl":8,"disp":360,"hp":245,"drat":3.21,"wt":3.57,"qsec":15.84,"vs":0,"am":0,"gear":3,"carb":4,"_row":"Duster 360"},{"mpg":24.4,"cyl":4,"disp":146.7,"hp":62,"drat":3.69,"wt":3.19,"qsec":20,"vs":1,"am":0,"gear":4,"carb":2,"_row":"Merc 240D"},{"mpg":22.8,"cyl":4,"disp":140.8,"hp":95,"drat":3.92,"wt":3.15,"qsec":22.9,"vs":1,"am":0,"gear":4,"carb":2,"_row":"Merc 230"},{"mpg":19.2,"cyl":6,"disp":167.6,"hp":123,"drat":3.92,"wt":3.44,"qsec":18.3,"vs":1,"am":0,"gear":4,"carb":4,"_row":"Merc 280"},{"mpg":17.8,"cyl":6,"disp":167.6,"hp":123,"drat":3.92,"wt":3.44,"qsec":18.9,"vs":1,"am":0,"gear":4,"carb":4,"_row":"Merc 280C"},{"mpg":16.4,"cyl":8,"disp":275.8,"hp":180,"drat":3.07,"wt":4.07,"qsec":17.4,"vs":0,"am":0,"gear":3,"carb":3,"_row":"Merc 450SE"},{"mpg":17.3,"cyl":8,"disp":275.8,"hp":180,"drat":3.07,"wt":3.73,"qsec":17.6,"vs":0,"am":0,"gear":3,"carb":3,"_row":"Merc 450SL"},{"mpg":15.2,"cyl":8,"disp":275.8,"hp":180,"drat":3.07,"wt":3.78,"qsec":18,"vs":0,"am":0,"gear":3,"carb":3,"_row":"Merc 450SLC"},{"mpg":10.4,"cyl":8,"disp":472,"hp":205,"drat":2.93,"wt":5.25,"qsec":17.98,"vs":0,"am":0,"gear":3,"carb":4,"_row":"Cadillac Fleetwood"},{"mpg":10.4,"cyl":8,"disp":460,"hp":215,"drat":3,"wt":5.424,"qsec":17.82,"vs":0,"am":0,"gear":3,"carb":4,"_row":"Lincoln Continental"},{"mpg":14.7,"cyl":8,"disp":440,"hp":230,"drat":3.23,"wt":5.345,"qsec":17.42,"vs":0,"am":0,"gear":3,"carb":4,"_row":"Chrysler Imperial"},{"mpg":32.4,"cyl":4,"disp":78.7,"hp":66,"drat":4.08,"wt":2.2,"qsec":19.47,"vs":1,"am":1,"gear":4,"carb":1,"_row":"Fiat 128"},{"mpg":30.4,"cyl":4,"disp":75.7,"hp":52,"drat":4.93,"wt":1.615,"qsec":18.52,"vs":1,"am":1,"gear":4,"carb":2,"_row":"Honda Civic"},{"mpg":33.9,"cyl":4,"disp":71.1,"hp":65,"drat":4.22,"wt":1.835,"qsec":19.9,"vs":1,"am":1,"gear":4,"carb":1,"_row":"Toyota Corolla"},{"mpg":21.5,"cyl":4,"disp":120.1,"hp":97,"drat":3.7,"wt":2.465,"qsec":20.01,"vs":1,"am":0,"gear":3,"carb":1,"_row":"Toyota Corona"},{"mpg":15.5,"cyl":8,"disp":318,"hp":150,"drat":2.76,"wt":3.52,"qsec":16.87,"vs":0,"am":0,"gear":3,"carb":2,"_row":"Dodge Challenger"},{"mpg":15.2,"cyl":8,"disp":304,"hp":150,"drat":3.15,"wt":3.435,"qsec":17.3,"vs":0,"am":0,"gear":3,"carb":2,"_row":"AMC Javelin"},{"mpg":13.3,"cyl":8,"disp":350,"hp":245,"drat":3.73,"wt":3.84,"qsec":15.41,"vs":0,"am":0,"gear":3,"carb":4,"_row":"Camaro Z28"},{"mpg":19.2,"cyl":8,"disp":400,"hp":175,"drat":3.08,"wt":3.845,"qsec":17.05,"vs":0,"am":0,"gear":3,"carb":2,"_row":"Pontiac Firebird"},{"mpg":27.3,"cyl":4,"disp":79,"hp":66,"drat":4.08,"wt":1.935,"qsec":18.9,"vs":1,"am":1,"gear":4,"carb":1,"_row":"Fiat X1-9"},{"mpg":26,"cyl":4,"disp":120.3,"hp":91,"drat":4.43,"wt":2.14,"qsec":16.7,"vs":0,"am":1,"gear":5,"carb":2,"_row":"Porsche 914-2"},{"mpg":30.4,"cyl":4,"disp":95.1,"hp":113,"drat":3.77,"wt":1.513,"qsec":16.9,"vs":1,"am":1,"gear":5,"carb":2,"_row":"Lotus Europa"},{"mpg":15.8,"cyl":8,"disp":351,"hp":264,"drat":4.22,"wt":3.17,"qsec":14.5,"vs":0,"am":1,"gear":5,"carb":4,"_row":"Ford Pantera L"},{"mpg":19.7,"cyl":6,"disp":145,"hp":175,"drat":3.62,"wt":2.77,"qsec":15.5,"vs":0,"am":1,"gear":5,"carb":6,"_row":"Ferrari Dino"},{"mpg":15,"cyl":8,"disp":301,"hp":335,"drat":3.54,"wt":3.57,"qsec":14.6,"vs":0,"am":1,"gear":5,"carb":8,"_row":"Maserati Bora"},{"mpg":21.4,"cyl":4,"disp":121,"hp":109,"drat":4.11,"wt":2.78,"qsec":18.6,"vs":1,"am":1,"gear":4,"carb":2,"_row":"Volvo 142E"}]

To put this in a java context: For jsonlite the following two examples works fine:

  @Test
  public void testJsonLite() throws ScriptException {
      RenjinScriptEngineFactory factory = new RenjinScriptEngineFactory();
      engine = factory.getScriptEngine();
      engine.eval("saveRDS(mtcars, 'mtcars.rds'); robj <- readRDS('mtcars.rds')");

      SEXP jobj = (SEXP)engine.eval("library('org.renjin.cran:jsonlite'); toJSON(robj)");
      System.out.println(jobj.asString());
  }

  @Test
  public void testJsonLiteAltOutput() throws ScriptException {
    RenjinScriptEngineFactory factory = new RenjinScriptEngineFactory();
    engine = factory.getScriptEngine();
    engine.eval("foo <- c(1,2,3); saveRDS(foo, 'foo.rds'); robj <- readRDS('foo.rds')");

    engine.eval("library('org.renjin.cran:jsonlite'); jobj <- toJSON(robj)");

    StringWriter outputWriter = new StringWriter();
    engine.getContext().setWriter(outputWriter);
    String pcmd = "print(jobj)";
    engine.eval(pcmd);
    String result = outputWriter.toString();
    System.out.println(result);
  }

...and the same ones also works fine with rjson. Either there is some specific quirk about your rds file or some other issue i failed to replicate but you should be able to do what you want with either jsonlite or rjson.

If you want more help to pinpoint the issue please specify renjin and package versions.