search

rabix / bunny

[Legacy] Executor for CWL workflows. Executes sbg:draft-2 and CWL 1.0
http://rabix.io
Apache License 2.0
74 stars 28 forks source link

loadContents fails for s3 files #420

Closed adamstruck closed 6 years ago

adamstruck commented 6 years ago

CWL

cwlVersion: v1.0
class: ExpressionTool
id: expression_createsequencegrouping
requirements:
  - class: InlineJavascriptRequirement

inputs:
  sequence_grouping_tsv:
    type: File
    inputBinding:
      loadContents: true

outputs:
  sequence_grouping_array:
    type:
      type: array
      items:
        type: array
        items: string

expression:
  "${
      var lines = inputs.sequence_grouping_tsv.contents.split('\\n');
      var nline = lines.length;
      var eachgroup = [];
      var grouparray = [];
      for (var i = 0; i < nline; i++) {
        eachgroup.push(lines[i].split('\\t'));
        grouparray[i] = lines[i].split('\\t');
      }
      return {'sequence_grouping_array': grouparray};
  }"

Input

{
  "sequence_grouping_tsv": {
    "class": "File",
    "location": "s3://s3.amazonaws.com/dev-kf-data/sequence_grouping.txt"
  }
}

Logs

[2018-02-12 09:26:37.428] [ERROR] Failed to use Bindings
org.rabix.bindings.BindingException: org.rabix.bindings.cwl.processor.CWLPortProcessorException: org.rabix.bindings.cwl.processor.CWLPortProcessorException: Error: /dev-kf-data/sequence_grouping.txt (No such file or directory) while processing value: {metadata=null, format=null, dirname=null, nameroot=sequence_grouping, path=/dev-kf-data/sequence_grouping.txt, basename=sequence_grouping.txt, nameext=.txt, size=89451, contents=null, checksum=null, location=s3://s3.amazonaws.com/dev-kf-data/sequence_grouping.txt, secondaryFiles=[], class=File}
    at org.rabix.bindings.cwl.CWLProcessor.preprocess(CWLProcessor.java:107) ~[classes/:na]
    at org.rabix.bindings.cwl.CWLBindings.preprocess(CWLBindings.java:79) ~[classes/:na]
    at org.rabix.backend.tes.service.impl.LocalTESWorkerServiceImpl$TaskRunCallable.call(LocalTESWorkerServiceImpl.java:236) [classes/:na]
    at org.rabix.backend.tes.service.impl.LocalTESWorkerServiceImpl$TaskRunCallable.call(LocalTESWorkerServiceImpl.java:220) [classes/:na]
    at java.util.concurrent.FutureTask.run(FutureTask.java:266) [na:1.8.0_152]
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [na:1.8.0_152]
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [na:1.8.0_152]
    at java.lang.Thread.run(Thread.java:748) [na:1.8.0_152]
Caused by: org.rabix.bindings.cwl.processor.CWLPortProcessorException: org.rabix.bindings.cwl.processor.CWLPortProcessorException: Error: /dev-kf-data/sequence_grouping.txt (No such file or directory) while processing value: {metadata=null, format=null, dirname=null, nameroot=sequence_grouping, path=/dev-kf-data/sequence_grouping.txt, basename=sequence_grouping.txt, nameext=.txt, size=89451, contents=null, checksum=null, location=s3://s3.amazonaws.com/dev-kf-data/sequence_grouping.txt, secondaryFiles=[], class=File}
    at org.rabix.bindings.cwl.processor.callback.CWLPortProcessorHelper.loadInputContents(CWLPortProcessorHelper.java:86) ~[classes/:na]
    at org.rabix.bindings.cwl.CWLProcessor.preprocess(CWLProcessor.java:98) ~[classes/:na]
    ... 7 common frames omitted
Caused by: org.rabix.bindings.cwl.processor.CWLPortProcessorException: Error: /dev-kf-data/sequence_grouping.txt (No such file or directory) while processing value: {metadata=null, format=null, dirname=null, nameroot=sequence_grouping, path=/dev-kf-data/sequence_grouping.txt, basename=sequence_grouping.txt, nameext=.txt, size=89451, contents=null, checksum=null, location=s3://s3.amazonaws.com/dev-kf-data/sequence_grouping.txt, secondaryFiles=[], class=File}
    at org.rabix.bindings.cwl.processor.CWLPortProcessor.processValues(CWLPortProcessor.java:56) ~[classes/:na]
    at org.rabix.bindings.cwl.processor.CWLPortProcessor.processInputs(CWLPortProcessor.java:31) ~[classes/:na]
    at org.rabix.bindings.cwl.processor.callback.CWLPortProcessorHelper.loadInputContents(CWLPortProcessorHelper.java:84) ~[classes/:na]
    ... 8 common frames omitted
Caused by: java.io.FileNotFoundException: /dev-kf-data/sequence_grouping.txt (No such file or directory)
    at java.io.FileInputStream.open0(Native Method) ~[na:1.8.0_152]
    at java.io.FileInputStream.open(FileInputStream.java:195) ~[na:1.8.0_152]
    at java.io.FileInputStream.<init>(FileInputStream.java:138) ~[na:1.8.0_152]
    at org.rabix.bindings.cwl.helper.CWLFileValueHelper.loadContents(CWLFileValueHelper.java:235) ~[classes/:na]
    at org.rabix.bindings.cwl.helper.CWLFileValueHelper.setContents(CWLFileValueHelper.java:138) ~[classes/:na]
    at org.rabix.bindings.cwl.processor.callback.CWLLoadContentsPortProcessorCallback.process(CWLLoadContentsPortProcessorCallback.java:25) ~[classes/:na]
    at org.rabix.bindings.cwl.processor.CWLPortProcessor.processValue(CWLPortProcessor.java:70) ~[classes/:na]
    at org.rabix.bindings.cwl.processor.CWLPortProcessor.processValues(CWLPortProcessor.java:54) ~[classes/:na]
    ... 10 common frames omitted
[2018-02-12 09:26:39.044] [ERROR] Failed to retrieve TESTask
java.util.concurrent.ExecutionException: org.rabix.backend.tes.service.TESServiceException: Failed to use Bindings
    at java.util.concurrent.FutureTask.report(FutureTask.java:122) [na:1.8.0_152]
    at java.util.concurrent.FutureTask.get(FutureTask.java:192) [na:1.8.0_152]
    at org.rabix.backend.tes.service.impl.LocalTESWorkerServiceImpl$1.run(LocalTESWorkerServiceImpl.java:169) ~[classes/:na]
    at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [na:1.8.0_152]
    at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) [na:1.8.0_152]
    at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) [na:1.8.0_152]
    at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) [na:1.8.0_152]
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [na:1.8.0_152]
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [na:1.8.0_152]
    at java.lang.Thread.run(Thread.java:748) [na:1.8.0_152]
Caused by: org.rabix.backend.tes.service.TESServiceException: Failed to use Bindings
    at org.rabix.backend.tes.service.impl.LocalTESWorkerServiceImpl$TaskRunCallable.call(LocalTESWorkerServiceImpl.java:302) ~[classes/:na]
    at org.rabix.backend.tes.service.impl.LocalTESWorkerServiceImpl$TaskRunCallable.call(LocalTESWorkerServiceImpl.java:220) ~[classes/:na]
    at java.util.concurrent.FutureTask.run(FutureTask.java:266) [na:1.8.0_152]
    ... 3 common frames omitted
Caused by: org.rabix.bindings.BindingException: org.rabix.bindings.cwl.processor.CWLPortProcessorException: org.rabix.bindings.cwl.processor.CWLPortProcessorException: Error: /dev-kf-data/sequence_grouping.txt (No such file or directory) while processing value: {metadata=null, format=null, dirname=null, nameroot=sequence_grouping, path=/dev-kf-data/sequence_grouping.txt, basename=sequence_grouping.txt, nameext=.txt, size=89451, contents=null, checksum=null, location=s3://s3.amazonaws.com/dev-kf-data/sequence_grouping.txt, secondaryFiles=[], class=File}
    at org.rabix.bindings.cwl.CWLProcessor.preprocess(CWLProcessor.java:107) ~[classes/:na]
    at org.rabix.bindings.cwl.CWLBindings.preprocess(CWLBindings.java:79) ~[classes/:na]
    at org.rabix.backend.tes.service.impl.LocalTESWorkerServiceImpl$TaskRunCallable.call(LocalTESWorkerServiceImpl.java:236) ~[classes/:na]
    ... 5 common frames omitted
Caused by: org.rabix.bindings.cwl.processor.CWLPortProcessorException: org.rabix.bindings.cwl.processor.CWLPortProcessorException: Error: /dev-kf-data/sequence_grouping.txt (No such file or directory) while processing value: {metadata=null, format=null, dirname=null, nameroot=sequence_grouping, path=/dev-kf-data/sequence_grouping.txt, basename=sequence_grouping.txt, nameext=.txt, size=89451, contents=null, checksum=null, location=s3://s3.amazonaws.com/dev-kf-data/sequence_grouping.txt, secondaryFiles=[], class=File}
    at org.rabix.bindings.cwl.processor.callback.CWLPortProcessorHelper.loadInputContents(CWLPortProcessorHelper.java:86) ~[classes/:na]
    at org.rabix.bindings.cwl.CWLProcessor.preprocess(CWLProcessor.java:98) ~[classes/:na]
    ... 7 common frames omitted
Caused by: org.rabix.bindings.cwl.processor.CWLPortProcessorException: Error: /dev-kf-data/sequence_grouping.txt (No such file or directory) while processing value: {metadata=null, format=null, dirname=null, nameroot=sequence_grouping, path=/dev-kf-data/sequence_grouping.txt, basename=sequence_grouping.txt, nameext=.txt, size=89451, contents=null, checksum=null, location=s3://s3.amazonaws.com/dev-kf-data/sequence_grouping.txt, secondaryFiles=[], class=File}
    at org.rabix.bindings.cwl.processor.CWLPortProcessor.processValues(CWLPortProcessor.java:56) ~[classes/:na]
    at org.rabix.bindings.cwl.processor.CWLPortProcessor.processInputs(CWLPortProcessor.java:31) ~[classes/:na]
    at org.rabix.bindings.cwl.processor.callback.CWLPortProcessorHelper.loadInputContents(CWLPortProcessorHelper.java:84) ~[classes/:na]
    ... 8 common frames omitted
Caused by: java.io.FileNotFoundException: /dev-kf-data/sequence_grouping.txt (No such file or directory)
    at java.io.FileInputStream.open0(Native Method) ~[na:1.8.0_152]
    at java.io.FileInputStream.open(FileInputStream.java:195) ~[na:1.8.0_152]
    at java.io.FileInputStream.<init>(FileInputStream.java:138) ~[na:1.8.0_152]
    at org.rabix.bindings.cwl.helper.CWLFileValueHelper.loadContents(CWLFileValueHelper.java:235) ~[classes/:na]
    at org.rabix.bindings.cwl.helper.CWLFileValueHelper.setContents(CWLFileValueHelper.java:138) ~[classes/:na]
    at org.rabix.bindings.cwl.processor.callback.CWLLoadContentsPortProcessorCallback.process(CWLLoadContentsPortProcessorCallback.java:25) ~[classes/:na]
    at org.rabix.bindings.cwl.processor.CWLPortProcessor.processValue(CWLPortProcessor.java:70) ~[classes/:na]
    at org.rabix.bindings.cwl.processor.CWLPortProcessor.processValues(CWLPortProcessor.java:54) ~[classes/:na]
    ... 10 common frames omitted
adamstruck commented 6 years ago

This seemed to do the trick:

diff --git a/rabix-bindings-cwl/src/main/java/org/rabix/bindings/cwl/helper/CWLFileValueHelper.java b/rabix-bindings-cwl/src/main/java/org/rabix/bindings/cwl/helper/CWLFileValueHelper.java
index af21b353..0670b507 100644
--- a/rabix-bindings-cwl/src/main/java/org/rabix/bindings/cwl/helper/CWLFileValueHelper.java
+++ b/rabix-bindings-cwl/src/main/java/org/rabix/bindings/cwl/helper/CWLFileValueHelper.java
@@ -229,10 +229,12 @@ public class CWLFileValueHelper extends CWLBeanHelper {
    */
   private static String loadContents(Object fileData) throws IOException {
     String path = getPath(fileData);
+    URI uri = URI.create(getLocation(fileData));
     InputStream inputStream = null;
     try {
       File file = new File(path);
-      inputStream = new FileInputStream(file);
+      Path fPath = Paths.get(uri);
+      inputStream = Files.newInputStream(fPath);
       int bufferSize = file.length() > 0 && file.length() < CONTENTS_NUMBER_OF_BYTES ? (int) file.length() : CONTENTS_NUMBER_OF_BYTES;
       byte [] buffer = new byte[bufferSize];
       inputStream.read(buffer);