Support projects at osf.io

[dmbates]

I have done some exploring of the API at https://api.osf.io. The way that I would see generating a DataDep for files in a project on that site is

• start with a project id, say kz483 which is the project "Stable Preview Difficulty Effects"

  julia> JSON.print(JSON.parse(String(HTTP.get("https://api.osf.io/v2/nodes/kz483").body)),2)
  {
  "meta": {
  "version": "2.0"
  },
  "data": {
  "links": {
    "self": "https://api.osf.io/v2/nodes/kz483/",
    "html": "https://osf.io/kz483/"
  },
  "attributes": {
    "public": true,
    "date_created": "2018-08-20T14:52:20.727535",
    "preprint": false,
    "registration": false,
    "current_user_permissions": [
      "read"
    ],
    "description": "Parafoveal word processing can affect the reader's eyes in at least two different ways. The present study focuses on the spatio-temporal effects of the preview difficulty and discusses implications for models of eye movement control during reading. ",
    "analytics_key": "84a49b0b67bd5a658a7f874d3f28007002adeacbb9f63236f7a6e15aa270af164fb050af6e220de246b51048ee444a0564bdd00582ac9c9f46d0d5a571865eed14a9a9d4e1df39b7ee3be6d6b949c5aeab72a9650f34dd1529047a535b4c17e81e230faccd4bdea149d04f017043fe02fa02196a11b1ad8659e8ecc3f6e0d1b6c42088ff47fbfe10ba160cceafc33354",
    "date_modified": "2018-08-21T11:21:42.324633",
    "fork": false,
    "subjects": [],
    "category": "project",
    "current_user_is_contributor": false,
    "current_user_can_comment": false,
    "node_license": null,
    "access_requests_enabled": true,
    "title": "Stable Preview Difficulty Effects",
    "tags": [
      "boundary paradigm",
      "delayed parafoveal-on-foveal effect",
      "eye movements",
      "E-Z Reader",
      "forced fixations",
      "gaze-contingent display changes",
      "Glenmore",
      "parafoveal-on-foveal effect",
      "parafoveal preview benefit",
      "reading",
      "SWIFT"
    ],
    "custom_citation": null,
    "wiki_enabled": true,
    "collection": false
  },
  "id": "kz483",
  "type": "nodes",
  "relationships": {
    "view_only_links": {
      "links": {
        "related": {
          "meta": {},
          "href": "https://api.osf.io/v2/nodes/kz483/view_only_links/"
        }
      }
    },
    "node_links": {
      "links": {
        "related": {
          "meta": {},
          "href": "https://api.osf.io/v2/nodes/kz483/node_links/"
        }
      }
    },
    "draft_registrations": {
      "links": {
        "related": {
          "meta": {},
          "href": "https://api.osf.io/v2/nodes/kz483/draft_registrations/"
        }
      }
    },
    "affiliated_institutions": {
      "links": {
        "self": {
          "meta": {},
          "href": "https://api.osf.io/v2/nodes/kz483/relationships/institutions/"
        },
        "related": {
          "meta": {},
          "href": "https://api.osf.io/v2/nodes/kz483/institutions/"
        }
      }
    },
    "root": {
      "links": {
        "related": {
          "meta": {},
          "href": "https://api.osf.io/v2/nodes/kz483/"
        }
      },
      "data": {
        "id": "kz483",
        "type": "nodes"
      }
    },
    "forks": {
      "links": {
        "related": {
          "meta": {},
          "href": "https://api.osf.io/v2/nodes/kz483/forks/"
        }
      }
    },
    "linked_by_nodes": {
      "links": {
        "related": {
          "meta": {},
          "href": "https://api.osf.io/v2/nodes/kz483/linked_by_nodes/"
        }
      }
    },
    "preprints": {
      "links": {
        "related": {
          "meta": {},
          "href": "https://api.osf.io/v2/nodes/kz483/preprints/"
        }
      }
    },
    "wikis": {
      "links": {
        "related": {
          "meta": {},
          "href": "https://api.osf.io/v2/nodes/kz483/wikis/"
        }
      }
    },
    "logs": {
      "links": {
        "related": {
          "meta": {},
          "href": "https://api.osf.io/v2/nodes/kz483/logs/"
        }
      }
    },
    "linked_by_registrations": {
      "links": {
        "related": {
          "meta": {},
          "href": "https://api.osf.io/v2/nodes/kz483/linked_by_registrations/"
        }
      }
    },
    "settings": {
      "links": {
        "related": {
          "meta": {},
          "href": "https://api.osf.io/v2/nodes/kz483/settings/"
        }
      }
    },
    "files": {
      "links": {
        "related": {
          "meta": {},
          "href": "https://api.osf.io/v2/nodes/kz483/files/"
        }
      }
    },
    "registrations": {
      "links": {
        "related": {
          "meta": {},
          "href": "https://api.osf.io/v2/nodes/kz483/registrations/"
        }
      }
    },
    "comments": {
      "links": {
        "related": {
          "meta": {},
          "href": "https://api.osf.io/v2/nodes/kz483/comments/?filter%5Btarget%5D=kz483"
        }
      }
    },
    "children": {
      "links": {
        "related": {
          "meta": {},
          "href": "https://api.osf.io/v2/nodes/kz483/children/"
        }
      }
    },
    "identifiers": {
      "links": {
        "related": {
          "meta": {},
          "href": "https://api.osf.io/v2/nodes/kz483/identifiers/"
        }
      }
    },
    "linked_registrations": {
      "links": {
        "self": {
          "meta": {},
          "href": "https://api.osf.io/v2/nodes/kz483/relationships/linked_registrations/"
        },
        "related": {
          "meta": {},
          "href": "https://api.osf.io/v2/nodes/kz483/linked_registrations/"
        }
      }
    },
    "citation": {
      "links": {
        "related": {
          "meta": {},
          "href": "https://api.osf.io/v2/nodes/kz483/citation/"
        }
      }
    },
    "region": {
      "links": {
        "related": {
          "meta": {},
          "href": "https://api.osf.io/v2/regions/us/"
        }
      },
      "data": {
        "id": "us",
        "type": "regions"
      }
    },
    "contributors": {
      "links": {
        "related": {
          "meta": {},
          "href": "https://api.osf.io/v2/nodes/kz483/contributors/"
        }
      }
    },
    "linked_nodes": {
      "links": {
        "self": {
          "meta": {},
          "href": "https://api.osf.io/v2/nodes/kz483/relationships/linked_nodes/"
        },
        "related": {
          "meta": {},
          "href": "https://api.osf.io/v2/nodes/kz483/linked_nodes/"
        }
      }
    },
    "implicit_contributors": {
      "links": {
        "related": {
          "meta": {},
          "href": "https://api.osf.io/v2/nodes/kz483/implicit_contributors/"
        }
      }
    }
  }
  }
  }

  The files relationship takes us to osfstorage

  julia> JSON.print(JSON.parse(String(HTTP.get("https://api.osf.io/v2/nodes/kz483/files").body)),2)
  {
  "meta": {
  "version": "2.0"
  },
  "links": {
  "prev": null,
  "meta": {
    "total": 1,
    "per_page": 10
  },
  "next": null,
  "first": null,
  "last": null
  },
  "data": [
  {
    "links": {
      "new_folder": "https://files.osf.io/v1/resources/kz483/providers/osfstorage/?kind=folder",
      "upload": "https://files.osf.io/v1/resources/kz483/providers/osfstorage/",
      "storage_addons": "https://api.osf.io/v2/addons/?filter%5Bcategories%5D=storage"
    },
    "attributes": {
      "kind": "folder",
      "name": "osfstorage",
      "provider": "osfstorage",
      "node": "kz483",
      "path": "/"
    },
    "id": "kz483:osfstorage",
    "type": "files",
    "relationships": {
      "files": {
        "links": {
          "related": {
            "meta": {},
            "href": "https://api.osf.io/v2/nodes/kz483/files/osfstorage/"
          }
        }
      }
    }
  }
  ]
  }

  within which there is information on several file folders, including R_Data.

  julia> JSON.print(JSON.parse(String(HTTP.get("https://api.osf.io/v2/nodes/kz483/files/osfstorage").body)),2)
  {
  "meta": {
  "version": "2.0"
  },
  "links": {
  "prev": null,
  "meta": {
    "total": 4,
    "per_page": 10
  },
  "next": null,
  "first": null,
  "last": null
  },
  "data": [
  {
    "links": {
      "delete": "https://files.osf.io/v1/resources/kz483/providers/osfstorage/5b7adf0684f087001cdb4431/",
      "self": "https://api.osf.io/v2/files/5b7adf0684f087001cdb4431/",
      "new_folder": "https://files.osf.io/v1/resources/kz483/providers/osfstorage/5b7adf0684f087001cdb4431/?kind=folder",
      "upload": "https://files.osf.io/v1/resources/kz483/providers/osfstorage/5b7adf0684f087001cdb4431/",
      "info": "https://api.osf.io/v2/files/5b7adf0684f087001cdb4431/",
      "move": "https://files.osf.io/v1/resources/kz483/providers/osfstorage/5b7adf0684f087001cdb4431/"
    },
    "attributes": {
      "date_created": null,
      "current_version": 1,
      "materialized_path": "/R_Analyses/",
      "path": "/5b7adf0684f087001cdb4431/",
      "name": "R_Analyses",
      "guid": null,
      "date_modified": null,
      "extra": {
        "hashes": {
          "sha256": null,
          "md5": null
        }
      },
      "last_touched": null,
      "checkout": null,
      "size": null,
      "current_user_can_comment": false,
      "kind": "folder",
      "provider": "osfstorage",
      "tags": []
    },
    "id": "5b7adf0684f087001cdb4431",
    "type": "files",
    "relationships": {
      "files": {
        "links": {
          "related": {
            "meta": {},
            "href": "https://api.osf.io/v2/nodes/kz483/files/osfstorage/5b7adf0684f087001cdb4431/"
          }
        }
      },
      "target": {
        "links": {
          "related": {
            "meta": {
              "type": "node"
            },
            "href": "https://api.osf.io/v2/nodes/kz483/"
          }
        }
      },
      "node": {
        "links": {
          "related": {
            "meta": {},
            "href": "https://api.osf.io/v2/nodes/kz483/"
          }
        },
        "data": {
          "id": "kz483",
          "type": "nodes"
        }
      }
    }
  },
  {
    "links": {
      "delete": "https://files.osf.io/v1/resources/kz483/providers/osfstorage/5b7adfa925e63c0018e407e2/",
      "self": "https://api.osf.io/v2/files/5b7adfa925e63c0018e407e2/",
      "new_folder": "https://files.osf.io/v1/resources/kz483/providers/osfstorage/5b7adfa925e63c0018e407e2/?kind=folder",
      "upload": "https://files.osf.io/v1/resources/kz483/providers/osfstorage/5b7adfa925e63c0018e407e2/",
      "info": "https://api.osf.io/v2/files/5b7adfa925e63c0018e407e2/",
      "move": "https://files.osf.io/v1/resources/kz483/providers/osfstorage/5b7adfa925e63c0018e407e2/"
    },
    "attributes": {
      "date_created": null,
      "current_version": 1,
      "materialized_path": "/R_Functions/",
      "path": "/5b7adfa925e63c0018e407e2/",
      "name": "R_Functions",
      "guid": null,
      "date_modified": null,
      "extra": {
        "hashes": {
          "sha256": null,
          "md5": null
        }
      },
      "last_touched": null,
      "checkout": null,
      "size": null,
      "current_user_can_comment": false,
      "kind": "folder",
      "provider": "osfstorage",
      "tags": []
    },
    "id": "5b7adfa925e63c0018e407e2",
    "type": "files",
    "relationships": {
      "files": {
        "links": {
          "related": {
            "meta": {},
            "href": "https://api.osf.io/v2/nodes/kz483/files/osfstorage/5b7adfa925e63c0018e407e2/"
          }
        }
      },
      "target": {
        "links": {
          "related": {
            "meta": {
              "type": "node"
            },
            "href": "https://api.osf.io/v2/nodes/kz483/"
          }
        }
      },
      "node": {
        "links": {
          "related": {
            "meta": {},
            "href": "https://api.osf.io/v2/nodes/kz483/"
          }
        },
        "data": {
          "id": "kz483",
          "type": "nodes"
        }
      }
    }
  },
  {
    "links": {
      "delete": "https://files.osf.io/v1/resources/kz483/providers/osfstorage/5b7adfc325e63c0019e40676/",
      "self": "https://api.osf.io/v2/files/5b7adfc325e63c0019e40676/",
      "new_folder": "https://files.osf.io/v1/resources/kz483/providers/osfstorage/5b7adfc325e63c0019e40676/?kind=folder",
      "upload": "https://files.osf.io/v1/resources/kz483/providers/osfstorage/5b7adfc325e63c0019e40676/",
      "info": "https://api.osf.io/v2/files/5b7adfc325e63c0019e40676/",
      "move": "https://files.osf.io/v1/resources/kz483/providers/osfstorage/5b7adfc325e63c0019e40676/"
    },
    "attributes": {
      "date_created": null,
      "current_version": 1,
      "materialized_path": "/R_Data/",
      "path": "/5b7adfc325e63c0019e40676/",
      "name": "R_Data",
      "guid": null,
      "date_modified": null,
      "extra": {
        "hashes": {
          "sha256": null,
          "md5": null
        }
      },
      "last_touched": null,
      "checkout": null,
      "size": null,
      "current_user_can_comment": false,
      "kind": "folder",
      "provider": "osfstorage",
      "tags": []
    },
    "id": "5b7adfc325e63c0019e40676",
    "type": "files",
    "relationships": {
      "files": {
        "links": {
          "related": {
            "meta": {},
            "href": "https://api.osf.io/v2/nodes/kz483/files/osfstorage/5b7adfc325e63c0019e40676/"
          }
        }
      },
      "target": {
        "links": {
          "related": {
            "meta": {
              "type": "node"
            },
            "href": "https://api.osf.io/v2/nodes/kz483/"
          }
        }
      },
      "node": {
        "links": {
          "related": {
            "meta": {},
            "href": "https://api.osf.io/v2/nodes/kz483/"
          }
        },
        "data": {
          "id": "kz483",
          "type": "nodes"
        }
      }
    }
  },
  {
    "links": {
      "delete": "https://files.osf.io/v1/resources/kz483/providers/osfstorage/5b7ae01425e63c0018e40863/",
      "self": "https://api.osf.io/v2/files/5b7ae01425e63c0018e40863/",
      "new_folder": "https://files.osf.io/v1/resources/kz483/providers/osfstorage/5b7ae01425e63c0018e40863/?kind=folder",
      "upload": "https://files.osf.io/v1/resources/kz483/providers/osfstorage/5b7ae01425e63c0018e40863/",
      "info": "https://api.osf.io/v2/files/5b7ae01425e63c0018e40863/",
      "move": "https://files.osf.io/v1/resources/kz483/providers/osfstorage/5b7ae01425e63c0018e40863/"
    },
    "attributes": {
      "date_created": null,
      "current_version": 1,
      "materialized_path": "/Manuscript/",
      "path": "/5b7ae01425e63c0018e40863/",
      "name": "Manuscript",
      "guid": null,
      "date_modified": null,
      "extra": {
        "hashes": {
          "sha256": null,
          "md5": null
        }
      },
      "last_touched": null,
      "checkout": null,
      "size": null,
      "current_user_can_comment": false,
      "kind": "folder",
      "provider": "osfstorage",
      "tags": []
    },
    "id": "5b7ae01425e63c0018e40863",
    "type": "files",
    "relationships": {
      "files": {
        "links": {
          "related": {
            "meta": {},
            "href": "https://api.osf.io/v2/nodes/kz483/files/osfstorage/5b7ae01425e63c0018e40863/"
          }
        }
      },
      "target": {
        "links": {
          "related": {
            "meta": {
              "type": "node"
            },
            "href": "https://api.osf.io/v2/nodes/kz483/"
          }
        }
      },
      "node": {
        "links": {
          "related": {
            "meta": {},
            "href": "https://api.osf.io/v2/nodes/kz483/"
          }
        },
        "data": {
          "id": "kz483",
          "type": "nodes"
        }
      }
    }
  }
  ]
  }

and, within that directory, the files to download

julia> JSON.print(JSON.parse(String(HTTP.get("https://files.osf.io/v1/resources/kz483/providers/osfstorage/5b7adfc325e63c0019e40676/").body)),2)
{
  "data": [
    {
      "links": {
        "delete": "https://files.osf.io/v1/resources/kz483/providers/osfstorage/5b7adfd384f087001cdb4514",
        "download": "https://files.osf.io/v1/resources/kz483/providers/osfstorage/5b7adfd384f087001cdb4514",
        "upload": "https://files.osf.io/v1/resources/kz483/providers/osfstorage/5b7adfd384f087001cdb4514?kind=file",
        "move": "https://files.osf.io/v1/resources/kz483/providers/osfstorage/5b7adfd384f087001cdb4514"
      },
      "attributes": {
        "path": "/5b7adfd384f087001cdb4514",
        "resource": "kz483",
        "name": "pb2_n0.rda",
        "contentType": "application/octet-stream",
        "etag": "837bdf186ba88e402c0d273a2d9f5c62d3bad67b44bbc97dc4d3501eab01d78a",
        "extra": {
          "downloads": 4,
          "latestVersionSeen": null,
          "checkout": null,
          "hashes": {
            "sha256": "1ceb4c1defde581b0b4cf89725fb665b618d0fab47c900875731480632e60284",
            "md5": "1efd49f796f39427c53719cbc45dd425"
          },
          "guid": null,
          "version": 1
        },
        "modified_utc": "2018-08-20T15:35:47.815296+00:00",
        "created_utc": "2018-08-20T15:35:47.815296+00:00",
        "size": 417405,
        "provider": "osfstorage",
        "kind": "file",
        "modified": "2018-08-20T15:35:47.815296+00:00",
        "sizeInt": 417405,
        "materialized": "/R_Data/pb2_n0.rda"
      },
      "id": "osfstorage/5b7adfd384f087001cdb4514",
      "type": "files"
    },
    {
      "links": {
        "delete": "https://files.osf.io/v1/resources/kz483/providers/osfstorage/5b7adfdb84f087001bdb429d",
        "download": "https://files.osf.io/v1/resources/kz483/providers/osfstorage/5b7adfdb84f087001bdb429d",
        "upload": "https://files.osf.io/v1/resources/kz483/providers/osfstorage/5b7adfdb84f087001bdb429d?kind=file",
        "move": "https://files.osf.io/v1/resources/kz483/providers/osfstorage/5b7adfdb84f087001bdb429d"
      },
      "attributes": {
        "path": "/5b7adfdb84f087001bdb429d",
        "resource": "kz483",
        "name": "pb2_n1.rda",
        "contentType": "application/octet-stream",
        "etag": "1158afb0ab29e41be2eaef74a2d63a8a02fd55d5b1828017dc20461223bed9db",
        "extra": {
          "downloads": 3,
          "latestVersionSeen": null,
          "checkout": null,
          "hashes": {
            "sha256": "ba12a76ded3720182864505095e607e62228ea4d9ed2afedcd3b4771c13e33be",
            "md5": "a58ac2e0a9092feea70f14590146d711"
          },
          "guid": null,
          "version": 1
        },
        "modified_utc": "2018-08-20T15:35:55.705473+00:00",
        "created_utc": "2018-08-20T15:35:55.705473+00:00",
        "size": 359380,
        "provider": "osfstorage",
        "kind": "file",
        "modified": "2018-08-20T15:35:55.705473+00:00",
        "sizeInt": 359380,
        "materialized": "/R_Data/pb2_n1.rda"
      },
      "id": "osfstorage/5b7adfdb84f087001bdb429d",
      "type": "files"
    },
    {
      "links": {
        "delete": "https://files.osf.io/v1/resources/kz483/providers/osfstorage/5b7adfe025e63c0019e406a1",
        "download": "https://files.osf.io/v1/resources/kz483/providers/osfstorage/5b7adfe025e63c0019e406a1",
        "upload": "https://files.osf.io/v1/resources/kz483/providers/osfstorage/5b7adfe025e63c0019e406a1?kind=file",
        "move": "https://files.osf.io/v1/resources/kz483/providers/osfstorage/5b7adfe025e63c0019e406a1"
      },
      "attributes": {
        "path": "/5b7adfe025e63c0019e406a1",
        "resource": "kz483",
        "name": "pb2_n2.rda",
        "contentType": "application/octet-stream",
        "etag": "baba1dbf08c5a0b6c90d233419c8116e88ff998e7136b8feef2ab6b4d2257db8",
        "extra": {
          "downloads": 2,
          "latestVersionSeen": null,
          "checkout": null,
          "hashes": {
            "sha256": "d2de410cfebed370bf4f525512f847991c7ed2a7bc20a71a43aa36f6e689d571",
            "md5": "513c3603111f2ff821ff7f32de53393d"
          },
          "guid": null,
          "version": 1
        },
        "modified_utc": "2018-08-20T15:36:00.347061+00:00",
        "created_utc": "2018-08-20T15:36:00.347061+00:00",
        "size": 326401,
        "provider": "osfstorage",
        "kind": "file",
        "modified": "2018-08-20T15:36:00.347061+00:00",
        "sizeInt": 326401,
        "materialized": "/R_Data/pb2_n2.rda"
      },
      "id": "osfstorage/5b7adfe025e63c0019e406a1",
      "type": "files"
    },
    {
      "links": {
        "delete": "https://files.osf.io/v1/resources/kz483/providers/osfstorage/5b7adfe484f087001cdb4531",
        "download": "https://files.osf.io/v1/resources/kz483/providers/osfstorage/5b7adfe484f087001cdb4531",
        "upload": "https://files.osf.io/v1/resources/kz483/providers/osfstorage/5b7adfe484f087001cdb4531?kind=file",
        "move": "https://files.osf.io/v1/resources/kz483/providers/osfstorage/5b7adfe484f087001cdb4531"
      },
      "attributes": {
        "path": "/5b7adfe484f087001cdb4531",
        "resource": "kz483",
        "name": "typeNgramfreqs.RData",
        "contentType": "application/octet-stream",
        "etag": "a99e130396dd632feb2fd8fc9084336add922bb57b38f0d9bf722112d38c7261",
        "extra": {
          "downloads": 6,
          "latestVersionSeen": null,
          "checkout": null,
          "hashes": {
            "sha256": "f5511bc475d89358f1bcf7cd15d01aee7d5e228b28477c5079115d57a4e9f695",
            "md5": "659538398510833d7c461a49143a0a45"
          },
          "guid": null,
          "version": 1
        },
        "modified_utc": "2018-08-20T15:36:04.354320+00:00",
        "created_utc": "2018-08-20T15:36:04.354320+00:00",
        "size": 11646,
        "provider": "osfstorage",
        "kind": "file",
        "modified": "2018-08-20T15:36:04.35432+00:00",
        "sizeInt": 11646,
        "materialized": "/R_Data/typeNgramfreqs.RData"
      },
      "id": "osfstorage/5b7adfe484f087001cdb4531",
      "type": "files"
    }
  ]
}

The actual download link has a hexadecimal name but the information on what it should be named as a file is available in the attributes.

@oxinabox Can you tell me if you think this would fit in with the DataDepsGenerators scheme and, if so, where I should start?

[oxinabox]

Yes, this look like it should work well.

Take a look at https://github.com/oxinabox/DataDepsGenerators.jl/blob/master/src/APIs/Figshare.jl

Or really anything else from the APIs folder. There are basically a set of methods to overload. To add support for a new API generator.

You can see how all the methods get invoked Here https://github.com/oxinabox/DataDepsGenerators.jl/blob/master/src/generate.jl#L71