fsspec / gdrivefs

Google drive implementation of fsspec
BSD 2-Clause "Simplified" License
38 stars 20 forks source link

How do I write with a mapper? #3

Closed rabernat closed 5 years ago

rabernat commented 5 years ago

Although writing was implemented in #2, I can't figure out how to use it with zarr.

If, at the end of the example notebook, I run

mapper = gdfs.get_mapper('/woa_t_an_COPY.zarr/')
dsl.to_zarr(mapper)

I get the error:

---------------------------------------------------------------------------
FileNotFoundError                         Traceback (most recent call last)
<ipython-input-7-850afca92d3b> in <module>
      1 mapper = gdfs.get_mapper('/woa_t_an_COPY.zarr/')
----> 2 dsl.to_zarr(mapper)

/srv/conda/envs/notebook/lib/python3.7/site-packages/xarray/core/dataset.py in to_zarr(self, store, mode, synchronizer, group, encoding, compute, consolidated, append_dim)
   1614             compute=compute,
   1615             consolidated=consolidated,
-> 1616             append_dim=append_dim,
   1617         )
   1618 

/srv/conda/envs/notebook/lib/python3.7/site-packages/xarray/backends/api.py in to_zarr(dataset, store, mode, synchronizer, group, encoding, compute, consolidated, append_dim)
   1317         synchronizer=synchronizer,
   1318         group=group,
-> 1319         consolidate_on_close=consolidated,
   1320     )
   1321     zstore.append_dim = append_dim

/srv/conda/envs/notebook/lib/python3.7/site-packages/xarray/backends/zarr.py in open_group(cls, store, mode, synchronizer, group, consolidated, consolidate_on_close)
    258             zarr_group = zarr.open_consolidated(store, **open_kwargs)
    259         else:
--> 260             zarr_group = zarr.open_group(store, **open_kwargs)
    261         return cls(zarr_group, consolidate_on_close)
    262 

/srv/conda/envs/notebook/lib/python3.7/site-packages/zarr/hierarchy.py in open_group(store, mode, cache_attrs, synchronizer, path, chunk_store)
   1131             err_contains_group(path)
   1132         else:
-> 1133             init_group(store, path=path, chunk_store=chunk_store)
   1134 
   1135     # determine read only status

/srv/conda/envs/notebook/lib/python3.7/site-packages/zarr/storage.py in init_group(store, overwrite, path, chunk_store)
    430     # initialise metadata
    431     _init_group_metadata(store=store, overwrite=overwrite, path=path,
--> 432                          chunk_store=chunk_store)
    433 
    434 

/srv/conda/envs/notebook/lib/python3.7/site-packages/zarr/storage.py in _init_group_metadata(store, overwrite, path, chunk_store)
    451     meta = dict()
    452     key = _path_to_prefix(path) + group_meta_key
--> 453     store[key] = encode_group_metadata(meta)
    454 
    455 

/srv/conda/envs/notebook/lib/python3.7/site-packages/fsspec/mapping.py in __setitem__(self, key, value)
     94         self.fs.mkdirs(self.fs._parent(key), exist_ok=True)
     95         with self.fs.open(key, "wb") as f:
---> 96             f.write(value)
     97 
     98     def keys(self):

/srv/conda/envs/notebook/lib/python3.7/site-packages/fsspec/spec.py in __exit__(self, *args)
   1159 
   1160     def __exit__(self, *args):
-> 1161         self.close()

/srv/conda/envs/notebook/lib/python3.7/site-packages/fsspec/spec.py in close(self)
   1127         else:
   1128             if not self.forced:
-> 1129                 self.flush(force=True)
   1130 
   1131             if self.fs is not None:

/srv/conda/envs/notebook/lib/python3.7/site-packages/fsspec/spec.py in flush(self, force)
   1002             # Initialize a multipart upload
   1003             self.offset = 0
-> 1004             self._initiate_upload()
   1005 
   1006         if self._upload_chunk(final=force) is not False:

~/gdrivefs/core.py in _initiate_upload(self)
    311     def _initiate_upload(self):
    312         """ Create multi-upload """
--> 313         parent_id = self.fs.path_to_file_id(self.fs._parent(self.path))
    314         head = {"Content-Type": "application/json; charset=UTF-8"}
    315         # also allows description, MIME type, version, thumbnail...

~/gdrivefs/core.py in path_to_file_id(self, path, parent, trashed)
    181             parent = self.root_file_id
    182         top_file_id = self._get_directory_child_by_name(items[0], parent,
--> 183                                                         trashed=trashed)
    184         if len(items) == 1:
    185             return top_file_id

~/gdrivefs/core.py in _get_directory_child_by_name(self, child_name, directory_file_id, trashed)
    199         if len(possible_children) == 0:
    200             raise FileNotFoundError(
--> 201                 f'Directory {directory_file_id} has no child '
    202                 f'named {child_name}')
    203         if len(possible_children) == 1:

FileNotFoundError: Directory 1FQzXM2E28WF6fV7vy1K7HdxNV-w6z_Wx has no child named woa_t_an_COPY.zarr

We need to be creating a new directory somehow.

martindurant commented 5 years ago

Ah, need to define makedirs. Bear with me!