search

CCI-Tools / cate

ESA CCI Toolbox (Cate)
MIT License
50 stars 15 forks source link

Cannot open Fire CCI products from ODP #1033

Closed forman closed 2 years ago

forman commented 2 years ago

Please read this carefully before submitting a new issue:

When opening any Fire product from the DATA SOURCES panel without constraints and with caching enabled, user see the error message "cannot reshape array of size 1036800 into shape (240,720,1440)".

When limiting the time range to a single slice, e.g. 2001-01-01, 2001-01-31, it works fine.

Traceback

Traceback (most recent call last):
  File "/opt/conda/envs/xcube/lib/python3.9/site-packages/xcube-0.9.2-py3.9.egg/xcube/core/store/fs/impl/dataset.py", line 205, in write_data
    data.to_zarr(zarr_store,
  File "/opt/conda/envs/xcube/lib/python3.9/site-packages/xarray/core/dataset.py", line 2037, in to_zarr
    return to_zarr(
  File "/opt/conda/envs/xcube/lib/python3.9/site-packages/xarray/backends/api.py", line 1432, in to_zarr
    writes = writer.sync(compute=compute)
  File "/opt/conda/envs/xcube/lib/python3.9/site-packages/xarray/backends/common.py", line 166, in sync
    delayed_store = da.store(
  File "/opt/conda/envs/xcube/lib/python3.9/site-packages/dask/array/core.py", line 1084, in store
    compute_as_if_collection(Array, store_dsk, map_keys, **kwargs)
  File "/opt/conda/envs/xcube/lib/python3.9/site-packages/dask/base.py", line 315, in compute_as_if_collection
    return schedule(dsk2, keys, **kwargs)
  File "/opt/conda/envs/xcube/lib/python3.9/site-packages/dask/threaded.py", line 79, in get
    results = get_async(
  File "/opt/conda/envs/xcube/lib/python3.9/site-packages/dask/local.py", line 507, in get_async
    raise_exception(exc, tb)
  File "/opt/conda/envs/xcube/lib/python3.9/site-packages/dask/local.py", line 315, in reraise
    raise exc
  File "/opt/conda/envs/xcube/lib/python3.9/site-packages/dask/local.py", line 220, in execute_task
    result = _execute_task(task, data)
  File "/opt/conda/envs/xcube/lib/python3.9/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/envs/xcube/lib/python3.9/site-packages/dask/array/core.py", line 116, in getter
    c = np.asarray(c)
  File "/opt/conda/envs/xcube/lib/python3.9/site-packages/numpy/core/_asarray.py", line 102, in asarray
    return array(a, dtype, copy=False, order=order)
  File "/opt/conda/envs/xcube/lib/python3.9/site-packages/xarray/core/indexing.py", line 357, in __array__
    return np.asarray(self.array, dtype=dtype)
  File "/opt/conda/envs/xcube/lib/python3.9/site-packages/numpy/core/_asarray.py", line 102, in asarray
    return array(a, dtype, copy=False, order=order)
  File "/opt/conda/envs/xcube/lib/python3.9/site-packages/xarray/core/indexing.py", line 521, in __array__
    return np.asarray(self.array, dtype=dtype)
  File "/opt/conda/envs/xcube/lib/python3.9/site-packages/numpy/core/_asarray.py", line 102, in asarray
    return array(a, dtype, copy=False, order=order)
  File "/opt/conda/envs/xcube/lib/python3.9/site-packages/xarray/core/indexing.py", line 422, in __array__
    return np.asarray(array[self.key], dtype=None)
  File "/opt/conda/envs/xcube/lib/python3.9/site-packages/xarray/backends/zarr.py", line 73, in __getitem__
    return array[key.tuple]
  File "/opt/conda/envs/xcube/lib/python3.9/site-packages/zarr/core.py", line 673, in __getitem__
    return self.get_basic_selection(selection, fields=fields)
  File "/opt/conda/envs/xcube/lib/python3.9/site-packages/zarr/core.py", line 798, in get_basic_selection
    return self._get_basic_selection_nd(selection=selection, out=out,
  File "/opt/conda/envs/xcube/lib/python3.9/site-packages/zarr/core.py", line 841, in _get_basic_selection_nd
    return self._get_selection(indexer=indexer, out=out, fields=fields)
  File "/opt/conda/envs/xcube/lib/python3.9/site-packages/zarr/core.py", line 1131, in _get_selection
    self._chunk_getitem(chunk_coords, chunk_selection, out, out_selection,
  File "/opt/conda/envs/xcube/lib/python3.9/site-packages/zarr/core.py", line 1813, in _chunk_getitem
    self._process_chunk(out, cdata, chunk_selection, drop_axes,
  File "/opt/conda/envs/xcube/lib/python3.9/site-packages/zarr/core.py", line 1724, in _process_chunk
    chunk = chunk.reshape(self._chunks, order=self._order)
ValueError: cannot reshape array of size 1036800 into shape (240,720,1440)

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/opt/conda/envs/xcube/lib/python3.9/site-packages/cate-3.1.2-py3.9.egg/cate/util/web/jsonrpchandler.py", line 214, in send_service_method_result
    result = future.result()
  File "/opt/conda/envs/xcube/lib/python3.9/concurrent/futures/_base.py", line 438, in result
    return self.__get_result()
  File "/opt/conda/envs/xcube/lib/python3.9/concurrent/futures/_base.py", line 390, in __get_result
    raise self._exception
  File "/opt/conda/envs/xcube/lib/python3.9/concurrent/futures/thread.py", line 52, in run
    result = self.fn(*self.args, **self.kwargs)
  File "/opt/conda/envs/xcube/lib/python3.9/site-packages/cate-3.1.2-py3.9.egg/cate/util/web/jsonrpchandler.py", line 311, in call_service_method
    result = method(*method_params, monitor=monitor)
  File "/opt/conda/envs/xcube/lib/python3.9/site-packages/cate-3.1.2-py3.9.egg/cate/webapi/websocket.py", line 373, in set_workspace_resource
    self.workspace_manager.set_workspace_resource(base_dir,
  File "/opt/conda/envs/xcube/lib/python3.9/site-packages/cate-3.1.2-py3.9.egg/cate/core/wsmanag.py", line 465, in set_workspace_resource
    workspace.execute_workflow(res_name=res_name, monitor=monitor)
  File "/opt/conda/envs/xcube/lib/python3.9/site-packages/cate-3.1.2-py3.9.egg/cate/core/workspace.py", line 668, in execute_workflow
    self.workflow.invoke_steps(steps, context=self._new_context(), monitor=monitor)
  File "/opt/conda/envs/xcube/lib/python3.9/site-packages/cate-3.1.2-py3.9.egg/cate/core/workflow.py", line 627, in invoke_steps
    steps[0].invoke(context=context, monitor=monitor)
  File "/opt/conda/envs/xcube/lib/python3.9/site-packages/cate-3.1.2-py3.9.egg/cate/core/workflow.py", line 318, in invoke
    self._invoke_impl(_new_context(context, step=self), monitor=monitor)
  File "/opt/conda/envs/xcube/lib/python3.9/site-packages/cate-3.1.2-py3.9.egg/cate/core/workflow.py", line 980, in _invoke_impl
    return_value = self._op(monitor=monitor, **input_values)
  File "/opt/conda/envs/xcube/lib/python3.9/site-packages/cate-3.1.2-py3.9.egg/cate/core/op.py", line 216, in __call__
    return_value = self._wrapped_op(**input_values)
  File "/opt/conda/envs/xcube/lib/python3.9/site-packages/cate-3.1.2-py3.9.egg/cate/ops/io.py", line 85, in open_dataset
    ds, ds_id = cate.core.ds.open_dataset(dataset_id=ds_id,
  File "/opt/conda/envs/xcube/lib/python3.9/site-packages/cate-3.1.2-py3.9.egg/cate/core/ds.py", line 457, in open_dataset
    dataset, dataset_id = make_local(data=dataset,
  File "/opt/conda/envs/xcube/lib/python3.9/site-packages/cate-3.1.2-py3.9.egg/cate/core/ds.py", line 488, in make_local
    local_data_id = local_store.write_data(data=data, data_id=local_name)
  File "/opt/conda/envs/xcube/lib/python3.9/site-packages/xcube-0.9.2-py3.9.egg/xcube/core/store/fs/store.py", line 299, in write_data
    writer.write_data(data,
  File "/opt/conda/envs/xcube/lib/python3.9/site-packages/xcube-0.9.2-py3.9.egg/xcube/core/store/fs/impl/dataset.py", line 210, in write_data
    raise DataStoreError(f'Failed to write'
xcube.core.store.error.DataStoreError: Failed to write dataset '/home/xcube/.cate/data_stores/local/local.esacci.FIRE.mon.L4.BA.MODIS.Terra.MODIS_TERRA.v5-1.grid.6.zarr': cannot reshape array of size 1036800 into shape (240,720,1440)
TonioF commented 2 years ago

Closed by https://github.com/dcs4cop/xcube-cci/pull/57