iterative / mlem

🐶 A tool to package, serve, and deploy any ML model on any platform. Archived to be resurrected one day🤞
https://mlem.ai
Apache License 2.0
716 stars 44 forks source link

IsADirectoryError: problems with re-saving MLEM model #576

Open aguschin opened 1 year ago

aguschin commented 1 year ago

Let's save something that will create bug/ directory:

from mlem.core.objects import MlemModel, ModelAnalyzer

model = MlemModel()
model.add_processor("fx", ModelAnalyzer.analyze(lambda x: x, sample_data="anything"))
model.call_orders["fx"] = [("fx", "__call__")]

model.dump("bug")

and then let's save something that wants to write file called bug:

from mlem.api import save, load_meta

def f(x):
    return x

save(f, "bug", sample_data=1)

and we get:

---------------------------------------------------------------------------
IsADirectoryError                         Traceback (most recent call last)
Cell In[20], line 8
      4 def f(x):
      5     return x
----> 8 save(f, "bug", sample_data=1)

File ~/Git/iterative/mlem/mlem/telemetry.py:50, in api_telemetry.<locals>.inner(*args, **kwargs)
     48 with telemetry.event_scope("api", f.__name__) as event:
     49     try:
---> 50         return f(*args, **kwargs)
     51     except Exception as exc:
     52         event.error = exc.__class__.__name__

File ~/Git/iterative/mlem/mlem/core/metadata.py:131, in save(obj, path, project, sample_data, fs, params, preprocess, postprocess)
    129 log_meta_params(meta, add_object_type=True)
    130 path = os.fspath(path)
--> 131 meta.dump(path, fs=fs, project=project)
    132 return meta

File ~/Git/iterative/mlem/mlem/core/objects.py:523, in _WithArtifacts.dump(self, path, fs, project)
    521 except (MlemObjectNotFound, FileNotFoundError, ValidationError):
    522     pass
--> 523 self.artifacts = self.get_artifacts()
    524 self._write_meta(location)
    525 return self

File ~/Git/iterative/mlem/mlem/core/objects.py:588, in _WithArtifacts.get_artifacts(self)
    586 def get_artifacts(self):
    587     if self.artifacts is None:
--> 588         return self.write_value()
    589     return {
    590         name: a.relative_to(self.loc)
    591         if isinstance(a, PlaceholderArtifact)
    592         else a
    593         for name, a in self.artifacts.items()
    594     }

File ~/Git/iterative/mlem/mlem/core/objects.py:807, in MlemModel.write_value(self)
    805 if self.is_single_model:
    806     if self.model_type.model is not None:
--> 807         return self.model_type.io.dump(
    808             self.storage,
    809             posixpath.basename(self.name),
    810             self.model_type.model,
    811         )
    812     raise ValueError("Meta is not binded to actual model")
    813 artifacts = {}

File ~/Git/iterative/mlem/mlem/contrib/callable.py:45, in PickleModelIO.dump(self, storage, path, model)
     43 arts = {}
     44 if len(refs) == 0:
---> 45     with storage.open(path) as (f, art):
     46         f.write(model_blob)
     47         return {self.file_name: art}

File ~/.pyenv/versions/3.9.5/lib/python3.9/contextlib.py:117, in _GeneratorContextManager.__enter__(self)
    115 del self.args, self.kwds, self.func
    116 try:
--> 117     return next(self.gen)
    118 except StopIteration:
    119     raise RuntimeError("generator didn't yield") from None

File ~/Git/iterative/mlem/mlem/core/artifacts.py:309, in LocalStorage.open(self, path)
    307 @contextlib.contextmanager
    308 def open(self, path) -> Iterator[Tuple[IO, "LocalArtifact"]]:
--> 309     with super().open(path) as (io, art):
    310         local_art = LocalArtifact(uri=path, size=-1, hash="")
    311         yield io, local_art

File ~/.pyenv/versions/3.9.5/lib/python3.9/contextlib.py:117, in _GeneratorContextManager.__enter__(self)
    115 del self.args, self.kwds, self.func
    116 try:
--> 117     return next(self.gen)
    118 except StopIteration:
    119     raise RuntimeError("generator didn't yield") from None

File ~/Git/iterative/mlem/mlem/core/artifacts.py:241, in FSSpecStorage.open(self, path)
    239 fs.makedirs(posixpath.dirname(fullpath), exist_ok=True)
    240 art = FSSpecArtifact(uri=(self.create_uri(path)), size=-1, hash="")
--> 241 with fs.open(fullpath, "wb") as f:
    242     yield f, art
    243 file_info = get_file_info(fullpath, fs)

File ~/.local/share/virtualenvs/mlem-Utz6DvOn/lib/python3.9/site-packages/fsspec/spec.py:1106, in AbstractFileSystem.open(self, path, mode, block_size, cache_options, compression, **kwargs)
   1104 else:
   1105     ac = kwargs.pop("autocommit", not self._intrans)
-> 1106     f = self._open(
   1107         path,
   1108         mode=mode,
   1109         block_size=block_size,
   1110         autocommit=ac,
   1111         cache_options=cache_options,
   1112         **kwargs,
   1113     )
   1114     if compression is not None:
   1115         from fsspec.compression import compr

File ~/.local/share/virtualenvs/mlem-Utz6DvOn/lib/python3.9/site-packages/fsspec/implementations/local.py:175, in LocalFileSystem._open(self, path, mode, block_size, **kwargs)
    173 if self.auto_mkdir and "w" in mode:
    174     self.makedirs(self._parent(path), exist_ok=True)
--> 175 return LocalFileOpener(path, mode, fs=self, **kwargs)

File ~/.local/share/virtualenvs/mlem-Utz6DvOn/lib/python3.9/site-packages/fsspec/implementations/local.py:273, in LocalFileOpener.__init__(self, path, mode, autocommit, fs, compression, **kwargs)
    271 self.compression = get_compression(path, compression)
    272 self.blocksize = io.DEFAULT_BUFFER_SIZE
--> 273 self._open()

File ~/.local/share/virtualenvs/mlem-Utz6DvOn/lib/python3.9/site-packages/fsspec/implementations/local.py:278, in LocalFileOpener._open(self)
    276 if self.f is None or self.f.closed:
    277     if self.autocommit or "w" not in self.mode:
--> 278         self.f = open(self.path, mode=self.mode)
    279         if self.compression:
    280             compress = compr[self.compression]

IsADirectoryError: [Errno 21] Is a directory: '/Users/aguschin/Git/iterative/mlem/hat-player/bug'
mike0sv commented 1 year ago

Please check if that fixed it #526