AWS Lambda deployment - service is deployed but fails to launch

[adamwawrzynski]

Describe the bug After model deployment service is not responsive due to error.

To Reproduce

1. Follow steps in aws-lambda-deplyoment repository: https://github.com/bentoml/aws-lambda-deploy.
2. Test in AWS Lambda console.

Expected behavior

Screenshots/Logs

AWS Lambda logs after testing:

{
  "errorMessage": "operation failed, [Errno 30] Read-only file system: b'/home/bentoml/bento/models/iris_clf/latest'",
  "errorType": "OperationFailed",
  "requestId": "",
  "stackTrace": [
    "  File \"/usr/local/lib/python3.10/importlib/__init__.py\", line 126, in import_module\n    return _bootstrap._gcd_import(name[level:], package, level)\n",
    "  File \"<frozen importlib._bootstrap>\", line 1050, in _gcd_import\n",
    "  File \"<frozen importlib._bootstrap>\", line 1027, in _find_and_load\n",
    "  File \"<frozen importlib._bootstrap>\", line 1006, in _find_and_load_unlocked\n",
    "  File \"<frozen importlib._bootstrap>\", line 688, in _load_unlocked\n",
    "  File \"<frozen importlib._bootstrap_external>\", line 883, in exec_module\n",
    "  File \"<frozen importlib._bootstrap>\", line 241, in _call_with_frames_removed\n",
    "  File \"/home/bentoml/bento/app.py\", line 8, in <module>\n    bento_service = load(\"./\")\n",
    "  File \"/usr/local/lib/python3.10/site-packages/bentoml/_internal/service/loader.py\", line 336, in load\n    svc = load_bento_dir(bento_path, standalone_load=standalone_load)\n",
    "  File \"/usr/local/lib/python3.10/site-packages/bentoml/_internal/service/loader.py\", line 236, in load_bento_dir\n    return _load_bento(bento, standalone_load)\n",
    "  File \"/usr/local/lib/python3.10/site-packages/bentoml/_internal/service/loader.py\", line 254, in _load_bento\n    svc = import_service(\n",
    "  File \"/usr/local/lib/python3.10/site-packages/simple_di/__init__.py\", line 139, in _\n    return func(*_inject_args(bind.args), **_inject_kwargs(bind.kwargs))\n",
    "  File \"/usr/local/lib/python3.10/site-packages/bentoml/_internal/service/loader.py\", line 137, in import_service\n    module = importlib.import_module(module_name, package=working_dir)\n",
    "  File \"/usr/local/lib/python3.10/importlib/__init__.py\", line 126, in import_module\n    return _bootstrap._gcd_import(name[level:], package, level)\n",
    "  File \"<frozen importlib._bootstrap>\", line 1050, in _gcd_import\n",
    "  File \"<frozen importlib._bootstrap>\", line 1027, in _find_and_load\n",
    "  File \"<frozen importlib._bootstrap>\", line 1006, in _find_and_load_unlocked\n",
    "  File \"<frozen importlib._bootstrap>\", line 688, in _load_unlocked\n",
    "  File \"<frozen importlib._bootstrap_external>\", line 883, in exec_module\n",
    "  File \"<frozen importlib._bootstrap>\", line 241, in _call_with_frames_removed\n",
    "  File \"/home/bentoml/bento/src/service.py\", line 5, in <module>\n    iris_clf_runner = bentoml.sklearn.get(\"iris_clf:latest\").to_runner()\n",
    "  File \"/usr/local/lib/python3.10/site-packages/bentoml/_internal/frameworks/sklearn.py\", line 50, in get\n    model = bentoml.models.get(tag_like)\n",
    "  File \"/usr/local/lib/python3.10/site-packages/simple_di/__init__.py\", line 139, in _\n    return func(*_inject_args(bind.args), **_inject_kwargs(bind.kwargs))\n",
    "  File \"/usr/local/lib/python3.10/site-packages/bentoml/models.py\", line 45, in get\n    return _model_store.get(tag)\n",
    "  File \"/usr/local/lib/python3.10/site-packages/bentoml/_internal/store.py\", line 149, in get\n    self._recreate_latest(_tag)\n",
    "  File \"/usr/local/lib/python3.10/site-packages/bentoml/_internal/store.py\", line 132, in _recreate_latest\n    with self._fs.open(tag.latest_path(), \"w\") as latest_file:\n",
    "  File \"/usr/local/lib/python3.10/site-packages/fs/wrapfs.py\", line 472, in open\n    open_file = _fs.open(\n",
    "  File \"/usr/local/lib/python3.10/site-packages/fs/osfs.py\", line 643, in open\n    with convert_os_errors(\"open\", path):\n",
    "  File \"/usr/local/lib/python3.10/site-packages/fs/error_tools.py\", line 89, in __exit__\n    reraise(fserror, fserror(self._path, exc=exc_value), traceback)\n",
    "  File \"/usr/local/lib/python3.10/site-packages/six.py\", line 718, in reraise\n    raise value.with_traceback(tb)\n",
    "  File \"/usr/local/lib/python3.10/site-packages/fs/osfs.py\", line 647, in open\n    return io.open(\n"
  ]
}

Environment:

• OS: [e.g. MacOS 10.14.3]: Ubuntu 22.04.01
• Python Version [e.g. Python 3.7.1]: 3.10.12
• BentoML Version [e.g. BentoML-0.8.6]: 1.1.11
• Bentoctl Version [e.g. bentoctl-0.1]: 0.4.0

Additional context

[amustafi1986]

I am also facing the same issue while trying to test model deploymnet in lamda. I also used the sample model in the bentoml project to check, and the same error is coming up:

{"errorMessage": "operation failed, [Errno 30] Read-only file system: b'/home/bentoml/bento/models/iris_clf/latest'", "errorType": "OperationFailed", "requestId": "", "stackTrace": [" File \"/usr/local/lib/python3.11/importlib/init.py\", line 126, in import_module return _bootstrap._gcd_import(name[level:], package, level)", " File \"\", line 1204, in _gcd_import", " File \"\", line 1176, in _find_and_load", " File \"\", line 1147, in _find_and_load_unlocked", " File \"\", line 690, in _load_unlocked", " File \"\", line 940, in exec_module", " File \"\", line 241, in _call_with_frames_removed", " File \"/home/bentoml/bento/app.py\", line 8, in bento_service = load(\"./\")", " File \"/usr/local/lib/python3.11/site-packages/bentoml/_internal/service/loader.py\", line 336, in load svc = load_bento_dir(bento_path, standalone_load=standalone_load)", " File \"/usr/local/lib/python3.11/site-packages/bentoml/_internal/service/loader.py\", line 236, in load_bento_dir return _load_bento(bento, standalone_load)", " File \"/usr/local/lib/python3.11/site-packages/bentoml/_internal/service/loader.py\", line 254, in _load_bento svc = import_service(\n", " File \"/usr/local/lib/python3.11/site-packages/simpledi/init.py\", line 139, in return func(*_inject_args(bind.args), *_inject_kwargs(bind.kwargs))", " File \"/usr/local/lib/python3.11/site-packages/bentoml/_internal/service/loader.py\", line 137, in import_service module = importlib.import_module(module_name, package=working_dir)", " File \"/usr/local/lib/python3.11/importlib/init.py\", line 126, in import_module return _bootstrap._gcd_import(name[level:], package, level)", " File \"\", line 1204, in _gcd_import", " File \"\", line 1176, in _find_and_load", " File \"\", line 1147, in _find_and_load_unlocked", " File \"\", line 690, in _load_unlocked", " File \"\", line 940, in exec_module", " File \"\", line 241, in _call_with_frames_removed", " File \"/home/bentoml/bento/src/iris_classifier.py\", line 7, in iris_clf_runner = bentoml.sklearn.get(\"iris_clf:latest\").to_runner()", " File \"/usr/local/lib/python3.11/site-packages/bentoml/_internal/frameworks/sklearn.py\", line 50, in get model = bentoml.models.get(tag_like)", " File \"/usr/local/lib/python3.11/site-packages/simpledi/init.py\", line 139, in return func(_inject_args(bind.args), **_inject_kwargs(bind.kwargs))", " File \"/usr/local/lib/python3.11/site-packages/bentoml/models.py\", line 45, in get return _model_store.get(tag)", " File \"/usr/local/lib/python3.11/site-packages/bentoml/_internal/store.py\", line 149, in get self._recreate_latest(_tag)\n", " File \"/usr/local/lib/python3.11/site-packages/bentoml/_internal/store.py\", line 132, in _recreate_latest with self._fs.open(tag.latest_path(), \"w\") as latest_file: ", " File \"/usr/local/lib/python3.11/site-packages/fs/wrapfs.py\", line 472, in open open_file = _fs.open(\n", " File \"/usr/local/lib/python3.11/site-packages/fs/osfs.py\", line 643, in open with convert_os_errors(\"open\", path): ", " File \"/usr/local/lib/python3.11/site-packages/fs/error_tools.py\", line 89, in exit reraise(fserror, fserror(self._path, exc=exc_value), traceback)\n", " File \"/usr/local/lib/python3.11/site-packages/six.py\", line 718, in reraise raise value.with_traceback(tb)\n", " File \"/usr/local/lib/python3.11/site-packages/fs/osfs.py\", line 647, in open return io.open(\n"]}