anitaokoh
commented
10 months ago After changes were done on the repo,
I used the pip config below @blythed
pip install git+https://github.com/SuperDuperDB/superduperdb.git@main
Please know that I used the super-duper notebook environment to test it. The code is the same apart from the pip installation I got a different error. Below is the traceback
2023-Nov-28 17:46:00.13| ERROR | jupyter-1e14c797-2de3c6-2d45f0-2db499-2d86995704ee2c| 01120d03-5aed-495c-9b57-cf9565ad7531| superduperdb.base.exceptions:25 | Error while adding object with id: pymongo-docs-all-MiniLM-L6-v2
Traceback (most recent call last):
File "/home/superduper/.local/lib/python3.11/site-packages/superduperdb/base/datalayer.py", line 914, in _add
object.post_create(self)
│ │ └ <superduperdb.base.datalayer.Datalayer object at 0x7fd6083f3d90>
│ └ <function VectorIndex.post_create at 0x7fd608e49940>
└ VectorIndex(identifier='pymongo-docs-all-MiniLM-L6-v2', indexing_listener=Listener(key='details', model=Model(identifier='all...
File "/home/superduper/.local/lib/python3.11/site-packages/superduperdb/components/vector_index.py", line 49, in post_create
create(self)
│ └ VectorIndex(identifier='pymongo-docs-all-MiniLM-L6-v2', indexing_listener=Listener(key='details', model=Model(identifier='all...
└ <bound method MongoDataBackend.create_vector_index of <superduperdb.backends.mongodb.data_backend.MongoDataBackend object at ...
File "/home/superduper/.local/lib/python3.11/site-packages/superduperdb/backends/mongodb/data_backend.py", line 169, in create_vector_index
self.db.command(index_definition)
│ │ └ {'createSearchIndexes': 'customer_details', 'indexes': [{'name': 'pymongo-docs-all-MiniLM-L6-v2', 'definition': {'mappings': ...
│ └ <property object at 0x7fd60c826bb0>
└ <superduperdb.backends.mongodb.data_backend.MongoDataBackend object at 0x7fd6084abc90>
File "/home/superduper/.local/lib/python3.11/site-packages/pymongo/_csot.py", line 107, in csot_wrapper
return func(self, *args, **kwargs)
│ │ │ └ {}
│ │ └ ({'createSearchIndexes': 'customer_details', 'indexes': [{'name': 'pymongo-docs-all-MiniLM-L6-v2', 'definition': {'mappings':...
│ └ Database(MongoClient(host=['ac-ntz1rjd-shard-00-02.tgtxnux.mongodb.net:27017', 'ac-ntz1rjd-shard-00-01.tgtxnux.mongodb.net:27...
└ <function Database.command at 0x7fd64baa9440>
File "/home/superduper/.local/lib/python3.11/site-packages/pymongo/database.py", line 894, in command
return self._command(
│ └ <function Database._command at 0x7fd64baa91c0>
└ Database(MongoClient(host=['ac-ntz1rjd-shard-00-02.tgtxnux.mongodb.net:27017', 'ac-ntz1rjd-shard-00-01.tgtxnux.mongodb.net:27...
File "/home/superduper/.local/lib/python3.11/site-packages/pymongo/database.py", line 743, in _command
return conn.command(
│ └ <function _handle_reauth.<locals>.inner at 0x7fd64b14be20>
└ Connection(<ssl.SSLSocket fd=60, family=2, type=1, proto=6, laddr=('10.0.3.25', 44096), raddr=('52.59.36.30', 27017)>) at 140...
File "/home/superduper/.local/lib/python3.11/site-packages/pymongo/helpers.py", line 322, in inner
return func(*args, **kwargs)
│ │ └ {'write_concern': None, 'parse_write_concern_error': False, 'session': <pymongo.client_session.ClientSession object at 0x7fd5...
│ └ (Connection(<ssl.SSLSocket fd=60, family=2, type=1, proto=6, laddr=('10.0.3.25', 44096), raddr=('52.59.36.30', 27017)>) at 14...
└ <function Connection.command at 0x7fd64b14bd80>
File "/home/superduper/.local/lib/python3.11/site-packages/pymongo/pool.py", line 968, in command
return command(
└ <function command at 0x7fd64baabec0>
File "/home/superduper/.local/lib/python3.11/site-packages/pymongo/network.py", line 192, in command
helpers._check_command_response(
│ └ <function _check_command_response at 0x7fd64bba7380>
└ <module 'pymongo.helpers' from '/home/superduper/.local/lib/python3.11/site-packages/pymongo/helpers.py'>
File "/home/superduper/.local/lib/python3.11/site-packages/pymongo/helpers.py", line 230, in _check_command_response
raise OperationFailure(errmsg, code, response, max_wire_version)
│ │ │ │ └ 17
│ │ │ └ {'ok': 0, 'errmsg': 'command not found', 'code': 59, 'codeName': 'CommandNotFound'}
│ │ └ 59
│ └ 'command not found'
└ <class 'pymongo.errors.OperationFailure'>
pymongo.errors.OperationFailure: command not found, full error: {'ok': 0, 'errmsg': 'command not found', 'code': 59, 'codeName': 'CommandNotFound'}
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "<frozen runpy>", line 198, in _run_module_as_main
File "<frozen runpy>", line 88, in _run_code
File "/usr/local/lib/python3.11/site-packages/ipykernel_launcher.py", line 17, in <module>
app.launch_new_instance()
│ └ <bound method Application.launch_instance of <class 'ipykernel.kernelapp.IPKernelApp'>>
└ <module 'ipykernel.kernelapp' from '/usr/local/lib/python3.11/site-packages/ipykernel/kernelapp.py'>
File "/usr/local/lib/python3.11/site-packages/traitlets/config/application.py", line 1046, in launch_instance
app.start()
│ └ <function IPKernelApp.start at 0x7fd6738d7920>
└ <ipykernel.kernelapp.IPKernelApp object at 0x7fd6766b5f10>
File "/usr/local/lib/python3.11/site-packages/ipykernel/kernelapp.py", line 736, in start
self.io_loop.start()
│ │ └ <function BaseAsyncIOLoop.start at 0x7fd67537fce0>
│ └ <tornado.platform.asyncio.AsyncIOMainLoop object at 0x7fd67390a390>
└ <ipykernel.kernelapp.IPKernelApp object at 0x7fd6766b5f10>
File "/usr/local/lib/python3.11/site-packages/tornado/platform/asyncio.py", line 195, in start
self.asyncio_loop.run_forever()
│ │ └ <function BaseEventLoop.run_forever at 0x7fd675c95c60>
│ └ <_UnixSelectorEventLoop running=True closed=False debug=False>
└ <tornado.platform.asyncio.AsyncIOMainLoop object at 0x7fd67390a390>
File "/usr/local/lib/python3.11/asyncio/base_events.py", line 607, in run_forever
self._run_once()
│ └ <function BaseEventLoop._run_once at 0x7fd675c97a60>
└ <_UnixSelectorEventLoop running=True closed=False debug=False>
File "/usr/local/lib/python3.11/asyncio/base_events.py", line 1922, in _run_once
handle._run()
│ └ <function Handle._run at 0x7fd6762289a0>
└ <Handle Task.task_wakeup(<Future finis...dd0>, ...],))>)>
File "/usr/local/lib/python3.11/asyncio/events.py", line 80, in _run
self._context.run(self._callback, *self._args)
│ │ │ │ │ └ <member '_args' of 'Handle' objects>
│ │ │ │ └ <Handle Task.task_wakeup(<Future finis...dd0>, ...],))>)>
│ │ │ └ <member '_callback' of 'Handle' objects>
│ │ └ <Handle Task.task_wakeup(<Future finis...dd0>, ...],))>)>
│ └ <member '_context' of 'Handle' objects>
└ <Handle Task.task_wakeup(<Future finis...dd0>, ...],))>)>
File "/usr/local/lib/python3.11/site-packages/ipykernel/kernelbase.py", line 516, in dispatch_queue
await self.process_one()
│ └ <function Kernel.process_one at 0x7fd674132fc0>
└ <ipykernel.ipkernel.IPythonKernel object at 0x7fd673901f10>
File "/usr/local/lib/python3.11/site-packages/ipykernel/kernelbase.py", line 505, in process_one
await dispatch(*args)
│ └ ([<zmq.sugar.frame.Frame object at 0x7fd673945610>, <zmq.sugar.frame.Frame object at 0x7fd6739456d0>, <zmq.sugar.frame.Frame ...
└ <bound method Kernel.dispatch_shell of <ipykernel.ipkernel.IPythonKernel object at 0x7fd673901f10>>
File "/usr/local/lib/python3.11/site-packages/ipykernel/kernelbase.py", line 412, in dispatch_shell
await result
└ <coroutine object Kernel.execute_request at 0x7fd673ca1220>
File "/usr/local/lib/python3.11/site-packages/ipykernel/kernelbase.py", line 740, in execute_request
reply_content = await reply_content
└ <coroutine object IPythonKernel.do_execute at 0x7fd60c8d5e40>
File "/usr/local/lib/python3.11/site-packages/ipykernel/ipkernel.py", line 422, in do_execute
res = shell.run_cell(
│ └ <function ZMQInteractiveShell.run_cell at 0x7fd6738d4d60>
└ <ipykernel.zmqshell.ZMQInteractiveShell object at 0x7fd673900e50>
File "/usr/local/lib/python3.11/site-packages/ipykernel/zmqshell.py", line 546, in run_cell
return super().run_cell(*args, **kwargs)
│ └ {'store_history': True, 'silent': False, 'cell_id': '46ce7a59-cdd2-46e5-a218-77ef73df7a95'}
└ ("from superduperdb import Listener, VectorIndex\n\ndb.add(\n VectorIndex(\n identifier=f'pymongo-docs-{model.ident...
File "/usr/local/lib/python3.11/site-packages/IPython/core/interactiveshell.py", line 3024, in run_cell
result = self._run_cell(
│ └ <function InteractiveShell._run_cell at 0x7fd674bb6a20>
└ <ipykernel.zmqshell.ZMQInteractiveShell object at 0x7fd673900e50>
File "/usr/local/lib/python3.11/site-packages/IPython/core/interactiveshell.py", line 3079, in _run_cell
result = runner(coro)
│ └ <coroutine object InteractiveShell.run_cell_async at 0x7fd5355f4520>
└ <function _pseudo_sync_runner at 0x7fd674ba1bc0>
File "/usr/local/lib/python3.11/site-packages/IPython/core/async_helpers.py", line 129, in _pseudo_sync_runner
coro.send(None)
│ └ <method 'send' of 'coroutine' objects>
└ <coroutine object InteractiveShell.run_cell_async at 0x7fd5355f4520>
File "/usr/local/lib/python3.11/site-packages/IPython/core/interactiveshell.py", line 3284, in run_cell_async
has_raised = await self.run_ast_nodes(code_ast.body, cell_name,
│ │ │ │ └ '/tmp/ipykernel_781/640084429.py'
│ │ │ └ [<ast.ImportFrom object at 0x7fd60f385810>, <ast.Expr object at 0x7fd60f385060>, <ast.Expr object at 0x7fd60f385750>]
│ │ └ <ast.Module object at 0x7fd60f387d90>
│ └ <function InteractiveShell.run_ast_nodes at 0x7fd674bb6d40>
└ <ipykernel.zmqshell.ZMQInteractiveShell object at 0x7fd673900e50>
File "/usr/local/lib/python3.11/site-packages/IPython/core/interactiveshell.py", line 3466, in run_ast_nodes
if await self.run_code(code, result, async_=asy):
│ │ │ │ └ False
│ │ │ └ <ExecutionResult object at 7fd608418e10, execution_count=8 error_before_exec=None error_in_exec=Error while adding object wit...
│ │ └ <code object <module> at 0x7fd53be268e0, file "/tmp/ipykernel_781/640084429.py", line 1>
│ └ <function InteractiveShell.run_code at 0x7fd674bb6de0>
└ <ipykernel.zmqshell.ZMQInteractiveShell object at 0x7fd673900e50>
> File "/usr/local/lib/python3.11/site-packages/IPython/core/interactiveshell.py", line 3526, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
│ │ │ │ └ {'__name__': '__main__', '__doc__': 'Automatically created module for IPython interactive environment', '__package__': None, ...
│ │ │ └ <ipykernel.zmqshell.ZMQInteractiveShell object at 0x7fd673900e50>
│ │ └ <property object at 0x7fd674b9e570>
│ └ <ipykernel.zmqshell.ZMQInteractiveShell object at 0x7fd673900e50>
└ <code object <module> at 0x7fd53be268e0, file "/tmp/ipykernel_781/640084429.py", line 1>
File "/tmp/ipykernel_781/640084429.py", line 3, in <module>
db.add(
│ └ <function Datalayer.add at 0x7fd608e37e20>
└ <superduperdb.base.datalayer.Datalayer object at 0x7fd6083f3d90>
File "/home/superduper/.local/lib/python3.11/site-packages/superduperdb/base/datalayer.py", line 556, in add
return self._add(object=object, dependencies=dependencies)
│ │ │ └ ()
│ │ └ VectorIndex(identifier='pymongo-docs-all-MiniLM-L6-v2', indexing_listener=Listener(key='details', model=Model(identifier='all...
│ └ <function Datalayer._add at 0x7fd608e482c0>
└ <superduperdb.base.datalayer.Datalayer object at 0x7fd6083f3d90>
File "/home/superduper/.local/lib/python3.11/site-packages/superduperdb/base/datalayer.py", line 918, in _add
raise exceptions.DatalayerException(
│ └ <class 'superduperdb.base.exceptions.DatalayerException'>
└ <module 'superduperdb.base.exceptions' from '/home/superduper/.local/lib/python3.11/site-packages/superduperdb/base/exception...
superduperdb.base.exceptions.DatalayerException: Error while adding object with id: pymongo-docs-all-MiniLM-L6-v2
---------------------------------------------------------------------------
OperationFailure Traceback (most recent call last)
File ~/.local/lib/python3.11/site-packages/superduperdb/base/datalayer.py:914, in Datalayer._add(self, object, dependencies, serialized, parent)
912 self.metadata.create_parent_child(parent, object.unique_id)
--> 914 object.post_create(self)
915 object.on_load(self)
File ~/.local/lib/python3.11/site-packages/superduperdb/components/vector_index.py:49, in VectorIndex.post_create(self, db)
48 raise ValueError(msg)
---> 49 create(self)
File ~/.local/lib/python3.11/site-packages/superduperdb/backends/mongodb/data_backend.py:169, in MongoDataBackend.create_vector_index(self, vector_index, dry_run)
168 if not dry_run:
--> 169 self.db.command(index_definition)
File ~/.local/lib/python3.11/site-packages/pymongo/_csot.py:107, in apply.<locals>.csot_wrapper(self, *args, **kwargs)
106 return func(self, *args, **kwargs)
--> 107 return func(self, *args, **kwargs)
File ~/.local/lib/python3.11/site-packages/pymongo/database.py:894, in Database.command(self, command, value, check, allowable_errors, read_preference, codec_options, session, comment, **kwargs)
890 with self.__client._conn_for_reads(read_preference, session) as (
891 connection,
892 read_preference,
893 ):
--> 894 return self._command(
895 connection,
896 command,
897 value,
898 check,
899 allowable_errors,
900 read_preference,
901 opts,
902 session=session,
903 **kwargs,
904 )
File ~/.local/lib/python3.11/site-packages/pymongo/database.py:743, in Database._command(self, conn, command, value, check, allowable_errors, read_preference, codec_options, write_concern, parse_write_concern_error, session, **kwargs)
742 with self.__client._tmp_session(session) as s:
--> 743 return conn.command(
744 self.__name,
745 command,
746 read_preference,
747 codec_options,
748 check,
749 allowable_errors,
750 write_concern=write_concern,
751 parse_write_concern_error=parse_write_concern_error,
752 session=s,
753 client=self.__client,
754 )
File ~/.local/lib/python3.11/site-packages/pymongo/helpers.py:322, in _handle_reauth.<locals>.inner(*args, **kwargs)
321 try:
--> 322 return func(*args, **kwargs)
323 except OperationFailure as exc:
File ~/.local/lib/python3.11/site-packages/pymongo/pool.py:968, in Connection.command(self, dbname, spec, read_preference, codec_options, check, allowable_errors, read_concern, write_concern, parse_write_concern_error, collation, session, client, retryable_write, publish_events, user_fields, exhaust_allowed)
967 try:
--> 968 return command(
969 self,
970 dbname,
971 spec,
972 self.is_mongos,
973 read_preference,
974 codec_options,
975 session,
976 client,
977 check,
978 allowable_errors,
979 self.address,
980 listeners,
981 self.max_bson_size,
982 read_concern,
983 parse_write_concern_error=parse_write_concern_error,
984 collation=collation,
985 compression_ctx=self.compression_context,
986 use_op_msg=self.op_msg_enabled,
987 unacknowledged=unacknowledged,
988 user_fields=user_fields,
989 exhaust_allowed=exhaust_allowed,
990 write_concern=write_concern,
991 )
992 except (OperationFailure, NotPrimaryError):
File ~/.local/lib/python3.11/site-packages/pymongo/network.py:192, in command(conn, dbname, spec, is_mongos, read_preference, codec_options, session, client, check, allowable_errors, address, listeners, max_bson_size, read_concern, parse_write_concern_error, collation, compression_ctx, use_op_msg, unacknowledged, user_fields, exhaust_allowed, write_concern)
191 if check:
--> 192 helpers._check_command_response(
193 response_doc,
194 conn.max_wire_version,
195 allowable_errors,
196 parse_write_concern_error=parse_write_concern_error,
197 )
198 except Exception as exc:
File ~/.local/lib/python3.11/site-packages/pymongo/helpers.py:230, in _check_command_response(response, max_wire_version, allowable_errors, parse_write_concern_error)
228 raise CursorNotFound(errmsg, code, response, max_wire_version)
--> 230 raise OperationFailure(errmsg, code, response, max_wire_version)
OperationFailure: command not found, full error: {'ok': 0, 'errmsg': 'command not found', 'code': 59, 'codeName': 'CommandNotFound'}
The above exception was the direct cause of the following exception:
DatalayerException Traceback (most recent call last)
Cell In[8], line 3
1 from superduperdb import Listener, VectorIndex
----> 3 db.add(
4 VectorIndex(
5 identifier=f'pymongo-docs-{model.identifier}',
6 indexing_listener=Listener(
7 select=doc_collection.find(),
8 key='details',
9 model=model,
10 predict_kwargs={'max_chunk_size': 1000},
11 ),
12 )
13 )
15 db.show('vector_index')
File ~/.local/lib/python3.11/site-packages/superduperdb/base/datalayer.py:556, in Datalayer.add(self, object, dependencies)
548 return type(object)(
549 self._add(
550 object=component,
(...)
553 for component in object
554 )
555 elif isinstance(object, Component):
--> 556 return self._add(object=object, dependencies=dependencies)
557 else:
558 return self._add(superduper(object))
File ~/.local/lib/python3.11/site-packages/superduperdb/base/datalayer.py:918, in Datalayer._add(self, object, dependencies, serialized, parent)
916 return object.schedule_jobs(self, dependencies=dependencies), object
917 except Exception as e:
--> 918 raise exceptions.DatalayerException(
919 f'Error while adding object with id: {object.identifier}'
920 ) from e
DatalayerException: Error while adding object with id: pymongo-docs-all-MiniLM-L6-v2
ratecheckr-rohith
Contact Details [Optional]
No response
System Information
Mongodb atlas : mongodb+srv://:@cluster0.tgtxnux.mongodb.net
database: customer_database collection: customer_details
mongo compass version : Version 1.40.4 (1.40.4) MongoDB 6.0.11 Atlas
[tool.poetry.dependencies] pymongo version = "^4.6.0" superduperdb version = "^0.0.16" python version = "^3.11" pandas version = "^2.1.3" sentence-transformers = "^2.2.2"
( I used placeholders for the username and password for privacy reasons)
What happened?
I am trying to add a model( hugging face) to the datalayer of MongoDB
and I get this error at
db.add()This is what I am expecting to get
Is there something that should be done before connecting it to Atlas?
Steps to reproduce
Relevant log output