search

JohnSnowLabs / spark-nlp

State of the Art Natural Language Processing
https://sparknlp.org/
Apache License 2.0
3.76k stars 704 forks source link

An error occurred while calling z:com.johnsnowlabs.nlp.pretrained.PythonResourceDownloader.downloadModel. : java.lang.UnsatisfiedLinkError: no jnitensorflow in java.library.path: /Users/alexc./Library/Java/Extensions:/Library/Java/Extensions:/Network/Library/Java/Extensions:/System/Library/Java/Extensions:/usr/lib/java:. #14219

Open acort088 opened 3 months ago

acort088 commented 3 months ago

My Code: embeddings = BertSentenceEmbeddings.pretrained() \ .setInputCols(["sentence"]) \ .setOutputCol("sentence_bert_embeddings")\ .setCaseSensitive(True) \ .setMaxSentenceLength(512)

The Error message: An error occurred while calling z:com.johnsnowlabs.nlp.pretrained.PythonResourceDownloader.downloadModel. : java.lang.UnsatisfiedLinkError: no jnitensorflow in java.library.path: /Users/alexc./Library/Java/Extensions:/Library/Java/Extensions:/Network/Library/Java/Extensions:/System/Library/Java/Extensions:/usr/lib/java:. at java.base/java.lang.ClassLoader.loadLibrary(ClassLoader.java:2444) at java.base/java.lang.Runtime.loadLibrary0(Runtime.java:848) at java.base/java.lang.System.loadLibrary(System.java:2047) at org.bytedeco.javacpp.Loader.loadLibrary(Loader.java:1738) at org.bytedeco.javacpp.Loader.load(Loader.java:1345) at org.bytedeco.javacpp.Loader.load(Loader.java:1157) at org.bytedeco.javacpp.Loader.load(Loader.java:1133) at org.tensorflow.internal.c_api.global.tensorflow.(tensorflow.java:12) at java.base/java.lang.Class.forName0(Native Method) at java.base/java.lang.Class.forName(Class.java:495) at java.base/java.lang.Class.forName(Class.java:474) at org.bytedeco.javacpp.Loader.load(Loader.java:1212) at org.bytedeco.javacpp.Loader.load(Loader.java:1157) at org.bytedeco.javacpp.Loader.load(Loader.java:1149) at org.tensorflow.NativeLibrary.load(NativeLibrary.java:64) at org.tensorflow.TensorFlow.(TensorFlow.java:156) at java.base/java.lang.Class.forName0(Native Method) at java.base/java.lang.Class.forName(Class.java:390) at java.base/java.lang.Class.forName(Class.java:381) at org.tensorflow.Graph.(Graph.java:1341) at com.johnsnowlabs.ml.tensorflow.TensorflowWrapper$.readGraph(TensorflowWrapper.scala:415) at com.johnsnowlabs.ml.tensorflow.TensorflowWrapper$.unpackWithoutBundle(TensorflowWrapper.scala:330) at com.johnsnowlabs.ml.tensorflow.TensorflowWrapper$.read(TensorflowWrapper.scala:484) at com.johnsnowlabs.ml.tensorflow.ReadTensorflowModel.readTensorflowModel(TensorflowSerializeModel.scala:154) at com.johnsnowlabs.ml.tensorflow.ReadTensorflowModel.readTensorflowModel$(TensorflowSerializeModel.scala:123) at com.johnsnowlabs.nlp.embeddings.BertSentenceEmbeddings$.readTensorflowModel(BertSentenceEmbeddings.scala:520) at com.johnsnowlabs.nlp.embeddings.ReadBertSentenceDLModel.readModel(BertSentenceEmbeddings.scala:454) at com.johnsnowlabs.nlp.embeddings.ReadBertSentenceDLModel.readModel$(BertSentenceEmbeddings.scala:449) at com.johnsnowlabs.nlp.embeddings.BertSentenceEmbeddings$.readModel(BertSentenceEmbeddings.scala:520) at com.johnsnowlabs.nlp.embeddings.ReadBertSentenceDLModel.$anonfun$$init$$1(BertSentenceEmbeddings.scala:473) at com.johnsnowlabs.nlp.embeddings.ReadBertSentenceDLModel.$anonfun$$init$$1$adapted(BertSentenceEmbeddings.scala:473) at com.johnsnowlabs.nlp.ParamsAndFeaturesReadable.$anonfun$onRead$1(ParamsAndFeaturesReadable.scala:50) at com.johnsnowlabs.nlp.ParamsAndFeaturesReadable.$anonfun$onRead$1$adapted(ParamsAndFeaturesReadable.scala:49) at scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62) at scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49) at com.johnsnowlabs.nlp.ParamsAndFeaturesReadable.onRead(ParamsAndFeaturesReadable.scala:49) at com.johnsnowlabs.nlp.ParamsAndFeaturesReadable.$anonfun$read$1(ParamsAndFeaturesReadable.scala:61) at com.johnsnowlabs.nlp.ParamsAndFeaturesReadable.$anonfun$read$1$adapted(ParamsAndFeaturesReadable.scala:61) at com.johnsnowlabs.nlp.FeaturesReader.load(ParamsAndFeaturesReadable.scala:38) at com.johnsnowlabs.nlp.FeaturesReader.load(ParamsAndFeaturesReadable.scala:24) at com.johnsnowlabs.nlp.pretrained.ResourceDownloader$.downloadModel(ResourceDownloader.scala:515) at com.johnsnowlabs.nlp.pretrained.ResourceDownloader$.downloadModel(ResourceDownloader.scala:507) at com.johnsnowlabs.nlp.pretrained.PythonResourceDownloader$.downloadModel(ResourceDownloader.scala:713) at com.johnsnowlabs.nlp.pretrained.PythonResourceDownloader.downloadModel(ResourceDownloader.scala) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:76) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:52) at java.base/java.lang.reflect.Method.invoke(Method.java:578) at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) at py4j.Gateway.invoke(Gateway.java:282) at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) at py4j.commands.CallCommand.execute(CallCommand.java:79) at py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) at py4j.ClientServerConnection.run(ClientServerConnection.java:106) at java.base/java.lang.Thread.run(Thread.java:1589) Caused by: java.lang.UnsatisfiedLinkError: Could not find jnitensorflow in class, module, and library paths. at org.bytedeco.javacpp.Loader.loadLibrary(Loader.java:1705) ... 53 more [OK!]

Py4JJavaError Traceback (most recent call last) Cell In[7], line 1 ----> 1 embeddings = BertSentenceEmbeddings.pretrained() \ 2 .setInputCols(["sentence"]) \ 3 .setOutputCol("sentence_bert_embeddings")\ 4 .setCaseSensitive(True) \ 5 .setMaxSentenceLength(512)

File ~/anaconda3/lib/python3.11/site-packages/sparknlp/annotator/embeddings/bert_sentence_embeddings.py:222, in BertSentenceEmbeddings.pretrained(name, lang, remote_loc) 204 """Downloads and loads a pretrained model. 205 206 Parameters (...) 219 The restored model 220 """ 221 from sparknlp.pretrained import ResourceDownloader --> 222 return ResourceDownloader.downloadModel(BertSentenceEmbeddings, name, lang, remote_loc)

File ~/anaconda3/lib/python3.11/site-packages/sparknlp/pretrained/resource_downloader.py:99, in ResourceDownloader.downloadModel(reader, name, language, remote_loc, j_dwn) 97 except Py4JJavaError as e: 98 sys.stdout.write("\n" + str(e)) ---> 99 raise e 100 finally: 101 stop_threads = True

File ~/anaconda3/lib/python3.11/site-packages/sparknlp/pretrained/resource_downloader.py:96, in ResourceDownloader.downloadModel(reader, name, language, remote_loc, j_dwn) 94 t1.start() 95 try: ---> 96 j_obj = _internal._DownloadModel(reader.name, name, language, remote_loc, j_dwn).apply() 97 except Py4JJavaError as e: 98 sys.stdout.write("\n" + str(e))

File ~/anaconda3/lib/python3.11/site-packages/sparknlp/internal/init.py:364, in _DownloadModel.init(self, reader, name, language, remote_loc, validator) 363 def init(self, reader, name, language, remote_loc, validator): --> 364 super(_DownloadModel, self).init("com.johnsnowlabs.nlp.pretrained." + validator + ".downloadModel", reader, 365 name, language, remote_loc)

File ~/anaconda3/lib/python3.11/site-packages/sparknlp/internal/extended_java_wrapper.py:27, in ExtendedJavaWrapper.init(self, java_obj, args) 25 super(ExtendedJavaWrapper, self).init(java_obj) 26 self.sc = SparkContext._active_spark_context ---> 27 self._java_obj = self.new_java_obj(java_obj, args) 28 self.java_obj = self._java_obj

File ~/anaconda3/lib/python3.11/site-packages/sparknlp/internal/extended_java_wrapper.py:37, in ExtendedJavaWrapper.new_java_obj(self, java_class, args) 36 def new_java_obj(self, java_class, args): ---> 37 return self._new_java_obj(java_class, *args)

File ~/anaconda3/lib/python3.11/site-packages/pyspark/ml/wrapper.py:86, in JavaWrapper._new_java_obj(java_class, args) 84 java_obj = getattr(java_obj, name) 85 java_args = [_py2java(sc, arg) for arg in args] ---> 86 return java_obj(java_args)

File ~/anaconda3/lib/python3.11/site-packages/py4j/java_gateway.py:1322, in JavaMember.call(self, *args) 1316 command = proto.CALL_COMMAND_NAME +\ 1317 self.command_header +\ 1318 args_command +\ 1319 proto.END_COMMAND_PART 1321 answer = self.gateway_client.send_command(command) -> 1322 return_value = get_return_value( 1323 answer, self.gateway_client, self.target_id, self.name) 1325 for temp_arg in temp_args: 1326 if hasattr(temp_arg, "_detach"):

File ~/anaconda3/lib/python3.11/site-packages/pyspark/errors/exceptions/captured.py:179, in capture_sql_exception..deco(*a, kw) 177 def deco(*a: Any, *kw: Any) -> Any: 178 try: --> 179 return f(a, kw) 180 except Py4JJavaError as e: 181 converted = convert_exception(e.java_exception)

File ~/anaconda3/lib/python3.11/site-packages/py4j/protocol.py:326, in get_return_value(answer, gateway_client, target_id, name) 324 value = OUTPUT_CONVERTER[type](answer[2:], gateway_client) 325 if answer[1] == REFERENCE_TYPE: --> 326 raise Py4JJavaError( 327 "An error occurred while calling {0}{1}{2}.\n". 328 format(target_id, ".", name), value) 329 else: 330 raise Py4JError( 331 "An error occurred while calling {0}{1}{2}. Trace:\n{3}\n". 332 format(target_id, ".", name, value))

Py4JJavaError: An error occurred while calling z:com.johnsnowlabs.nlp.pretrained.PythonResourceDownloader.downloadModel. : java.lang.UnsatisfiedLinkError: no jnitensorflow in java.library.path: /Users/alexc./Library/Java/Extensions:/Library/Java/Extensions:/Network/Library/Java/Extensions:/System/Library/Java/Extensions:/usr/lib/java:. at java.base/java.lang.ClassLoader.loadLibrary(ClassLoader.java:2444) at java.base/java.lang.Runtime.loadLibrary0(Runtime.java:848) at java.base/java.lang.System.loadLibrary(System.java:2047) at org.bytedeco.javacpp.Loader.loadLibrary(Loader.java:1738) at org.bytedeco.javacpp.Loader.load(Loader.java:1345) at org.bytedeco.javacpp.Loader.load(Loader.java:1157) at org.bytedeco.javacpp.Loader.load(Loader.java:1133) at org.tensorflow.internal.c_api.global.tensorflow.(tensorflow.java:12) at java.base/java.lang.Class.forName0(Native Method) at java.base/java.lang.Class.forName(Class.java:495) at java.base/java.lang.Class.forName(Class.java:474) at org.bytedeco.javacpp.Loader.load(Loader.java:1212) at org.bytedeco.javacpp.Loader.load(Loader.java:1157) at org.bytedeco.javacpp.Loader.load(Loader.java:1149) at org.tensorflow.NativeLibrary.load(NativeLibrary.java:64) at org.tensorflow.TensorFlow.(TensorFlow.java:156) at java.base/java.lang.Class.forName0(Native Method) at java.base/java.lang.Class.forName(Class.java:390) at java.base/java.lang.Class.forName(Class.java:381) at org.tensorflow.Graph.(Graph.java:1341) at com.johnsnowlabs.ml.tensorflow.TensorflowWrapper$.readGraph(TensorflowWrapper.scala:415) at com.johnsnowlabs.ml.tensorflow.TensorflowWrapper$.unpackWithoutBundle(TensorflowWrapper.scala:330) at com.johnsnowlabs.ml.tensorflow.TensorflowWrapper$.read(TensorflowWrapper.scala:484) at com.johnsnowlabs.ml.tensorflow.ReadTensorflowModel.readTensorflowModel(TensorflowSerializeModel.scala:154) at com.johnsnowlabs.ml.tensorflow.ReadTensorflowModel.readTensorflowModel$(TensorflowSerializeModel.scala:123) at com.johnsnowlabs.nlp.embeddings.BertSentenceEmbeddings$.readTensorflowModel(BertSentenceEmbeddings.scala:520) at com.johnsnowlabs.nlp.embeddings.ReadBertSentenceDLModel.readModel(BertSentenceEmbeddings.scala:454) at com.johnsnowlabs.nlp.embeddings.ReadBertSentenceDLModel.readModel$(BertSentenceEmbeddings.scala:449) at com.johnsnowlabs.nlp.embeddings.BertSentenceEmbeddings$.readModel(BertSentenceEmbeddings.scala:520) at com.johnsnowlabs.nlp.embeddings.ReadBertSentenceDLModel.$anonfun$$init$$1(BertSentenceEmbeddings.scala:473) at com.johnsnowlabs.nlp.embeddings.ReadBertSentenceDLModel.$anonfun$$init$$1$adapted(BertSentenceEmbeddings.scala:473) at com.johnsnowlabs.nlp.ParamsAndFeaturesReadable.$anonfun$onRead$1(ParamsAndFeaturesReadable.scala:50) at com.johnsnowlabs.nlp.ParamsAndFeaturesReadable.$anonfun$onRead$1$adapted(ParamsAndFeaturesReadable.scala:49) at scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62) at scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49) at com.johnsnowlabs.nlp.ParamsAndFeaturesReadable.onRead(ParamsAndFeaturesReadable.scala:49) at com.johnsnowlabs.nlp.ParamsAndFeaturesReadable.$anonfun$read$1(ParamsAndFeaturesReadable.scala:61) at com.johnsnowlabs.nlp.ParamsAndFeaturesReadable.$anonfun$read$1$adapted(ParamsAndFeaturesReadable.scala:61) at com.johnsnowlabs.nlp.FeaturesReader.load(ParamsAndFeaturesReadable.scala:38) at com.johnsnowlabs.nlp.FeaturesReader.load(ParamsAndFeaturesReadable.scala:24) at com.johnsnowlabs.nlp.pretrained.ResourceDownloader$.downloadModel(ResourceDownloader.scala:515) at com.johnsnowlabs.nlp.pretrained.ResourceDownloader$.downloadModel(ResourceDownloader.scala:507) at com.johnsnowlabs.nlp.pretrained.PythonResourceDownloader$.downloadModel(ResourceDownloader.scala:713) at com.johnsnowlabs.nlp.pretrained.PythonResourceDownloader.downloadModel(ResourceDownloader.scala) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:76) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:52) at java.base/java.lang.reflect.Method.invoke(Method.java:578) at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) at py4j.Gateway.invoke(Gateway.java:282) at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) at py4j.commands.CallCommand.execute(CallCommand.java:79) at py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) at py4j.ClientServerConnection.run(ClientServerConnection.java:106) at java.base/java.lang.Thread.run(Thread.java:1589) Caused by: java.lang.UnsatisfiedLinkError: Could not find jnitensorflow in class, module, and library paths. at org.bytedeco.javacpp.Loader.loadLibrary(Loader.java:1705) ... 53 more

maziyarpanahi commented 3 months ago

@acort088 could you please use the bug report template to provide all the required information? it could be many things resulting in this error