search

databrickslabs / geoscan

Geospatial clustering at massive scale
Other
97 stars 19 forks source link

Error in model.save() #67

Closed kunalchiddarwar closed 2 years ago

kunalchiddarwar commented 2 years ago

Log model using mlflow.spark.log_model

from geoscan import Geoscan
import mlflow
import mlflow.spark

with mlflow.start_run(run_name='GEOSCAN_MIAMI') as run:

  geoscan = Geoscan() \
    .setLatitudeCol('latitude') \
    .setLongitudeCol('longitude') \
    .setPredictionCol('cluster') \
    .setEpsilon(800) \
    .setMinPts(14)

  mlflow.log_param('epsilon', 800)
  mlflow.log_param('minPts', 14)

  model = geoscan.fit(points_sampled)
  mlflow.spark.log_model(model, "geoscan_miami")
  run_id = run.info.run_id

Getting error :

---------------------------------------------------------------------------
Py4JJavaError                             Traceback (most recent call last)
/local_disk0/.ephemeral_nfs/envs/pythonEnv-40fec3ee-4f4a-4a71-8af7-a0a5b2dd478a/lib/python3.9/site-packages/mlflow/spark.py in log_model(spark_model, artifact_path, conda_env, code_paths, dfs_tmpdir, sample_input, registered_model_name, signature, input_example, await_registration_for, pip_requirements, extra_pip_requirements)
    235     try:
--> 236         spark_model.save(posixpath.join(model_dir, _SPARK_MODEL_PATH_SUB))
    237     except Py4JError:

/databricks/spark/python/pyspark/ml/util.py in save(self, path)
    338         """Save this ML instance to the given path, a shortcut of 'write().save(path)'."""
--> 339         self.write().save(path)
    340 

/databricks/spark/python/pyspark/ml/util.py in save(self, path)
    289             raise TypeError("path should be a string, got type %s" % type(path))
--> 290         self._jwrite.save(path)
    291 

/databricks/spark/python/lib/py4j-0.10.9.5-src.zip/py4j/java_gateway.py in __call__(self, *args)
   1320         answer = self.gateway_client.send_command(command)
-> 1321         return_value = get_return_value(
   1322             answer, self.gateway_client, self.target_id, self.name)

/databricks/spark/python/pyspark/sql/utils.py in deco(*a, **kw)
    195         try:
--> 196             return f(*a, **kw)
    197         except Py4JJavaError as e:

/databricks/spark/python/lib/py4j-0.10.9.5-src.zip/py4j/protocol.py in get_return_value(answer, gateway_client, target_id, name)
    325             if answer[1] == REFERENCE_TYPE:
--> 326                 raise Py4JJavaError(
    327                     "An error occurred while calling {0}{1}{2}.\n".

Py4JJavaError: An error occurred while calling o782.save.
: com.databricks.backend.daemon.data.common.InvalidMountException: Error while using path /databricks/mlflow-tracking/4141878947756653/45fc970fa3574628862c444810fc904c/artifacts/geoscan_miami/sparkml for resolving path '/4141878947756653/45fc970fa3574628862c444810fc904c/artifacts/geoscan_miami/sparkml' within mount at '/databricks/mlflow-tracking'.
    at com.databricks.backend.daemon.data.common.InvalidMountException$.apply(DataMessages.scala:690)
    at com.databricks.backend.daemon.data.filesystem.MountEntryResolver.resolve(MountEntryResolver.scala:84)
    at com.databricks.backend.daemon.data.client.DBFSV2.resolve(DatabricksFileSystemV2.scala:83)
    at com.databricks.backend.daemon.data.client.DatabricksFileSystemV2.$anonfun$getFileStatus$2(DatabricksFileSystemV2.scala:864)
    at com.databricks.s3a.S3AExceptionUtils$.convertAWSExceptionToJavaIOException(DatabricksStreamUtils.scala:66)
    at com.databricks.backend.daemon.data.client.DatabricksFileSystemV2.$anonfun$getFileStatus$1(DatabricksFileSystemV2.scala:863)
    at com.databricks.logging.UsageLogging.$anonfun$recordOperation$1(UsageLogging.scala:413)
    at com.databricks.logging.UsageLogging.executeThunkAndCaptureResultTags$1(UsageLogging.scala:507)
    at com.databricks.logging.UsageLogging.$anonfun$recordOperationWithResultTags$4(UsageLogging.scala:528)
    at com.databricks.logging.Log4jUsageLoggingShim$.$anonfun$withAttributionContext$1(Log4jUsageLoggingShim.scala:29)
    at scala.util.DynamicVariable.withValue(DynamicVariable.scala:62)
    at com.databricks.logging.AttributionContext$.withValue(AttributionContext.scala:94)
    at com.databricks.logging.Log4jUsageLoggingShim$.withAttributionContext(Log4jUsageLoggingShim.scala:27)
    at com.databricks.logging.UsageLogging.withAttributionContext(UsageLogging.scala:283)
    at com.databricks.logging.UsageLogging.withAttributionContext$(UsageLogging.scala:282)
    at com.databricks.backend.daemon.data.client.DatabricksFileSystemV2.withAttributionContext(DatabricksFileSystemV2.scala:512)
    at com.databricks.logging.UsageLogging.withAttributionTags(UsageLogging.scala:318)
    at com.databricks.logging.UsageLogging.withAttributionTags$(UsageLogging.scala:303)
    at com.databricks.backend.daemon.data.client.DatabricksFileSystemV2.withAttributionTags(DatabricksFileSystemV2.scala:512)
    at com.databricks.logging.UsageLogging.recordOperationWithResultTags(UsageLogging.scala:502)
    at com.databricks.logging.UsageLogging.recordOperationWithResultTags$(UsageLogging.scala:422)
    at com.databricks.backend.daemon.data.client.DatabricksFileSystemV2.recordOperationWithResultTags(DatabricksFileSystemV2.scala:512)
    at com.databricks.logging.UsageLogging.recordOperation(UsageLogging.scala:413)
    at com.databricks.logging.UsageLogging.recordOperation$(UsageLogging.scala:385)
    at com.databricks.backend.daemon.data.client.DatabricksFileSystemV2.recordOperation(DatabricksFileSystemV2.scala:512)
    at com.databricks.backend.daemon.data.client.DatabricksFileSystemV2.getFileStatus(DatabricksFileSystemV2.scala:863)
    at com.databricks.backend.daemon.data.client.DatabricksFileSystem.getFileStatus(DatabricksFileSystem.scala:215)
    at org.apache.hadoop.fs.FileSystem.exists(FileSystem.java:1760)
    at org.apache.spark.ml.util.FileSystemOverwrite.handleOverwrite(ReadWrite.scala:675)
    at org.apache.spark.ml.util.MLWriter.save(ReadWrite.scala:167)
    at org.apache.spark.ml.PipelineModel$PipelineModelWriter.super$save(Pipeline.scala:344)
    at org.apache.spark.ml.PipelineModel$PipelineModelWriter.$anonfun$save$4(Pipeline.scala:344)
    at org.apache.spark.ml.MLEvents.withSaveInstanceEvent(events.scala:175)
    at org.apache.spark.ml.MLEvents.withSaveInstanceEvent$(events.scala:170)
    at org.apache.spark.ml.util.Instrumentation.withSaveInstanceEvent(Instrumentation.scala:43)
    at org.apache.spark.ml.PipelineModel$PipelineModelWriter.$anonfun$save$3(Pipeline.scala:344)
    at org.apache.spark.ml.PipelineModel$PipelineModelWriter.$anonfun$save$3$adapted(Pipeline.scala:344)
    at org.apache.spark.ml.util.Instrumentation$.$anonfun$instrumented$1(Instrumentation.scala:284)
    at scala.util.Try$.apply(Try.scala:213)
    at org.apache.spark.ml.util.Instrumentation$.instrumented(Instrumentation.scala:284)
    at org.apache.spark.ml.PipelineModel$PipelineModelWriter.save(Pipeline.scala:344)
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.lang.reflect.Method.invoke(Method.java:498)
    at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
    at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)
    at py4j.Gateway.invoke(Gateway.java:306)
    at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
    at py4j.commands.CallCommand.execute(CallCommand.java:79)
    at py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:195)
    at py4j.ClientServerConnection.run(ClientServerConnection.java:115)
    at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.hadoop.fs.UnsupportedFileSystemException: No FileSystem for scheme "unsupported-access-mechanism-for-path--use-mlflow-client"
    at org.apache.hadoop.fs.FileSystem.getFileSystemClass(FileSystem.java:3443)
    at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:3466)
    at org.apache.hadoop.fs.FileSystem.access$300(FileSystem.java:174)
    at org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:3574)
    at org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:3521)
    at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:540)
    at com.databricks.backend.daemon.data.client.DatabricksFileSystemV2Factory.createFileSystem(DatabricksFileSystemV2Factory.scala:123)
    at com.databricks.backend.daemon.data.filesystem.MountEntryResolver.$anonfun$resolve$1(MountEntryResolver.scala:67)
    at com.databricks.logging.UsageLogging.$anonfun$recordOperation$1(UsageLogging.scala:413)
    at com.databricks.logging.UsageLogging.executeThunkAndCaptureResultTags$1(UsageLogging.scala:507)
    at com.databricks.logging.UsageLogging.$anonfun$recordOperationWithResultTags$4(UsageLogging.scala:528)
    at com.databricks.logging.Log4jUsageLoggingShim$.$anonfun$withAttributionContext$1(Log4jUsageLoggingShim.scala:29)
    at scala.util.DynamicVariable.withValue(DynamicVariable.scala:62)
    at com.databricks.logging.AttributionContext$.withValue(AttributionContext.scala:94)
    at com.databricks.logging.Log4jUsageLoggingShim$.withAttributionContext(Log4jUsageLoggingShim.scala:27)
    at com.databricks.logging.UsageLogging.withAttributionContext(UsageLogging.scala:283)
    at com.databricks.logging.UsageLogging.withAttributionContext$(UsageLogging.scala:282)
    at com.databricks.common.util.locks.LoggedLock$.withAttributionContext(LoggedLock.scala:74)
    at com.databricks.logging.UsageLogging.withAttributionTags(UsageLogging.scala:318)
    at com.databricks.logging.UsageLogging.withAttributionTags$(UsageLogging.scala:303)
    at com.databricks.common.util.locks.LoggedLock$.withAttributionTags(LoggedLock.scala:74)
    at com.databricks.logging.UsageLogging.recordOperationWithResultTags(UsageLogging.scala:502)
    at com.databricks.logging.UsageLogging.recordOperationWithResultTags$(UsageLogging.scala:422)
    at com.databricks.common.util.locks.LoggedLock$.recordOperationWithResultTags(LoggedLock.scala:74)
    at com.databricks.logging.UsageLogging.recordOperation(UsageLogging.scala:413)
    at com.databricks.logging.UsageLogging.recordOperation$(UsageLogging.scala:385)
    at com.databricks.common.util.locks.LoggedLock$.recordOperation(LoggedLock.scala:74)
    at com.databricks.common.util.locks.LoggedLock$.withLock(LoggedLock.scala:121)
    at com.databricks.common.util.locks.PerKeyLock.withLock(PerKeyLock.scala:36)
    at com.databricks.backend.daemon.data.filesystem.MountEntryResolver.resolve(MountEntryResolver.scala:64)
    ... 51 more

During handling of the above exception, another exception occurred:

Py4JJavaError                             Traceback (most recent call last)
<command-4141878947756665> in <cell line: 5>()
     16 
     17   model = geoscan.fit(points_sampled)
---> 18   mlflow.spark.log_model(model, "geoscan_miami")
     19   run_id = run.info.run_id

/local_disk0/.ephemeral_nfs/envs/pythonEnv-40fec3ee-4f4a-4a71-8af7-a0a5b2dd478a/lib/python3.9/site-packages/mlflow/spark.py in log_model(spark_model, artifact_path, conda_env, code_paths, dfs_tmpdir, sample_input, registered_model_name, signature, input_example, await_registration_for, pip_requirements, extra_pip_requirements)
    236         spark_model.save(posixpath.join(model_dir, _SPARK_MODEL_PATH_SUB))
    237     except Py4JError:
--> 238         return Model.log(
    239             artifact_path=artifact_path,
    240             flavor=mlflow.spark,

/local_disk0/.ephemeral_nfs/envs/pythonEnv-40fec3ee-4f4a-4a71-8af7-a0a5b2dd478a/lib/python3.9/site-packages/mlflow/models/model.py in log(cls, artifact_path, flavor, registered_model_name, await_registration_for, **kwargs)
    292             run_id = mlflow.tracking.fluent._get_or_start_run().info.run_id
    293             mlflow_model = cls(artifact_path=artifact_path, run_id=run_id)
--> 294             flavor.save_model(path=local_path, mlflow_model=mlflow_model, **kwargs)
    295             mlflow.tracking.fluent.log_artifacts(local_path, artifact_path)
    296             try:

/local_disk0/.ephemeral_nfs/envs/pythonEnv-40fec3ee-4f4a-4a71-8af7-a0a5b2dd478a/lib/python3.9/site-packages/mlflow/spark.py in save_model(spark_model, path, mlflow_model, conda_env, code_paths, dfs_tmpdir, sample_input, signature, input_example, pip_requirements, extra_pip_requirements)
    585         dfs_tmpdir = DFS_TMP
    586     tmp_path = _tmp_path(dfs_tmpdir)
--> 587     spark_model.save(tmp_path)
    588     sparkml_data_path = os.path.abspath(os.path.join(path, _SPARK_MODEL_PATH_SUB))
    589     # We're copying the Spark model from DBFS to the local filesystem if (a) the temporary DFS URI

/databricks/spark/python/pyspark/ml/util.py in save(self, path)
    337     def save(self, path: str) -> None:
    338         """Save this ML instance to the given path, a shortcut of 'write().save(path)'."""
--> 339         self.write().save(path)
    340 
    341 

/databricks/spark/python/pyspark/ml/util.py in save(self, path)
    288         if not isinstance(path, str):
    289             raise TypeError("path should be a string, got type %s" % type(path))
--> 290         self._jwrite.save(path)
    291 
    292     def overwrite(self) -> "JavaMLWriter":

/databricks/spark/python/lib/py4j-0.10.9.5-src.zip/py4j/java_gateway.py in __call__(self, *args)
   1319 
   1320         answer = self.gateway_client.send_command(command)
-> 1321         return_value = get_return_value(
   1322             answer, self.gateway_client, self.target_id, self.name)
   1323 

/databricks/spark/python/pyspark/sql/utils.py in deco(*a, **kw)
    194     def deco(*a: Any, **kw: Any) -> Any:
    195         try:
--> 196             return f(*a, **kw)
    197         except Py4JJavaError as e:
    198             converted = convert_exception(e.java_exception)

/databricks/spark/python/lib/py4j-0.10.9.5-src.zip/py4j/protocol.py in get_return_value(answer, gateway_client, target_id, name)
    324             value = OUTPUT_CONVERTER[type](answer[2:], gateway_client)
    325             if answer[1] == REFERENCE_TYPE:
--> 326                 raise Py4JJavaError(
    327                     "An error occurred while calling {0}{1}{2}.\n".
    328                     format(target_id, ".", name), value)

Py4JJavaError: An error occurred while calling o803.save.
: java.lang.NoSuchMethodError: org.json4s.JsonDSL$.pair2Assoc(Lscala/Tuple2;Lscala/Function1;)Lorg/json4s/JsonDSL$JsonAssoc;
    at com.databricks.labs.gis.ml.ModelIOUtils$.getMetadataToSave(ModelIOUtils.scala:72)
    at com.databricks.labs.gis.ml.ModelIOUtils$.saveMetadata(ModelIOUtils.scala:58)
    at com.databricks.labs.gis.ml.GeoscanModel$GeoscanModelWriter.saveImpl(GeoscanModel.scala:119)
    at org.apache.spark.ml.util.MLWriter.save(ReadWrite.scala:168)
    at org.apache.spark.ml.Pipeline$SharedReadWrite$.$anonfun$saveImpl$5(Pipeline.scala:257)
    at org.apache.spark.ml.MLEvents.withSaveInstanceEvent(events.scala:175)
    at org.apache.spark.ml.MLEvents.withSaveInstanceEvent$(events.scala:170)
    at org.apache.spark.ml.util.Instrumentation.withSaveInstanceEvent(Instrumentation.scala:43)
    at org.apache.spark.ml.Pipeline$SharedReadWrite$.$anonfun$saveImpl$4(Pipeline.scala:257)
    at org.apache.spark.ml.Pipeline$SharedReadWrite$.$anonfun$saveImpl$4$adapted(Pipeline.scala:254)
    at scala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)
    at scala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)
    at scala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)
    at org.apache.spark.ml.Pipeline$SharedReadWrite$.$anonfun$saveImpl$1(Pipeline.scala:254)
    at org.apache.spark.ml.Pipeline$SharedReadWrite$.$anonfun$saveImpl$1$adapted(Pipeline.scala:247)
    at org.apache.spark.ml.util.Instrumentation$.$anonfun$instrumented$1(Instrumentation.scala:284)
    at scala.util.Try$.apply(Try.scala:213)
    at org.apache.spark.ml.util.Instrumentation$.instrumented(Instrumentation.scala:284)
    at org.apache.spark.ml.Pipeline$SharedReadWrite$.saveImpl(Pipeline.scala:247)
    at org.apache.spark.ml.PipelineModel$PipelineModelWriter.saveImpl(Pipeline.scala:346)
    at org.apache.spark.ml.util.MLWriter.save(ReadWrite.scala:168)
    at org.apache.spark.ml.PipelineModel$PipelineModelWriter.super$save(Pipeline.scala:344)
    at org.apache.spark.ml.PipelineModel$PipelineModelWriter.$anonfun$save$4(Pipeline.scala:344)
    at org.apache.spark.ml.MLEvents.withSaveInstanceEvent(events.scala:175)
    at org.apache.spark.ml.MLEvents.withSaveInstanceEvent$(events.scala:170)
    at org.apache.spark.ml.util.Instrumentation.withSaveInstanceEvent(Instrumentation.scala:43)
    at org.apache.spark.ml.PipelineModel$PipelineModelWriter.$anonfun$save$3(Pipeline.scala:344)
    at org.apache.spark.ml.PipelineModel$PipelineModelWriter.$anonfun$save$3$adapted(Pipeline.scala:344)
    at org.apache.spark.ml.util.Instrumentation$.$anonfun$instrumented$1(Instrumentation.scala:284)
    at scala.util.Try$.apply(Try.scala:213)
    at org.apache.spark.ml.util.Instrumentation$.instrumented(Instrumentation.scala:284)
    at org.apache.spark.ml.PipelineModel$PipelineModelWriter.save(Pipeline.scala:344)
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.lang.reflect.Method.invoke(Method.java:498)
    at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
    at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)
    at py4j.Gateway.invoke(Gateway.java:306)
    at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
    at py4j.commands.CallCommand.execute(CallCommand.java:79)
    at py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:195)
    at py4j.ClientServerConnection.run(ClientServerConnection.java:115)
    at java.lang.Thread.run(Thread.java:748)

DBR : 11.0 ML

When tried to just save the model

model.save("dbfs:/tmp/geojson-model")

Getting similar error :

---------------------------------------------------------------------------
Py4JJavaError                             Traceback (most recent call last)
<command-3409347987432082> in <cell line: 1>()
----> 1 model.save("dbfs:/tmp/geojson-model")

/databricks/spark/python/pyspark/ml/util.py in save(self, path)
    337     def save(self, path: str) -> None:
    338         """Save this ML instance to the given path, a shortcut of 'write().save(path)'."""
--> 339         self.write().save(path)
    340 
    341 

/databricks/spark/python/pyspark/ml/util.py in save(self, path)
    288         if not isinstance(path, str):
    289             raise TypeError("path should be a string, got type %s" % type(path))
--> 290         self._jwrite.save(path)
    291 
    292     def overwrite(self) -> "JavaMLWriter":

/databricks/spark/python/lib/py4j-0.10.9.5-src.zip/py4j/java_gateway.py in __call__(self, *args)
   1319 
   1320         answer = self.gateway_client.send_command(command)
-> 1321         return_value = get_return_value(
   1322             answer, self.gateway_client, self.target_id, self.name)
   1323 

/databricks/spark/python/pyspark/sql/utils.py in deco(*a, **kw)
    194     def deco(*a: Any, **kw: Any) -> Any:
    195         try:
--> 196             return f(*a, **kw)
    197         except Py4JJavaError as e:
    198             converted = convert_exception(e.java_exception)

/databricks/spark/python/lib/py4j-0.10.9.5-src.zip/py4j/protocol.py in get_return_value(answer, gateway_client, target_id, name)
    324             value = OUTPUT_CONVERTER[type](answer[2:], gateway_client)
    325             if answer[1] == REFERENCE_TYPE:
--> 326                 raise Py4JJavaError(
    327                     "An error occurred while calling {0}{1}{2}.\n".
    328                     format(target_id, ".", name), value)

Py4JJavaError: An error occurred while calling o714.save.
: java.lang.NoSuchMethodError: org.json4s.JsonDSL$.pair2Assoc(Lscala/Tuple2;Lscala/Function1;)Lorg/json4s/JsonDSL$JsonAssoc;
    at com.databricks.labs.gis.ml.ModelIOUtils$.getMetadataToSave(ModelIOUtils.scala:72)
    at com.databricks.labs.gis.ml.ModelIOUtils$.saveMetadata(ModelIOUtils.scala:58)
    at com.databricks.labs.gis.ml.GeoscanModel$GeoscanModelWriter.saveImpl(GeoscanModel.scala:119)
    at org.apache.spark.ml.util.MLWriter.save(ReadWrite.scala:168)
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.lang.reflect.Method.invoke(Method.java:498)
    at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
    at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)
    at py4j.Gateway.invoke(Gateway.java:306)
    at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
    at py4j.commands.CallCommand.execute(CallCommand.java:79)
    at py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:195)
    at py4j.ClientServerConnection.run(ClientServerConnection.java:115)
    at java.lang.Thread.run(Thread.java:748)
aamend commented 2 years ago

Just so you know I'm looking at this issue. Not ignoring you :) This solution was initially designed for 9.1LTS runtime. Still works fine for 9.1, looking at what is causing that problem for 10.4

kunalchiddarwar commented 2 years ago

Will it able to run on 9.1 or later?

aamend commented 2 years ago

it runs on 9.1. Some issues with json4s deps on higher runtime I'll try to resolve