search

Azure / Test-Drive-Azure-Synapse-with-a-1-click-POC

MIT License
61 stars 459 forks source link

Cannot properly save lrModel #3

Open waddahAldrobi opened 3 years ago

waddahAldrobi commented 3 years ago

You can still make the rest of the notebook work by commenting out this line: lrModel.save(logRegDirfilename)

And this is the error:

Py4JJavaError: An error occurred while calling o1156.save.
: Operation failed: "This request is not authorized to perform this operation using this permission.", 403, HEAD, https://mstbniiuq2crxcdapoc.dfs.core.windows.net/dlsmstpocfs1/user/trusted-service-user/lrModel_08-03-2021-1628019120?upn=false&action=getStatus&timeout=90
    at org.apache.hadoop.fs.azurebfs.services.AbfsRestOperation.execute(AbfsRestOperation.java:166)
    at org.apache.hadoop.fs.azurebfs.services.AbfsClient.getPathStatus(AbfsClient.java:414)
    at org.apache.hadoop.fs.azurebfs.AzureBlobFileSystemStore.getFileStatus(AzureBlobFileSystemStore.java:551)
    at org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem.getFileStatus(AzureBlobFileSystem.java:430)
    at org.apache.hadoop.fs.FileSystem.exists(FileSystem.java:1627)
    at org.apache.spark.ml.util.FileSystemOverwrite.handleOverwrite(ReadWrite.scala:696)
    at org.apache.spark.ml.util.MLWriter.save(ReadWrite.scala:179)
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.lang.reflect.Method.invoke(Method.java:498)
    at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
    at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
    at py4j.Gateway.invoke(Gateway.java:282)
    at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
    at py4j.commands.CallCommand.execute(CallCommand.java:79)
    at py4j.GatewayConnection.run(GatewayConnection.java:238)
    at java.lang.Thread.run(Thread.java:748)

Traceback (most recent call last):

  File "/opt/spark/python/lib/pyspark.zip/pyspark/ml/util.py", line 244, in save
    self.write().save(path)

  File "/opt/spark/python/lib/pyspark.zip/pyspark/ml/util.py", line 183, in save
    self._jwrite.save(path)

  File "/opt/spark/python/lib/py4j-0.10.7-src.zip/py4j/java_gateway.py", line 1257, in __call__
    answer, self.gateway_client, self.target_id, self.name)

  File "/opt/spark/python/lib/pyspark.zip/pyspark/sql/utils.py", line 69, in deco
    return f(*a, **kw)

  File "/opt/spark/python/lib/py4j-0.10.7-src.zip/py4j/protocol.py", line 328, in get_return_value
    format(target_id, ".", name), value)

py4j.protocol.Py4JJavaError: An error occurred while calling o1156.save.
: Operation failed: "This request is not authorized to perform this operation using this permission.", 403, HEAD, https://mstbniiuq2crxcdapoc.dfs.core.windows.net/dlsmstpocfs1/user/trusted-service-user/lrModel_08-03-2021-1628019120?upn=false&action=getStatus&timeout=90
    at org.apache.hadoop.fs.azurebfs.services.AbfsRestOperation.execute(AbfsRestOperation.java:166)
    at org.apache.hadoop.fs.azurebfs.services.AbfsClient.getPathStatus(AbfsClient.java:414)
    at org.apache.hadoop.fs.azurebfs.AzureBlobFileSystemStore.getFileStatus(AzureBlobFileSystemStore.java:551)
    at org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem.getFileStatus(AzureBlobFileSystem.java:430)
    at org.apache.hadoop.fs.FileSystem.exists(FileSystem.java:1627)
    at org.apache.spark.ml.util.FileSystemOverwrite.handleOverwrite(ReadWrite.scala:696)
    at org.apache.spark.ml.util.MLWriter.save(ReadWrite.scala:179)
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.lang.reflect.Method.invoke(Method.java:498)
    at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
    at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
    at py4j.Gateway.invoke(Gateway.java:282)
    at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
    at py4j.commands.CallCommand.execute(CallCommand.java:79)
    at py4j.GatewayConnection.run(GatewayConnection.java:238)
    at java.lang.Thread.run(Thread.java:748)
eddiet19 commented 3 years ago

Make the following directories in the Azure Storage Account. /dlsmstpocfs1/user/trusted-service-user/