~\anaconda3\lib\site-packages\pyspark\sql\dataframe.py in getattr(self, name)
1640 [Row(age=2), Row(age=5)]
1641 """
-> 1642 if name not in self.columns:
1643 raise AttributeError(
1644 "'%s' object has no attribute '%s'" % (self.class.name, name))
~\anaconda3\lib\site-packages\pyspark\sql\dataframe.py in columns(self)
1197 ['age', 'name']
1198 """
-> 1199 return [f.name for f in self.schema.fields]
1200
1201 def colRegex(self, colName):
~\anaconda3\lib\site-packages\pyspark\sql\dataframe.py in schema(self)
274 if self._schema is None:
275 try:
--> 276 self._schema = _parse_datatype_json_string(self._jdf.schema().json())
277 except AttributeError as e:
278 raise Exception(
~\anaconda3\lib\site-packages\py4j\protocol.py in get_return_value(answer, gateway_client, target_id, name)
324 value = OUTPUT_CONVERTER[type](answer[2:], gateway_client)
325 if answer[1] == REFERENCE_TYPE:
--> 326 raise Py4JJavaError(
327 "An error occurred while calling {0}{1}{2}.\n".
328 format(target_id, ".", name), value)
Py4JJavaError: An error occurred while calling o79.json.
: java.lang.NoSuchMethodError: org.json4s.JsonDSL$.pair2Assoc(Lscala/Tuple2;Lscala/Function1;)Lorg/json4s/JsonDSL$JsonAssoc;
at org.apache.spark.sql.types.StructType.jsonValue(StructType.scala:411)
at org.apache.spark.sql.types.StructType.jsonValue(StructType.scala:102)
at org.apache.spark.sql.types.DataType.json(DataType.scala:76)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
at java.lang.reflect.Method.invoke(Unknown Source)
at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
at py4j.Gateway.invoke(Gateway.java:282)
at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
at py4j.commands.CallCommand.execute(CallCommand.java:79)
at py4j.GatewayConnection.run(GatewayConnection.java:238)
at java.lang.Thread.run(Unknown Source)
I'm getting this when I'm trying to filter my data using PySpark on my Jupyter Notebook. Does anyone have any idea how to fix this?
Environment
Jupyter notebook
Error logs
Py4JJavaError Traceback (most recent call last) ~\AppData\Local\Temp/ipykernel_26672/3571456697.py in
----> 1 df2.filter(df2.type=="TV Show").show()
~\anaconda3\lib\site-packages\pyspark\sql\dataframe.py in getattr(self, name) 1640 [Row(age=2), Row(age=5)] 1641 """ -> 1642 if name not in self.columns: 1643 raise AttributeError( 1644 "'%s' object has no attribute '%s'" % (self.class.name, name))
~\anaconda3\lib\site-packages\pyspark\sql\dataframe.py in columns(self) 1197 ['age', 'name'] 1198 """ -> 1199 return [f.name for f in self.schema.fields] 1200 1201 def colRegex(self, colName):
~\anaconda3\lib\site-packages\pyspark\sql\dataframe.py in schema(self) 274 if self._schema is None: 275 try: --> 276 self._schema = _parse_datatype_json_string(self._jdf.schema().json()) 277 except AttributeError as e: 278 raise Exception(
~\anaconda3\lib\site-packages\py4j\java_gateway.py in call(self, *args) 1307 1308 answer = self.gateway_client.send_command(command) -> 1309 return_value = get_return_value( 1310 answer, self.gateway_client, self.target_id, self.name) 1311
~\anaconda3\lib\site-packages\pyspark\sql\utils.py in deco(*a, kw) 109 def deco(*a, *kw): 110 try: --> 111 return f(a, kw) 112 except py4j.protocol.Py4JJavaError as e: 113 converted = convert_exception(e.java_exception)
~\anaconda3\lib\site-packages\py4j\protocol.py in get_return_value(answer, gateway_client, target_id, name) 324 value = OUTPUT_CONVERTER[type](answer[2:], gateway_client) 325 if answer[1] == REFERENCE_TYPE: --> 326 raise Py4JJavaError( 327 "An error occurred while calling {0}{1}{2}.\n". 328 format(target_id, ".", name), value)
Py4JJavaError: An error occurred while calling o79.json. : java.lang.NoSuchMethodError: org.json4s.JsonDSL$.pair2Assoc(Lscala/Tuple2;Lscala/Function1;)Lorg/json4s/JsonDSL$JsonAssoc; at org.apache.spark.sql.types.StructType.jsonValue(StructType.scala:411) at org.apache.spark.sql.types.StructType.jsonValue(StructType.scala:102) at org.apache.spark.sql.types.DataType.json(DataType.scala:76) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) at java.lang.reflect.Method.invoke(Unknown Source) at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) at py4j.Gateway.invoke(Gateway.java:282) at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) at py4j.commands.CallCommand.execute(CallCommand.java:79) at py4j.GatewayConnection.run(GatewayConnection.java:238) at java.lang.Thread.run(Unknown Source)
Steps to reproduce
Other descriptions