[BUG] Sparkmagic errors out using iPython 7.33.0

[ryerobinson]

Describe the bug When running reload_ext sparkmagic.magics in a Jupyter Notebook in Amazon Sagemaker I get "AttributeError: 'IpyWidgetFactory' object has no attribute 'get_password'"

To Reproduce Steps to reproduce the behavior. Create livy connection using %reload_ext sparkmagic.magics in Amazon Linux 2 version of AWS SageMaker.

Expected behavior Notebook should start a spark connection.

Screenshots

AttributeError                            Traceback (most recent call last)
<ipython-input-4-87e8550c6845> in <module>
----> 1 get_ipython().run_line_magic('reload_ext', 'sparkmagic.magics')

~/anaconda3/envs/JupyterSystemEnv/lib/python3.7/site-packages/IPython/core/interactiveshell.py in run_line_magic(self, magic_name, line, _stack_depth)
   2349                 kwargs['local_ns'] = self.get_local_scope(stack_depth)
   2350             with self.builtin_trap:
-> 2351                 result = fn(*args, **kwargs)
   2352             return result
   2353 

~/anaconda3/envs/JupyterSystemEnv/lib/python3.7/site-packages/decorator.py in fun(*args, **kw)
    230             if not kwsyntax:
    231                 args, kw = fix(args, kw, sig)
--> 232             return caller(func, *(extras + args), **kw)
    233     fun.__name__ = func.__name__
    234     fun.__doc__ = func.__doc__

~/anaconda3/envs/JupyterSystemEnv/lib/python3.7/site-packages/IPython/core/magic.py in <lambda>(f, *a, **k)
    185     # but it's overkill for just that one bit of state.
    186     def magic_deco(arg):
--> 187         call = lambda f, *a, **k: f(*a, **k)
    188 
    189         if callable(arg):

~/anaconda3/envs/JupyterSystemEnv/lib/python3.7/site-packages/IPython/core/magics/extension.py in reload_ext(self, module_str)
     61         if not module_str:
     62             raise UsageError('Missing module name.')
---> 63         self.shell.extension_manager.reload_extension(module_str)

~/anaconda3/envs/JupyterSystemEnv/lib/python3.7/site-packages/IPython/core/extensions.py in reload_extension(self, module_str)
    128                 self.loaded.add(module_str)
    129         else:
--> 130             self.load_extension(module_str)
    131 
    132     def _call_load_ipython_extension(self, mod):

~/anaconda3/envs/JupyterSystemEnv/lib/python3.7/site-packages/IPython/core/extensions.py in load_extension(self, module_str)
     85                               dir=compress_user(self.ipython_extension_dir)))
     86             mod = sys.modules[module_str]
---> 87             if self._call_load_ipython_extension(mod):
     88                 self.loaded.add(module_str)
     89             else:

~/anaconda3/envs/JupyterSystemEnv/lib/python3.7/site-packages/IPython/core/extensions.py in _call_load_ipython_extension(self, mod)
    132     def _call_load_ipython_extension(self, mod):
    133         if hasattr(mod, 'load_ipython_extension'):
--> 134             mod.load_ipython_extension(self.shell)
    135             return True
    136 

~/anaconda3/envs/JupyterSystemEnv/lib/python3.7/site-packages/sparkmagic/magics/remotesparkmagics.py in load_ipython_extension(ip)
    322 
    323 def load_ipython_extension(ip):
--> 324     ip.register_magics(RemoteSparkMagics)

~/anaconda3/envs/JupyterSystemEnv/lib/python3.7/site-packages/IPython/core/magic.py in register(self, *magic_objects)
    403             if isinstance(m, type):
    404                 # If we're given an uninstantiated class
--> 405                 m = m(shell=self.shell)
    406 
    407             # Now that we have an instance, we can register it and update the

~/anaconda3/envs/JupyterSystemEnv/lib/python3.7/site-packages/sparkmagic/magics/remotesparkmagics.py in __init__(self, shell, data, widget)
     40         if widget is None:
     41             widget = MagicsControllerWidget(
---> 42                 self.spark_controller, IpyWidgetFactory(), self.ipython_display
     43             )
     44         self.manage_widget = widget

~/anaconda3/envs/JupyterSystemEnv/lib/python3.7/site-packages/sparkmagic/controllerwidget/magicscontrollerwidget.py in __init__(self, spark_controller, ipywidget_factory, ipython_display, endpoints)
     26         self.endpoints = endpoints
     27 
---> 28         self._refresh()
     29 
     30     def run(self):

~/anaconda3/envs/JupyterSystemEnv/lib/python3.7/site-packages/sparkmagic/controllerwidget/magicscontrollerwidget.py in _refresh(self)
     85             self.endpoints,
     86             self.endpoints_dropdown_widget,
---> 87             self._refresh,
     88         )
     89         self.manage_endpoint = ManageEndpointWidget(

~/anaconda3/envs/JupyterSystemEnv/lib/python3.7/site-packages/sparkmagic/controllerwidget/addendpointwidget.py in __init__(self, spark_controller, ipywidget_factory, ipython_display, endpoints, endpoints_dropdown_widget, refresh_method)
     32             events_handler_module = importlib.import_module(module)
     33             auth_class = getattr(events_handler_module, class_name)
---> 34             self.auth_instances[auth] = auth_class()
     35 
     36         self.auth_type = self.ipywidget_factory.get_dropdown(

~/anaconda3/envs/JupyterSystemEnv/lib/python3.7/site-packages/sparkmagic/auth/basic.py in __init__(self, parsed_attributes)
     33             self.password = "password"
     34         HTTPBasicAuth.__init__(self, self.username, self.password)
---> 35         Authenticator.__init__(self, parsed_attributes)
     36 
     37     def get_widgets(self, widget_width):

~/anaconda3/envs/JupyterSystemEnv/lib/python3.7/site-packages/sparkmagic/auth/customauth.py in __init__(self, parsed_attributes)
     22         else:
     23             self.url = "http://example.com/livy"
---> 24         self.widgets = self.get_widgets(WIDGET_WIDTH)
     25 
     26     def get_widgets(self, widget_width):

~/anaconda3/envs/JupyterSystemEnv/lib/python3.7/site-packages/sparkmagic/auth/basic.py in get_widgets(self, widget_width)
     50         )
     51 
---> 52         self.password_widget = ipywidget_factory.get_password(
     53             description="Password:", value=self.password, width=widget_width
     54         )

AttributeError: 'IpyWidgetFactory' object has no attribute 'get_password'

Versions:

• SparkMagic: 0.20.0
• Livy (if you know it)
• Spark

Additional context Add any other context about the problem here.

[devstein]

Hi @ryerobinson thanks for making an issue and providing context. Can you check if this is fixed on the latest Sparkmagic version (0.20.3)?