Error in gpflow.optimizers.NaturalGradient._natgrad_steps()

Short summary

I believe the reference to self._name in the NaturalGradient._natgrad_steps() method should be changed to self.name

An example where the error shows up

I was working through the Heteroskedastic Likelihood and Multi-Latent GP example in the latest version of the documentation and ran into the following error when calling optimisation_step()

AttributeError: 'NaturalGradient' object has no attribute '_name'

Based on this S.O. post, at least one other user has encountered the same issue. I found that the above change to gpflow/optimizers/natgrad.py resolved the issue.

The code from that example

# [1] Import packages
import matplotlib.pyplot as plt
import numpy as np
import tensorflow as tf
import tensorflow_probability as tfp

import gpflow as gpf

# [2]: Generate data to be modeled
N = 1001

np.random.seed(0)
tf.random.set_seed(0)

# Build inputs X
X = np.linspace(0, 4 * np.pi, N)[:, None]  # X must be of shape [N, 1]

# Deterministic functions in place of latent ones
f1 = np.sin
f2 = np.cos

# Use transform = exp to ensure positive-only scale values
transform = np.exp

# Compute loc and scale as functions of input X
loc = f1(X)
scale = transform(f2(X))

# Sample outputs Y from Gaussian Likelihood
Y = np.random.normal(loc, scale)

# [3]: Plot the data generated above to see what they look like
def plot_distribution(X, Y, loc, scale):
    plt.figure(figsize=(15, 5))
    x = X.squeeze()
    for k in (1, 2):
        lb = (loc - k * scale).squeeze()
        ub = (loc + k * scale).squeeze()
        plt.fill_between(x, lb, ub, color="silver", alpha=1 - 0.05 * k ** 3)
    plt.plot(x, lb, color="silver")
    plt.plot(x, ub, color="silver")
    plt.plot(X, loc, color="black")
    plt.scatter(X, Y, color="gray", alpha=0.8)
    plt.show()
    plt.close()

#plot_distribution(X, Y, loc, scale)

# [4]: Set up the likelihood function for the model we want
likelihood = gpf.likelihoods.HeteroskedasticTFPConditional(
    distribution_class=tfp.distributions.Normal,  # Gaussian Likelihood
    scale_transform=tfp.bijectors.Exp(),  # Exponential Transform
)

print(f"Likelihood's expected latent_dim: {likelihood.latent_dim}")

# [5]: Specify the kernels of the mean and variance GPs
kernel = gpf.kernels.SeparateIndependent(
    [
        gpf.kernels.SquaredExponential(),  # This is k1, the kernel of f1
        gpf.kernels.SquaredExponential(),  # this is k2, the kernel of f2
    ]
)

# [6]: We will use the SVGP model which requires inducing points
M = 20  # Number of inducing variables for each f_i

# Initial inducing points position Z
Z = np.linspace(X.min(), X.max(), M)[:, None]  # Z must be of shape [M, 1]

inducing_variable = gpf.inducing_variables.SeparateIndependentInducingVariables(
    [
        gpf.inducing_variables.InducingPoints(Z),  # This is U1 = f1(Z1)
        gpf.inducing_variables.InducingPoints(Z),  # This is U2 = f2(Z2)
    ]
)

# [7]: Build the SVGP model using the components instantiated above
model = gpf.models.SVGP(
    kernel=kernel,
    likelihood=likelihood,
    inducing_variable=inducing_variable,
    num_latent_gps=likelihood.latent_dim,
)

#model

# [8]: Build optimizers (NatGrad + Adam)
data = (X, Y)
loss_fn = model.training_loss_closure(data)

gpf.utilities.set_trainable(model.q_mu, False)
gpf.utilities.set_trainable(model.q_sqrt, False)

variational_vars = [(model.q_mu, model.q_sqrt)]
natgrad_opt = gpf.optimizers.NaturalGradient(gamma=0.1)

adam_vars = model.trainable_variables
adam_opt = tf.optimizers.Adam(0.01)

@tf.function
def optimisation_step():
    natgrad_opt.minimize(loss_fn, variational_vars)
    adam_opt.minimize(loss_fn, adam_vars)

# [9]: Run optimization loop
epochs = 100
log_freq = 20

for epoch in range(1, epochs + 1):
    optimisation_step()  # Here's where the error occurs

    # For every 'log_freq' epochs, print the epoch and plot the predictions against the data
    if epoch % log_freq == 0 and epoch > 0:
        print(f"Epoch {epoch} - Loss: {loss_fn().numpy() : .4f}")
        Ymean, Yvar = model.predict_y(X)
        Ymean = Ymean.numpy().squeeze()
        Ystd = tf.sqrt(Yvar).numpy().squeeze()
        plot_distribution(X, Y, Ymean, Ystd)

Error traceback

I get the following error message using the current version of gpflow:

Traceback (most recent call last):
  File "C:\Users\scw8734\Desktop\gp_modeling\gpflow_heteroskedastic.py", line 116, in <module>
    optimisation_step()
  File "C:\Users\scw8734\Anaconda3\envs\gpflow\lib\site-packages\tensorflow\python\util\traceback_utils.py", line 153, in error_handler
    raise e.with_traceback(filtered_tb) from None
  File "C:\Users\scw8734\AppData\Local\Temp\__autograph_generated_filea1197q1y.py", line 8, in tf__optimisation_step
    ag__.converted_call(ag__.ld(natgrad_opt).minimize, (ag__.ld(loss_fn), ag__.ld(variational_vars)), None, fscope)
  File "C:\Users\scw8734\AppData\Local\Temp\__autograph_generated_filem1g113m3.py", line 13, in tf__wrapped_method
    retval_ = ag__.converted_call(ag__.ld(wrapped_function), (ag__.ld(self),) + tuple(ag__.ld(args)), dict(**ag__.ld(kwargs)), fscope)
  File "C:\Users\scw8734\AppData\Local\Temp\__autograph_generated_filegh3cmqhi.py", line 171, in tf__wrapped_function
    ag__.if_stmt(ag__.not_(ag__.converted_call(ag__.ld(get_enable_check_shapes), (), None, fscope)), if_body_4, else_body_4, get_state_7, set_state_7, ('arg_map', 'arg_map[RESULT_TOKEN]', 'checker', 'do_return', 'retval_'), 5)
  File "C:\Users\scw8734\AppData\Local\Temp\__autograph_generated_filegh3cmqhi.py", line 28, in if_body_4
    retval_ = ag__.converted_call(ag__.ld(func), tuple(ag__.ld(args)), dict(**ag__.ld(kwargs)), fscope)
  File "C:\Users\scw8734\AppData\Local\Temp\__autograph_generated_filel0u9lm6w.py", line 28, in tf__minimize
    ag__.converted_call(ag__.ld(self)._natgrad_steps, (ag__.ld(loss_fn), ag__.ld(parameters)), None, fscope)
  File "C:\Users\scw8734\AppData\Local\Temp\__autograph_generated_filem1g113m3.py", line 13, in tf__wrapped_method
    retval_ = ag__.converted_call(ag__.ld(wrapped_function), (ag__.ld(self),) + tuple(ag__.ld(args)), dict(**ag__.ld(kwargs)), fscope)
  File "C:\Users\scw8734\AppData\Local\Temp\__autograph_generated_filegh3cmqhi.py", line 171, in tf__wrapped_function
    ag__.if_stmt(ag__.not_(ag__.converted_call(ag__.ld(get_enable_check_shapes), (), None, fscope)), if_body_4, else_body_4, get_state_7, set_state_7, ('arg_map', 'arg_map[RESULT_TOKEN]', 'checker', 'do_return', 'retval_'), 5)
  File "C:\Users\scw8734\AppData\Local\Temp\__autograph_generated_filegh3cmqhi.py", line 28, in if_body_4
    retval_ = ag__.converted_call(ag__.ld(func), tuple(ag__.ld(args)), dict(**ag__.ld(kwargs)), fscope)
  File "C:\Users\scw8734\AppData\Local\Temp\__autograph_generated_filevzp834lq.py", line 23, in tf___natgrad_steps
    with ag__.ld(tf).name_scope(f'{ag__.ld(self)._name}/natural_gradient_steps'):
AttributeError: in user code:

    File "C:\Users\scw8734\Desktop\gp_modeling\gpflow_heteroskedastic.py", line 107, in optimisation_step  *
        natgrad_opt.minimize(loss_fn, variational_vars)
    File "C:\Users\scw8734\Anaconda3\envs\gpflow\lib\site-packages\check_shapes\integration\tf.py", line 208, in wrapped_method  *
        return wrapped_function(self, *args, **kwargs)
    File "C:\Users\scw8734\Anaconda3\envs\gpflow\lib\site-packages\check_shapes\decorator.py", line 120, in wrapped_function  *
        return func(*args, **kwargs)
    File "C:\Users\scw8734\Anaconda3\envs\gpflow\lib\site-packages\gpflow\optimizers\natgrad.py", line 236, in minimize  *
        self._natgrad_steps(loss_fn, parameters)
    File "C:\Users\scw8734\Anaconda3\envs\gpflow\lib\site-packages\check_shapes\integration\tf.py", line 208, in wrapped_method  *
        return wrapped_function(self, *args, **kwargs)
    File "C:\Users\scw8734\Anaconda3\envs\gpflow\lib\site-packages\check_shapes\decorator.py", line 120, in wrapped_function  *
        return func(*args, **kwargs)
    File "C:\Users\scw8734\Anaconda3\envs\gpflow\lib\site-packages\gpflow\optimizers\natgrad.py", line 266, in _natgrad_steps  *
        with tf.name_scope(f"{self._name}/natural_gradient_steps"):

    AttributeError: 'NaturalGradient' object has no attribute '_name'

Expected behavior

The above code should run without error, generating multiple plots over the course of the for loop with output to the console like this:

Epoch 20 - Loss:  1471.8678
Epoch 40 - Loss:  1452.4426
Epoch 60 - Loss:  1450.7381
Epoch 80 - Loss:  1450.0566
Epoch 100 - Loss:  1449.6073

System information

Package versions:

matplotlib: 3.7.1
numpy: 1.23.5
tensorflow: 2.12.0
tensorflow_probability: 0.19.0
gpflow: 2.7.1

Please let me know if this solution is not appropriate! I'm still just learning how to use gpflow (and getting refreshed on Python).

GPflow / GPflow