Conditional example_gan.py

[apozas]

Hi,

I am trying to do a version of example_gan.py in which both generator and discriminator admit a label that tells which number to generate/identify. This requires the the label to be untouched when the input is passed through eliminate_z. I thought of the following modification:

def eliminate_z(gan, latent_sampling, label_dim):
    """
    Eliminate z from GAN using latent_sampling
    :param gan: model with 2 inputs: z, x
    :param latent_sampling: layer that samples z with same batch size as x
    :return: Model x -> gan(latent_sampling(x), x)
    """
    x = gan.inputs[1]
    l = x[:,-label_dim:]
    z = concatenate([latent_sampling(x), l])
    model = Model(inputs=x, outputs=fix_names(gan([z, x]), gan.output_names), name=gan.name)
    return model

However, when running the following example (which is in essence example_gan.py with the modifications needed to do a conditional GAN)

import numpy as np
import os
from keras.layers import Reshape, Flatten, LeakyReLU, Activation, concatenate, Input, Lambda, Concatenate
from keras.models import Sequential, Model
from keras.optimizers import Adam, SGD
from keras.callbacks import TensorBoard
from keras.utils import to_categorical
from keras_adversarial.image_grid_callback import ImageGridCallback
from keras_adversarial import AdversarialModel, simple_gan, gan_targets
from keras_adversarial import normal_latent_sampling, AdversarialOptimizerSimultaneous
from keras_adversarial.legacy import l1l2, Dense, fit
import keras.backend as K
from keras.datasets import mnist

def model_generator(latent_dim, label_dim, image_shape, hidden_dim=1024, reg=lambda: l1l2(1e-5, 1e-5)):
    inputs = (Input(shape=(latent_dim + label_dim,), name='generator_input'))
    y = (Lambda(lambda x: x[:,-label_dim:], output_shape=(label_dim,)))(inputs)
    T = (Dense(int(hidden_dim / 4), name="generator_h1", W_regularizer=reg()))(inputs)
    T = (LeakyReLU(0.2))(T)
    T = (Dense(int(hidden_dim / 2), name="generator_h2", W_regularizer=reg()))(T)
    T = (LeakyReLU(0.2))(T)
    T = (Dense(hidden_dim, name="generator_h3", W_regularizer=reg()))(T)
    T = (LeakyReLU(0.2))(T)
    T = (Dense(np.prod(image_shape), name="generator_x_flat", W_regularizer=reg()))(T)
    T = (Activation('sigmoid'))(T)
    T = concatenate([T, y])    
    model = Model(inputs=inputs, outputs=T, name="generator")
    return model

def model_discriminator(imagepluslabel_shape, hidden_dim=1024, reg=lambda: l1l2(1e-5, 1e-5), output_activation="sigmoid"):
    inputs = (Input(shape=imagepluslabel_shape, name='discriminator_input'))
    T = (Dense(hidden_dim, name="discriminator_h1", W_regularizer=reg()))(inputs)
    T = (LeakyReLU(0.2))(T)
    T = (Dense(int(hidden_dim / 2), name="discriminator_h2", W_regularizer=reg()))(T)
    T = (LeakyReLU(0.2))(T)
    T = (Dense(int(hidden_dim / 4), name="discriminator_h3", W_regularizer=reg()))(T)
    T = (LeakyReLU(0.2))(T)
    T = (Dense(1, name="discriminator_y", W_regularizer=reg()))(T)
    T = (Activation(output_activation))(T)
    model = Model(inputs=inputs, outputs=T, name="discriminator")
    return model

def eliminate_z(gan, latent_sampling, label_dim):
    """
    Eliminate z from GAN using latent_sampling
    :param gan: model with 2 inputs: z, x
    :param latent_sampling: layer that samples z with same batch size as x
    :return: Model x -> gan(latent_sampling(x), x)
    """
    x = gan.inputs[1]
    l = x[:,-label_dim:]
    z = concatenate([latent_sampling(x), l])
    model = Model(inputs=x, outputs=fix_names(gan([z, x]), gan.output_names), name=gan.name)
    return model

def simple_gan(generator, discriminator, latent_sampling, label_dim):
    # build basic gan
    gan = build_gan(generator, discriminator)
    # generate z on gpu, eliminate one input
    if latent_sampling is None:
        return gan
    else:
        return eliminate_z(gan, latent_sampling, label_dim)

# z \in R^100
latent_dim = 100
# x \in R^{28x28}
image_shape = (int(28 * 28),)
# label in R^10
label_dim = 10

imagepluslabel_shape = (int(28 * 28 + label_dim),)
# generator ((z,l) -> (x,l))
generator = model_generator(latent_dim, label_dim, image_shape)
# discriminator ((x,l) -> y)
discriminator = model_discriminator(imagepluslabel_shape)
simple_gan(generator, discriminator, normal_latent_sampling((latent_dim,)), label_dim)

I get the following error:

---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
<ipython-input-40-432f035f7981> in <module>()
----> 1 simple_gan(generator, discriminator, normal_latent_sampling((latent_dim,)), label_dim)

<ipython-input-38-20b3c0a977c6> in simple_gan(generator, discriminator, latent_sampling, label_dim)
     34         return gan
     35     else:
---> 36         return eliminate_z(gan, latent_sampling, label_dim)
     37 
     38 

<ipython-input-38-20b3c0a977c6> in eliminate_z(gan, latent_sampling, label_dim)
     23     l = x[:,-label_dim:]
     24     z = concatenate([latent_sampling(x), l])
---> 25     model = Model(inputs=x, outputs=fix_names(gan([z, x]), gan.output_names), name=gan.name)
     26     return model
     27 

c:\python\python35\lib\site-packages\keras\legacy\interfaces.py in wrapper(*args, **kwargs)
     86                 warnings.warn('Update your `' + object_name +
     87                               '` call to the Keras 2 API: ' + signature, stacklevel=2)
---> 88             return func(*args, **kwargs)
     89         wrapper._legacy_support_signature = inspect.getargspec(func)
     90         return wrapper

c:\python\python35\lib\site-packages\keras\engine\topology.py in __init__(self, inputs, outputs, name)
   1652         for x in self.outputs:
   1653             seen_nodes = set()
-> 1654             build_map_of_graph(x, seen_nodes, depth=0)
   1655 
   1656         # Build a dict {depth: list of nodes with this depth}

c:\python\python35\lib\site-packages\keras\engine\topology.py in build_map_of_graph(tensor, seen_nodes, depth, layer, node_index, tensor_index)
   1648                 if node_marker not in seen_nodes:
   1649                     build_map_of_graph(x, seen_nodes, current_depth + 1,
-> 1650                                        layer, node_index, tensor_index)
   1651 
   1652         for x in self.outputs:

c:\python\python35\lib\site-packages\keras\engine\topology.py in build_map_of_graph(tensor, seen_nodes, depth, layer, node_index, tensor_index)
   1648                 if node_marker not in seen_nodes:
   1649                     build_map_of_graph(x, seen_nodes, current_depth + 1,
-> 1650                                        layer, node_index, tensor_index)
   1651 
   1652         for x in self.outputs:

c:\python\python35\lib\site-packages\keras\engine\topology.py in build_map_of_graph(tensor, seen_nodes, depth, layer, node_index, tensor_index)
   1643                 node_index = node.node_indices[i]
   1644                 tensor_index = node.tensor_indices[i]
-> 1645                 next_node = layer.inbound_nodes[node_index]
   1646                 # use node_marker to prevent cycles
   1647                 node_marker = make_node_marker(next_node, current_depth + 1)

AttributeError: 'NoneType' object has no attribute 'inbound_nodes'

As far as I understand it is the Model function that is failing, but I cannot still manage to know why. Help would be very much appreciated.

[bstriner]

@apozas sorry for not getting to this question earlier.

You should build the generator and discriminator submodels with multiple inputs if you are making a CGAN. If you're doing something custom, you don't really need elimate_z.

Something like this should work for you:

xreal = Input(...)
label = Input(...)
z = normal_latent_sampling((latent_dim,))(label)
# concatenate z and label or do whatever
# calculate xgen=G(Z,L) and yreal=D(X)
# create G and D submodels
g = Model([z,label], xgen)
d = Model([xreal,label], yreal)
# create combined model
yfake = d([xgen, label])
model = Model([z, label, xreal], [yreal, yfake])
# create adversarial model
amodel = AdversarialModel(player_weights=[g.weights, d.weights], ...)

I'll try to put an example along these lines up but it is going to be a busy semester.

Cheers

[apozas]

Hi @bstriner, thank you very much for your assistance. Now I have no problems with creating the model nor compiling it, but an error pops out when I try to fit it. It seems some kind of compatibility clash with the latest versions of keras (I am using keras 2.0.8). The code that triggers the error and the error itself are the following.

import numpy as np
import os
from keras.layers import Reshape, Flatten, LeakyReLU, Activation, concatenate, Input, Lambda
from keras.models import Sequential, Model
from keras.optimizers import Adam, SGD
from keras.callbacks import TensorBoard
from keras.utils import to_categorical
from keras_adversarial.image_grid_callback import ImageGridCallback
from keras_adversarial import AdversarialModel, simple_gan, gan_targets
from keras_adversarial import normal_latent_sampling, AdversarialOptimizerSimultaneous
from keras_adversarial.legacy import l1l2, Dense, fit
import keras.backend as K
from keras.datasets import mnist

latent_dim = 100
label_dim = 10
image_shape = (int(28 * 28),)
hidden_dim = 1024
reg=lambda: l1l2(1e-5, 1e-5)
output_activation='sigmoid'

label = (Input(shape=(label_dim,), name='label'))
z = normal_latent_sampling((latent_dim,))(label)
xreal = (Input(shape=image_shape, name='discriminator_input'))

# Generator
inputs_g = (concatenate([z, label], name='input_concatenation'))
T = (Dense(int(hidden_dim / 4), name="generator_h1", W_regularizer=reg()))(inputs_g)
T = (LeakyReLU(0.2))(T)
T = (Dense(int(hidden_dim / 2), name="generator_h2", W_regularizer=reg()))(T)
T = (LeakyReLU(0.2))(T)
T = (Dense(hidden_dim, name="generator_h3", W_regularizer=reg()))(T)
T = (LeakyReLU(0.2))(T)
T = (Dense(np.prod(image_shape), name="generator_x_flat", W_regularizer=reg()))(T)
T = (Activation('sigmoid'))(T)    
generator = Model(inputs=label, outputs=[T, label], name="generator")

# Discriminator
inputs_d = (concatenate([xreal, label], name='input_concatenation'))
T = (Dense(hidden_dim, name="discriminator_h1", W_regularizer=reg()))(inputs_d)
T = (LeakyReLU(0.2))(T)
T = (Dense(int(hidden_dim / 2), name="discriminator_h2", W_regularizer=reg()))(T)
T = (LeakyReLU(0.2))(T)
T = (Dense(int(hidden_dim / 4), name="discriminator_h3", W_regularizer=reg()))(T)
T = (LeakyReLU(0.2))(T)
T = (Dense(1, name="discriminator_y", W_regularizer=reg()))(T)
T = (Activation(output_activation))(T)
discriminator = Model(inputs=[xreal, label], outputs=T, name="discriminator")

yreal = Activation("linear", name="yreal")(discriminator(discriminator.inputs))
yfake = Activation("linear", name="yfake")(discriminator(generator(generator.inputs)))
gan = Model(inputs=discriminator.inputs, outputs=[yreal, yfake])
amodel = AdversarialModel(base_model=gan,
                      player_params=[generator.trainable_weights, discriminator.trainable_weights],
                      player_names=["generator", "discriminator"])
amodel.adversarial_compile(adversarial_optimizer=AdversarialOptimizerSimultaneous,
                                              player_optimizers=['SGD', 'SGD'],
                                              loss='binary_crossentropy')

 (x_train, y_train), (x_test, y_test) = mnist.load_data()
x_train = x_train.reshape(x_train.shape[0], 784)
x_train = x_train / 255
y_train = to_categorical(y_train)
y = gan_targets(x_train.shape[0])
fit(amodel, x=[x_train, y_train], y=y)

And the error spit is

---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
<ipython-input-7-c4767d075ffa> in <module>()
----> 1 fit(amodel, x=[x_train, y_train], y=y)

c:\python\python35\lib\site-packages\keras_adversarial-0.0.3-py3.5.egg\keras_adversarial\legacy.py in 
fit(model, x, y, nb_epoch, *args, **kwargs)
     16 def fit(model, x, y, nb_epoch=10, *args, **kwargs):
     17     if keras_2:
 ---> 18         return model.fit(x, y, *args, epochs=nb_epoch, **kwargs)
     19     else:
     20         return model.fit(x, y, *args, nb_epoch=nb_epoch, **kwargs)

c:\python\python35\lib\site-packages\keras\engine\training.py in fit(self, x, y, batch_size, epochs, 
verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, 
steps_per_epoch, validation_steps, **kwargs)
   1573         else:
   1574             ins = x + y + sample_weights
-> 1575         self._make_train_function()
   1576         f = self.train_function
   1577 

c:\python\python35\lib\site-packages\keras_adversarial-0.0.3-
py3.5.egg\keras_adversarial\adversarial_model.py in _make_train_function(self)
    159                                                                                  self.optimizers,
    160                                                                                  [model.constraints for model in
--> 161                                                                                   self.layers],
    162                                                                                  self.updates,
    163                                                                                  self._function_kwargs)

c:\python\python35\lib\site-packages\keras_adversarial-0.0.3-
py3.5.egg\keras_adversarial\adversarial_model.py in <listcomp>(.0)
    158                                                                                  self.player_params,
    159                                                                                  self.optimizers,
--> 160                                                                                  [model.constraints for model in
    161                                                                                   self.layers],
    162                                                                                  self.updates,

AttributeError: 'Model' object has no attribute 'constraints'

EDIT: Indeed, the changelog of Keras 2.0.7 says

Move constraint management to be based on variable attributes. Remove the now-unused constraints attribute on layers and models (not expected to affect any user).

So indeed new versions of Keras break keras-adversarial.

EDIT 2: I have been running the program in other versions of Keras. For versions below 2.0.7 and above 2.0.3, there is another error outputting when running fit:

---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-8-5da2a0409466> in <module>()
----> 1 fit(amodel, x=[x_train, y_train], y=y, nb_epoch=3, callbacks=[], validation_split=0.1)

c:\python\python35\lib\site-packages\keras_adversarial-0.0.3-py3.5.egg\keras_adversarial\legacy.py in 
fit(model, x, y, nb_epoch, *args, **kwargs)
     16 def fit(model, x, y, nb_epoch=10, *args, **kwargs):
     17     if keras_2:
---> 18         return model.fit(x, y, *args, epochs=nb_epoch, **kwargs)
     19     else:
     20         return model.fit(x, y, *args, nb_epoch=nb_epoch, **kwargs)

c:\python\python35\lib\site-packages\keras\engine\training.py in fit(self, x, y, batch_size, epochs, 
verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, 
**kwargs)
   1411         else:
   1412             ins = x + y + sample_weights
-> 1413         self._make_train_function()
   1414         f = self.train_function
   1415 

c:\python\python35\lib\site-packages\keras_adversarial-0.0.3-
py3.5.egg\keras_adversarial\adversarial_model.py in _make_train_function(self)
    161                                                                                   self.layers],
    162                                                                                  self.updates,
--> 163                                                                                  self._function_kwargs)
    164 
    165     def _make_test_function(self):

TypeError: make_train_function() missing 1 required positional argument: 'function_kwargs'

For version 2.0.3 and below, there is a warning and an Assertion error:

c:\python\python35\lib\site-packages\keras\engine\topology.py:1519: UserWarning: Model inputs 
must come from a Keras Input layer, they cannot be the output of a previous non-Input layer. Here, a 
tensor specified as input to "model_1" was not an Input tensor, it was generated by layer generator.
Note that input tensors are instantiated via `tensor = Input(shape)`.
The tensor that caused the issue was: label:0
  str(x.name))
---------------------------------------------------------------------------
AssertionError                            Traceback (most recent call last)
<ipython-input-4-d735d78a52bb> in <module>()
      1 yreal = Activation("linear", name="yreal")(discriminator(discriminator.inputs))
      2 yfake = Activation("linear", name="yfake")(discriminator(generator(generator.inputs)))
----> 3 gan = Model(inputs=discriminator.inputs, outputs=[yreal, yfake])

c:\python\python35\lib\site-packages\keras\legacy\interfaces.py in wrapper(*args, **kwargs)
     86                 warnings.warn('Update your `' + object_name +
     87                               '` call to the Keras 2 API: ' + signature, stacklevel=2)
---> 88             return func(*args, **kwargs)
     89         wrapper._legacy_support_signature = inspect.getargspec(func)
     90         return wrapper

c:\python\python35\lib\site-packages\keras\engine\topology.py in __init__(self, inputs, outputs, name)
   1556             # It's supposed to be an input layer, so only one node
   1557             # and one tensor output.
-> 1558             assert node_index == 0
   1559             assert tensor_index == 0
   1560             self.input_layers.append(layer)

AssertionError: 

[ghost]

@bstriner any update to resolve this issue?

[rjpg]

A complete example of AC-GAN using keras-adversarial would be nice !

I have ac-gan implemented here :

https://github.com/rjpg/bftensor/blob/master/Autoencoder/src/ac-gan2.py

but it is slow ... I notice that it only uses one core ... this keras-adversarial package seams to be optimized to work with all cores ... that is why it is important.

If someone takes this task, it is important to have the accuracy metric (train/test) of the softmax on the discriminator side among with loss ..

In AC-GAN: 1 - the discriminator input is only the image : [image] 2 - the discriminator output is one softmax with N classes and one sigmoid to identify real/fake [labels,valid] 3 - the generator input is : [noise,label] (then they are multiplied to give color to the noise to be oriented to generate one class ) 4 - the generator output is only one image : [image]

when training fake images the random labels to use in the generator to create images to train must be used also in the discriminator training , something like :

....
# Adversarial ground truths
valid = np.ones((batch_size, 1))
fake = np.zeros((batch_size, 1))
...

#inside train cycle
                ...
                # real data 
                imgs = X_train[index * batch_size:(index + 1) * batch_size]
                img_labels = y_train[index * batch_size:(index + 1) * batch_size]
                #end real data

                # generated data 
                noise = np.random.normal(0, 1, (batch_size, 100))

                # The labels of the digits that the generator tries to create an
                # image representation of
                sampled_labels = np.random.randint(0, 10, (batch_size, 1))

                # Generate a half batch of new images
                gen_imgs = self.generator.predict([noise, sampled_labels])
                #end generated data

                # Train the discriminator
                d_loss_real = self.discriminator.train_on_batch(imgs, [valid, img_labels])
                d_loss_fake = self.discriminator.train_on_batch(gen_imgs, [fake, sampled_labels])

                d_loss = 0.5 * np.add(d_loss_real, d_loss_fake)
                ...

if possible :-)