Closed saadirtza closed 7 years ago
Sorry copied the wrong code.. It is:
import tensorflow as tf import numpy as np import cPickle as pkl from sklearn.manifold import TSNE
from flip_gradient import flip_gradient from utils import * from numpy import genfromtxt import pdb
from tensorflow.examples.tutorials.mnist import input_data
mnist_train = genfromtxt('iv_train.csv', delimiter=',') mnist_test = genfromtxt('iv_train.csv', delimiter=',')
mnist_train_labels = genfromtxt('iv_train_label.csv', delimiter=',') mnist_test_labels = genfromtxt('iv_train_label.csv', delimiter=',')
mnist_train = mnist_train.reshape(5119, 20, 20, 1) mnist_train = np.concatenate([mnist_train, mnist_train, mnist_train], 3) mnist_test = mnist_test.reshape(5119, 20, 20, 1) mnist_test = np.concatenate([mnist_test, mnist_test, mnist_test], 3) print 'Source train data size',mnist_train.shape print 'Source test data size',mnist_train.shape
mnistm_train = genfromtxt('iv_test_half.csv', delimiter=',') mnistm_test = genfromtxt('iv_test_half.csv', delimiter=',') mnistm_valid = genfromtxt('iv_test_half.csv', delimiter=',')
mnistm_train = mnistm_train.reshape(9000, 20, 20, 1) mnistm_train = np.concatenate([mnistm_train, mnistm_train, mnistm_train], 3) mnistm_test = mnistm_test.reshape(9000, 20, 20, 1) mnistm_test = np.concatenate([mnistm_test, mnistm_test, mnistm_test], 3) mnistm_valid = mnistm_valid.reshape(9000, 20, 20, 1)
print 'Source train data size',mnistm_train.shape print 'Source test data size',mnistm_test.shape print 'Source test data size',mnistm_valid.shape
pixel_mean = np.vstack([mnist_train, mnistm_train]).mean((0, 1, 2))
num_test = 500 combined_test_imgs = np.vstack([mnist_test[:num_test], mnistm_test[:num_test]]) combined_test_labels = np.vstack([mnist_test_labels[:num_test], mnist_test_labels[:num_test]]) combined_test_domain = np.vstack([np.tile([1., 0.], [num_test, 1]), np.tile([0., 1.], [num_test, 1])])
batch_size = 12
class MNISTModel(object): """Simple MNIST domain adaptation model.""" def init(self): self._build_model()
def _build_model(self):
self.X = tf.placeholder(tf.uint8, [None, 20, 20, 3])
self.y = tf.placeholder(tf.float32, [None, 20])
self.domain = tf.placeholder(tf.float32, [None, 2])
self.l = tf.placeholder(tf.float32, [])
self.train = tf.placeholder(tf.bool, [])
X_input = tf.cast(self.X, tf.float32) - pixel_mean
# CNN model for feature extraction
with tf.variable_scope('feature_extractor'):
W_conv0 = weight_variable([5, 5, 3, 32])
b_conv0 = bias_variable([32])
h_conv0 = tf.nn.relu(conv2d(X_input, W_conv0) + b_conv0)
h_pool0 = max_pool_2x2(h_conv0)
W_conv1 = weight_variable([5, 5, 32, 48])
b_conv1 = bias_variable([48])
h_conv1 = tf.nn.relu(conv2d(h_pool0, W_conv1) + b_conv1)
h_pool1 = max_pool_2x2(h_conv1)
# The domain-invariant feature
self.feature = tf.reshape(h_pool1, [-1, 4*8*75])
# MLP for class prediction
with tf.variable_scope('label_predictor'):
# Switches to route target examples (second half of batch) differently
# depending on train or test mode.
all_features = lambda: self.feature
source_features = lambda: tf.slice(self.feature, [0, 0], [batch_size, -1])
classify_feats = tf.cond(self.train, source_features, all_features)
all_labels = lambda: self.y
source_labels = lambda: tf.slice(self.y, [0, 0], [batch_size, -1])
self.classify_labels = tf.cond(self.train, source_labels, all_labels)
W_fc0 = weight_variable([4 * 8 * 75, 100])
b_fc0 = bias_variable([100])
h_fc0 = tf.nn.relu(tf.matmul(classify_feats, W_fc0) + b_fc0)
W_fc1 = weight_variable([100, 100])
b_fc1 = bias_variable([100])
h_fc1 = tf.nn.relu(tf.matmul(h_fc0, W_fc1) + b_fc1)
W_fc2 = weight_variable([100, 20])
b_fc2 = bias_variable([20])
#logits = tf.placeholder(tf.float32, [12, 20])
logits = tf.matmul(h_fc1, W_fc2) + b_fc2
print 'logit_size', logits.get_shape()
self.pred = tf.nn.softmax(logits)
self.pred_loss = tf.nn.softmax_cross_entropy_with_logits(logits, self.classify_labels)
# Small MLP for domain prediction with adversarial loss
with tf.variable_scope('domain_predictor'):
# Flip the gradient when backpropagating through this operation
feat = flip_gradient(self.feature, self.l)
d_W_fc0 = weight_variable([4*8*75, 100])
d_b_fc0 = bias_variable([100])
d_h_fc0 = tf.nn.relu(tf.matmul(feat, d_W_fc0) + d_b_fc0)
d_W_fc1 = weight_variable([100, 2])
d_b_fc1 = bias_variable([2])
d_logits = tf.matmul(d_h_fc0, d_W_fc1) + d_b_fc1
self.domain_pred = tf.nn.softmax(d_logits)
self.domain_loss = tf.nn.softmax_cross_entropy_with_logits(d_logits, self.domain)
graph = tf.get_default_graph() with graph.as_default(): model = MNISTModel()
learning_rate = tf.placeholder(tf.float32, [])
pred_loss = tf.reduce_mean(model.pred_loss)
domain_loss = tf.reduce_mean(model.domain_loss)
total_loss = pred_loss + domain_loss
regular_train_op = tf.train.MomentumOptimizer(learning_rate, 0.9).minimize(pred_loss)
dann_train_op = tf.train.MomentumOptimizer(learning_rate, 0.9).minimize(total_loss)
# Evaluation
correct_label_pred = tf.equal(tf.argmax(model.classify_labels, 1), tf.argmax(model.pred, 1))
label_acc = tf.reduce_mean(tf.cast(correct_label_pred, tf.float32))
correct_domain_pred = tf.equal(tf.argmax(model.domain, 1), tf.argmax(model.domain_pred, 1))
domain_acc = tf.reduce_mean(tf.cast(correct_domain_pred, tf.float32))
num_steps = 8600
def train_and_evaluate(training_mode, graph, model, verbose=False): """Helper to run the model with different training modes."""
with tf.Session(graph=graph) as sess:
tf.initialize_all_variables().run()
# Batch generators
gen_source_batch = batch_generator(
[mnist_train, mnist_train_labels], batch_size)
gen_target_batch = batch_generator(
[mnistm_train, mnist_train_labels], batch_size)
gen_source_only_batch = batch_generator(
[mnist_train, mnist_train_labels], batch_size)
gen_target_only_batch = batch_generator(
[mnistm_train, mnist_train_labels], batch_size)
domain_labels = np.vstack([np.tile([1., 0.], [batch_size, 1]),
np.tile([0., 1.], [batch_size, 1])])
# Training loop
for i in range(num_steps):
# Adaptation param and learning rate schedule as described in the paper
p = float(i) / num_steps
l = 2. / (1. + np.exp(-10. * p)) - 1
lr = 0.01 / (1. + 10 * p)**0.75
# Training step
if training_mode == 'dann':
#pdb.set_trace()
X0, y0 = gen_source_batch.next()
pdb.set_trace()
X1, y1 = gen_target_batch.next()
X = np.vstack([X0, X1])
y = np.vstack([y0, y1])
_, batch_loss, dloss, ploss, d_acc, p_acc = \
sess.run([dann_train_op, total_loss, domain_loss, pred_loss, domain_acc, label_acc],
feed_dict={model.X: X, model.y: y, model.domain: domain_labels,
model.train: True, model.l: l, learning_rate: lr})
if verbose and i % 100 == 0:
print 'loss: %f d_acc: %f p_acc: %f p: %f l: %f lr: %f' % \
(batch_loss, d_acc, p_acc, p, l, lr)
elif training_mode == 'source':
X, y = gen_source_only_batch.next()
print 'label_size', y.shape
pdb.set_trace()
_, batch_loss = sess.run([regular_train_op, pred_loss],
feed_dict={model.X: X, model.y: y, model.train: False,
model.l: l, learning_rate: lr})
elif training_mode == 'target':
X, y = gen_target_only_batch.next()
_, batch_loss = sess.run([regular_train_op, pred_loss],
feed_dict={model.X: X, model.y: y, model.train: False,
model.l: l, learning_rate: lr})
# Compute final evaluation on test data
source_acc = sess.run(label_acc,
feed_dict={model.X: mnist_test, model.y: mnist_test_labels,
model.train: False})
target_acc = sess.run(label_acc,
feed_dict={model.X: mnistm_test, model.y: mnist_test_labels,
model.train: False})
test_domain_acc = sess.run(domain_acc,
feed_dict={model.X: combined_test_imgs,
model.domain: combined_test_domain, model.l: 1.0})
test_emb = sess.run(model.feature, feed_dict={model.X: combined_test_imgs})
return source_acc, target_acc, test_domain_acc, test_emb
print '\nSource only training' source_acc, targetacc, , source_only_emb = train_and_evaluate('source', graph, model) print 'Source (MNIST) accuracy:', source_acc print 'Target (MNIST-M) accuracy:', target_acc pdb.set_trace()
print '\nDomain adaptation training' source_acc, target_acc, d_acc, dann_emb = train_and_evaluate('dann', graph, model) print 'Source (MNIST) accuracy:', source_acc print 'Target (MNIST-M) accuracy:', target_acc print 'Domain accuracy:', d_acc
tsne = TSNE(perplexity=30, n_components=2, init='pca', n_iter=3000) source_only_tsne = tsne.fit_transform(source_only_emb)
tsne = TSNE(perplexity=30, n_components=2, init='pca', n_iter=3000) dann_tsne = tsne.fit_transform(dann_emb)
plot_embedding(source_only_tsne, combined_test_labels.argmax(1), combined_test_domain.argmax(1), 'Source only') plot_embedding(dann_tsne, combined_test_labels.argmax(1), combined_test_domain.argmax(1), 'Domain Adaptation')
The model graph expects a batch with 50% source and 50% target examples, domain labels for all examples, and class labels for the source examples (and all examples in testing mode). It seems like something is wrong with your batching. It is hard for me to debug this code, esp with the formatting, but note that in the original script batches are build by concatenating from two batch iterators that produce data with half the batch size:
gen_source_batch = batch_generator(
[mnist_train, mnist.train.labels], batch_size / 2)
gen_target_batch = batch_generator(
[mnistm_train, mnist.train.labels], batch_size / 2)
I noticed this is missing in your code, so that may be a place to look. Good luck.
Thanks alot for quick reply.. its working now.. :)
Regards
Hi,
Thanks for your code. I don't understand why you use mnistt labels with mnistm data-set??
like here
gen_target_batch = batch_generator( [mnistm_train, mnist.train.labels], batch_size / 2)
it is not sup[posed to be minstm labels??
Hi Pumpikano,
With your help i was able to run the code on MNIST dataset. i am trying to train the model on another dataset. i have tried to modified the code accordingly, i will be very thankful if you can help me to solve this issue. The code is
matplotlib inline
import tensorflow as tf import numpy as np import cPickle as pkl from sklearn.manifold import TSNE
from flip_gradient import flip_gradient from utils import * from numpy import genfromtxt import pdb
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets('MNIST_data', one_hot=True)
Process MNIST
mnist_train = (mnist.train.images > 0).reshape(55000, 28, 28, 1).astype(np.uint8) * 255
print(mnist_train.shape)
mnist_train = np.concatenate([mnist_train, mnist_train, mnist_train], 3)
mnist_test = (mnist.test.images > 0).reshape(10000, 28, 28, 1).astype(np.uint8) * 255
mnist_test = np.concatenate([mnist_test, mnist_test, mnist_test], 3)
mnist_train = genfromtxt('iv_train.csv', delimiter=',') mnist_test = genfromtxt('iv_train.csv', delimiter=',')
mnist_train_labels = genfromtxt('iv_train_label.csv', delimiter=',') mnist_test_labels = genfromtxt('iv_train_label.csv', delimiter=',')
mnist_train = mnist_train.reshape(5119, 20, 20, 1) mnist_train = np.concatenate([mnist_train, mnist_train, mnist_train], 3) mnist_test = mnist_test.reshape(5119, 20, 20, 1) mnist_test = np.concatenate([mnist_test, mnist_test, mnist_test], 3) print 'Source train data size',mnist_train.shape print 'Source test data size',mnist_train.shape
pdb.set_trace()
Load MNIST-M
mnistm = pkl.load(open('mnistm_data.pkl'))
mnistm_train = mnistm['train']
mnistm_test = mnistm['test']
mnistm_valid = mnistm['valid']
mnistm_train = genfromtxt('iv_test_half.csv', delimiter=',') mnistm_test = genfromtxt('iv_test_half.csv', delimiter=',') mnistm_valid = genfromtxt('iv_test_half.csv', delimiter=',')
mnistm_train = mnistm_train.reshape(9000, 20, 20, 1) mnistm_train = np.concatenate([mnistm_train, mnistm_train, mnistm_train], 3) mnistm_test = mnistm_test.reshape(9000, 20, 20, 1) mnistm_test = np.concatenate([mnistm_test, mnistm_test, mnistm_test], 3) mnistm_valid = mnistm_valid.reshape(9000, 20, 20, 1)
mnistm_valid = np.concatenate([mnistm_valid, mnistm_valid, mnistm_valid], 3)
print 'Source train data size',mnistm_train.shape print 'Source test data size',mnistm_test.shape print 'Source test data size',mnistm_valid.shape
pdb.set_trace()
Compute pixel mean for normalizing data
pixel_mean = np.vstack([mnist_train, mnistm_train]).mean((0, 1, 2))
Create a mixed dataset for TSNE visualization
num_test = 500 combined_test_imgs = np.vstack([mnist_test[:num_test], mnistm_test[:num_test]]) combined_test_labels = np.vstack([mnist_test_labels[:num_test], mnist_test_labels[:num_test]]) combined_test_domain = np.vstack([np.tile([1., 0.], [num_test, 1]), np.tile([0., 1.], [num_test, 1])])
imshow_grid(mnist_train)
imshow_grid(mnistm_train)
batch_size = 12
class MNISTModel(object): """Simple MNIST domain adaptation model.""" def init(self): self._build_model()
Build the model graph
graph = tf.get_default_graph() with graph.as_default(): model = MNISTModel()
Params
num_steps = 8600
def train_and_evaluate(training_mode, graph, model, verbose=False): """Helper to run the model with different training modes."""
print '\nSource only training' source_acc, targetacc, , source_only_emb = train_and_evaluate('source', graph, model) print 'Source (MNIST) accuracy:', source_acc print 'Target (MNIST-M) accuracy:', target_acc pdb.set_trace()
print '\nDomain adaptation training' source_acc, target_acc, d_acc, dann_emb = train_and_evaluate('dann', graph, model) print 'Source (MNIST) accuracy:', source_acc print 'Target (MNIST-M) accuracy:', target_acc print 'Domain accuracy:', d_acc
tsne = TSNE(perplexity=30, n_components=2, init='pca', n_iter=3000) source_only_tsne = tsne.fit_transform(source_only_emb)
tsne = TSNE(perplexity=30, n_components=2, init='pca', n_iter=3000) dann_tsne = tsne.fit_transform(dann_emb)
plot_embedding(source_only_tsne, combined_test_labels.argmax(1), combined_test_domain.argmax(1), 'Source only') plot_embedding(dann_tsne, combined_test_labels.argmax(1), combined_test_domain.argmax(1), 'Domain Adaptation')
The error i am getting is:
Source train data size (5119, 20, 20, 3) Source test data size (5119, 20, 20, 3) Source train data size (9000, 20, 20, 3) Source test data size (9000, 20, 20, 3) Source test data size (9000, 20, 20, 1) logit_size (?, 20) logit_size (?, 20) logit_size (?, 20)
Source only training label_size (12, 20)
The label size is (12,20) but logit size is (?,20).. How to fix this issue? Many Thanks Regards Saad