dgoldenberg-audiomack
commented
3 years ago The code is basically as below
class TfrsModel(tfrs.Model, ABC):
def __init__(self, user_model, item_model, loss_task, cached_train_event_ds, cached_test_event_ds):
super().__init__()
self.item_model: tf.keras.Model = item_model
self.user_model: tf.keras.Model = user_model
self.task: tf.keras.layers.Layer = loss_task
self.cached_train_event_ds = cached_train_event_ds
self.cached_test_event_ds = cached_test_event_ds
def compute_loss(self, features: Dict[Text, tf.Tensor], training=False) -> tf.Tensor:
# We pick out the user features and pass them into the user model.
user_embeddings = self.user_model(features["user_id"])
# Pick out the item features and pass them into the item model, getting embeddings back.
item_embeddings = self.item_model(features["item_id"])
# The task computes the loss and the metrics.
return self.task(user_embeddings, item_embeddings)
class TfrsModelMaker(object):
def __init__(self, items_path, users_path, events_path, num_items, num_users, num_events):
self.items_path = items_path
self.users_path = users_path
self.events_path = events_path
self.num_items = num_items
self.num_users = num_users
self.num_events = num_events
print()
print(">> Initializing TfrsModelMaker...")
print(">> items_path : {}".format(items_path))
print(">> users_path : {}".format(users_path))
print(">> events_path : {}".format(events_path))
print(">> num_items : {}".format(num_items))
print(">> num_users : {}".format(num_users))
print(">> num_events : {}".format(num_events))
print()
# Turn off the many Unverified HTTPS request warnings during file downloads.
urllib3.disable_warnings()
self.items_ds, self.events_ds = self._load_tf_datasets()
self.test_events_ds, self.train_events_ds = self._prepare_data()
def create_model(self):
embedding_dimension = 32
print(">> Strategy: {}".format(strategy))
with strategy.scope():
# tf.config.set_soft_device_placement(True)
# print(">> Number of devices: {}".format(strategy.num_replicas_in_sync))
user_ids_filepath = self.get_s3_filepaths(self.users_path, ".csv")[0]
item_ids_filepath = self.get_s3_filepaths(self.items_path, ".csv")[0]
# The query tower
u_lookup = tf.keras.layers.experimental.preprocessing.IntegerLookup(vocabulary=user_ids_filepath)
user_model = tf.keras.Sequential(
[
u_lookup,
# We add an additional embedding to account for unknown tokens.
tf.keras.layers.Embedding(u_lookup.vocab_size() + 1, embedding_dimension),
]
)
# The candidate tower
c_lookup = tf.keras.layers.experimental.preprocessing.IntegerLookup(vocabulary=item_ids_filepath)
item_model = tf.keras.Sequential(
[
c_lookup,
# We add an additional embedding to account for unknown tokens.
tf.keras.layers.Embedding(c_lookup.vocab_size() + 1, embedding_dimension),
]
)
# Metrics
cands = self.items_ds.map(item_model)
metrics = tfrs.metrics.FactorizedTopK(candidates=cands)
# Loss
task = tfrs.tasks.Retrieval(metrics=metrics)
cached_train_event_ds = self.train_events_ds.batch(8192).cache()
cached_test_event_ds = self.test_events_ds.batch(4096).cache()
# per_replica_batch_size = 64
# global_batch_size = per_replica_batch_size * strategy.num_replicas_in_sync
#
# print()
# print(">> BATCH SIZE: {}".format(global_batch_size))
# print()
#
# cached_train_event_ds = self.train_events_ds.batch(global_batch_size).cache()
# cached_test_event_ds = self.test_events_ds.batch(global_batch_size).cache()
model = TfrsModel(user_model, item_model, task, cached_train_event_ds, cached_test_event_ds)
model.compile(optimizer=tf.keras.optimizers.Adagrad(learning_rate=0.1))
# model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=0.1))
return model
@staticmethod
def train_and_evaluate(model, num_epochs):
print_with_date(">> Training the model...")
# Train the model
# model.fit(model.cached_train_event_ds, epochs=num_epochs)
# turn off keras progress (verbose=0) and use tqdm instead. For the callback:
# verbose=2 means separate progressbars for epochs and batches
# 1 means clear batch bars when done
# 0 means only show epochs (never show batch bars)
model.fit(model.cached_train_event_ds, epochs=num_epochs, verbose=0, callbacks=[TqdmCallback(verbose=2)])
print_with_date(">> Training of the model: done.")
# Evaluate the model
print_with_date(">> Evaluating the model...")
eval_results = model.evaluate(model.cached_test_event_ds, return_dict=True)
print_with_date(">> Evaluation of the model: done.")
print()
print(f">> Eval results (epochs={num_epochs}):")
print(str(eval_results))
print()
maciejkula
Setup:
The TQDM progress bar never moves. Execution appears hanging, also, with progress bar code removed.
Things attempted:
compilewith Adagrad vs. Adam. Tried a small batch size such as 192 vs. larger such as 8192. No difference for my prototype.Can I get any output out of TFRS at this point to tell what might be going wrong? Any other things to try?
Output on the command line: