this func has no writing action to pkl files, it seems to have no contribution for the next training?
for dataset in dataset_splits:
if dataset.mode != 'extract_data':
raise Exception('Dataset mode should be extract_data')
data_loaders = []
for dataset in dataset_splits:
dl = torch_data.DataLoader(
dataset, batch_size=batch_size,
shuffle=False, num_workers=num_workers)
data_loaders.append(dl)
print("Extracting pre-processed data...")
num_mini_batches = sum(
[len(data_loader) for data_loader in data_loaders])
mini_batch_count = 0
for data_loader in data_loaders:
for i, _ in enumerate(data_loader):
if verbose:
print(
"mini batch " + str(mini_batch_count + 1) + '/' + str(num_mini_batches))
mini_batch_count += 1
this func has no writing action to pkl files, it seems to have no contribution for the next training?