Closed arjun-kava closed 4 years ago
I think rather than loading whole dataset into memory, we need to think about batch to batch loading from memory.
Well there are several issues with reading triplet file. For example I have file with more 4M lines and I get the same issue. Reviewed code and found that it read whole file in memory (!!!). Modified it with reading file line by `line:
diff --git a/ImageDataGeneratorCustom.py b/ImageDataGeneratorCustom.py
index 0888f77..f769069 100644
--- a/ImageDataGeneratorCustom.py
+++ b/ImageDataGeneratorCustom.py
@@ -670,12 +670,14 @@ class Iterator(object):
def __init__(self, n, batch_size, shuffle, seed,triplet_path):
count = 0
- f = open(triplet_path)
- f_read = f.read()
- for line in f_read.split('\n'):
- if len(line)>1:
- count += 1
- f.close()
+ #f = open(triplet_path)
+ #f_read = f.read()
+ #for line in f_read.split('\n'):
+ with open(triplet_path) as fd:
+ for line in fd:
+ if line.strip():
+ count += 1
+ #f.close()
self.n = count * 3#n
self.batch_size = batch_size
@@ -852,18 +854,25 @@ def _list_valid_filenames_in_directory(directory, white_list_formats,
the filenames will be ["class1/file1.jpg", "class1/file2.jpg", ...]).
"""
def _recursive_list(subpath):
- f = open(triplet_path)
- f_read = f.read()
- triplets = f_read.split('\n')
- f.close()
+ #f = open(triplet_path)
+ #f_read = f.read()
+ #triplets = f_read.split('\n')
+ #f.close()
filenames = []
- for triplet in triplets:
- triplet = triplet.split(',')
- if len(triplet)!=3:
- continue
- filenames +=[triplet[0]]
- filenames +=[triplet[1]]
- filenames +=[triplet[2]]
+ #for triplet in triplets:
+ with open(triplet_path) as fd:
+ for line in fd:
+ triplet = line.strip()
+ triplet = triplet.split(',')
+ if len(triplet)!=3:
+ continue
+ filenames +=[triplet[0]]
+ filenames +=[triplet[1]]
+ filenames +=[triplet[2]]
to_return_tuple= tuple()
to_return_tuple = (os.path.abspath(subpath),[],filenames,)
return [to_return_tuple]
Now it's much better with memory. But I still can not understand logic of '_list_valid_filenames_in_directory' method. It creates in memory all triplets (list of each file in triplet file) list for each class (subfolder) !!!! WHY???? I can't understand it.
Killed. abnormally before start of training
2018-08-11 14:15:42.132175: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1405] Found device 0 with properties: name: GeForce GT 710 major: 3 minor: 5 memoryClockRate(GHz): 0.954 pciBusID: 0000:01:00.0 totalMemory: 1.95GiB freeMemory: 1.50GiB 2018-08-11 14:15:42.132206: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1484] Adding visible gpu devices: 0 2018-08-11 14:15:42.833257: I tensorflow/core/common_runtime/gpu/gpu_device.cc:965] Device interconnect StreamExecutor with strength 1 edge matrix: 2018-08-11 14:15:42.833307: I tensorflow/core/common_runtime/gpu/gpu_device.cc:971] 0 2018-08-11 14:15:42.833317: I tensorflow/core/common_runtime/gpu/gpu_device.cc:984] 0: N 2018-08-11 14:15:42.834399: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1097] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 1270 MB memory) -> physical GPU (device: 0, name: GeForce GT 710, pci bus id: 0000:01:00.0, compute capability: 3.5) input_1 (None, None, None, 3) block1_conv1 (None, None, None, 64) block1_conv2 (None, None, None, 64) block1_pool (None, None, None, 64) block2_conv1 (None, None, None, 128) block2_conv2 (None, None, None, 128) block2_pool (None, None, None, 128) block3_conv1 (None, None, None, 256) block3_conv2 (None, None, None, 256) block3_conv3 (None, None, None, 256) block3_pool (None, None, None, 256) block4_conv1 (None, None, None, 512) block4_conv2 (None, None, None, 512) block4_conv3 (None, None, None, 512) block4_pool (None, None, None, 512) block5_conv1 (None, None, None, 512) block5_conv2 (None, None, None, 512) block5_conv3 (None, None, None, 512) block5_pool (None, None, None, 512) input_2 (None, 224, 224, 3) input_3 (None, 224, 224, 3) global_average_pooling2d_1 (None, 512) conv2d_1 (None, 14, 14, 96) conv2d_2 (None, 7, 7, 96) dense_1 (None, 4096) max_pooling2d_1 (None, 4, 4, 96) max_pooling2d_2 (None, 4, 4, 96) dropout_1 (None, 4096) flatten_1 (None, 1536) flatten_2 (None, 1536) dense_2 (None, 4096) lambda_2 (None, 1536) lambda_3 (None, 1536) dropout_2 (None, 4096) concatenate_1 (None, 3072) lambda_1 (None, 4096) concatenate_2 (None, 7168) dense_3 (None, 4096) lambda_4 (None, 4096) Found 10066 images belonging to 1134 classes. Killed