Open cheisterkamp opened 7 years ago
Note that sometimes the error occurs when the program tries to compute_chips (getting the RGB pngs of the chips). However, if the user individually clicks on each chip in the chip table, the chips go through compute_chips individually rather than in parallel. In this case, the error occurs during the process of computing features.
====================
[hs] query database
====================
[mc3] FEAT_UID is different. Need to reload features
[mc3] Old: None
[mc3] New: _FEAT(hesaff+sift,0_9001)_CHIP(sz750)
[hs] unload_cxdata(cx='all')
[hs] Unloading all data
[hs] finished unloading all data
=============================
[cc2] Precomputing chips and loading chip paths: 'test3'
=============================
[cc2] chip_uid = '_CHIP(sz750)'
[parallel] Already computed 1 compute_chip tasks
[parallel] ... No compute_chip tasks left to compute!
[cc2] Done Precomputing chips and loading chip paths
[cc2]=============================
=============================
[fc2] Precomputing and loading features: 'test3'
=============================
[fc2] feat_uid = '_FEAT(hesaff+sift,0_9001)_CHIP(sz750)'
[io] fname='kpts_list_FEAT(hesaff+sift,0_9001)_CHIP(sz750)_cids((1,)d[z^3[m8).npy' does not exist
[io] fname='desc_list_FEAT(hesaff+sift,0_9001)_CHIP(sz750)_cids((1,)d[z^3[m8).npy' does not exist
[fc2] Loading _FEAT(hesaff+sift,0_9001)_CHIP(sz750) individually
[parallel] Already computed 0 precompute_hesaff tasks
tic('Executing 1 precompute_hesaff: tasks in serial')
precompute_hesaff:
precompute_hesaff: 1/1
...toc('Executing 1 precompute_hesaff: tasks in serial')=0.1942s
*** Error in `python': munmap_chunk(): invalid pointer: 0x000000000449fce0 ***
The function prequery_checks in match_chips3.py calls hs.refresh_features
def prequery_checks(hs, qdat):
query_uid = qdat.cfg.get_uid('noCHIP')
feat_uid = qdat.cfg._feat_cfg.get_uid()
query_hist_id = (feat_uid, query_uid)
if hs.query_history[-1][0] != feat_uid:
print('[mc3] FEAT_UID is different. Need to reload features')
print('[mc3] Old: ' + str(hs.query_history[-1][0]))
print('[mc3] New: ' + str(feat_uid))
hs.unload_cxdata('all')
hs.refresh_features()
elif hs.query_history[-1][1] != query_uid:
print('[mc3] QUERY_UID is different. Need to refresh features')
print('[mc3] Old: ' + str(hs.query_history[-1][1]))
print('[mc3] New: ' + str(query_uid))
hs.refresh_features()
hs.query_history.append(query_hist_id)
print('[mc3] query_dcxs(): query_uid = %r ' % query_uid)
refresh_features in HotSpotterAPI.py calls hs.load_chips and hs.load_features
def load_chips(hs, cx_list=None):
cc2.load_chips(hs, cx_list)
def load_features(hs, cx_list=None):
fc2.load_features(hs, cx_list=cx_list)
@profile
def refresh_features(hs, cx_list=None):
hs.load_chips(cx_list=cx_list)
hs.load_features(cx_list=cx_list)
load_chips in chip_compute2.py calls parallel_compute with function compute_chip
#--------------------------
# EXTRACT AND RESIZE CHIPS
#--------------------------
pcc_kwargs = {
'arg_list': [gfpath_list, cfpath_list, roi_list, theta_list, chipsz_lis$
'lazy': not hs.args.nocache_chips,
'num_procs': hs.args.num_procs,
'common_args': [filter_list]
}
# Compute all chips with paramatarized filters
parallel_compute(compute_chip, **pcc_kwargs)
this is the compute_chip function in chip_compute2:
# TODO: Change the force_gray to work a little nicer
def compute_chip(img_fpath, chip_fpath, roi, theta, new_size, filter_list, forc$
'''Extracts Chip; Applies Filters; Saves as png'''
#printDBG('[cc2] extracting chip')
chipBGR = extract_chip(img_fpath, roi, theta, new_size)
#printDBG('[cc2] extracted chip')
for func in filter_list:
#printDBG('[cc2] computing filter: %r' % func)
chipBGR = func(chipBGR)
cv2.imwrite(chip_fpath, chipBGR)
return True
load_features in feature_compute2.py calls _load_features_individually
@profile
def load_features(hs, cx_list=None, **kwargs):
print('\n=============================')
print('[fc2] Precomputing and loading features: %r' % hs.get_db_name())
print('=============================')
#----------------
# COMPUTE SETUP
#----------------
use_cache = not hs.args.nocache_feats
use_big_cache = use_cache and cx_list is None
feat_cfg = hs.prefs.feat_cfg
feat_uid = feat_cfg.get_uid()
if hs.feats.feat_uid != '' and hs.feats.feat_uid != feat_uid:
print('[fc2] Disagreement: OLD_feat_uid = %r' % hs.feats.feat_uid)
print('[fc2] Disagreement: NEW_feat_uid = %r' % feat_uid)
print('[fc2] Unloading all chip information')
hs.unload_all()
hs.load_chips(cx_list=cx_list)
print('[fc2] feat_uid = %r' % feat_uid)
# Get the list of chip features to load
cx_list = hs.get_valid_cxs() if cx_list is None else cx_list
if not np.iterable(cx_list):
cx_list = [cx_list]
if len(cx_list) == 0:
return # HACK
cx_list = np.array(cx_list) # HACK
if use_big_cache: # use only if all descriptors requested
kpts_list, desc_list = _load_features_bigcache(hs, cx_list)
else:
kpts_list, desc_list = _load_features_individualy(hs, cx_list)
# Extend the datastructure if needed
list_size = max(cx_list) + 1
helpers.ensure_list_size(hs.feats.cx2_kpts, list_size)
helpers.ensure_list_size(hs.feats.cx2_desc, list_size)
# Copy the values into the ChipPaths object
for lx, cx in enumerate(cx_list):
hs.feats.cx2_kpts[cx] = kpts_list[lx]
for lx, cx in enumerate(cx_list):
hs.feats.cx2_desc[cx] = desc_list[lx]
hs.feats.feat_uid = feat_uid
print('[fc2]=============================')
load_features_individually calls parallel_compute with function precompute_fn, which is mapped to precompute_hesaff in extern_feat.py
# Maps a preference string into a function
feat_type2_precompute = {
'hesaff+sift': extern_feat.precompute_hesaff,
}
@profile
def _load_features_individualy(hs, cx_list):
use_cache = not hs.args.nocache_feats
feat_cfg = hs.prefs.feat_cfg
feat_dir = hs.dirs.feat_dir
feat_uid = feat_cfg.get_uid()
print('[fc2] Loading ' + feat_uid + ' individually')
# Build feature paths
rchip_fpath_list = [hs.cpaths.cx2_rchip_path[cx] for cx in iter(cx_list)]
cid_list = hs.tables.cx2_cid[cx_list]
feat_fname_fmt = ''.join(('cid%d', feat_uid, '.npz'))
feat_fpath_list = [join(feat_dir, feat_fname_fmt % cid) for cid in cid_list]
# Compute features in parallel, saving them to disk
kwargs_list = [feat_cfg.get_dict_args()] * len(rchip_fpath_list)
precompute_args = [rchip_fpath_list, feat_fpath_list, kwargs_list]
pfc_kwargs = {'num_procs': hs.args.num_procs, 'lazy': use_cache}
precompute_fn = feat_type2_precompute[feat_cfg.feat_type]
parallel_compute(precompute_fn, precompute_args, **pfc_kwargs)
# Load precomputed features sequentially
kpts_list, desc_list = sequential_feat_load(feat_cfg, feat_fpath_list)
return kpts_list, desc_list
precompute_hesaff in extern_feat.py calls precompute, which calls compute_hesaff, which calls detect_kpts
#---------------------------------------
# Define precompute functions
def precompute(rchip_fpath, feat_fpath, dict_args, compute_fn):
kpts, desc = compute_fn(rchip_fpath, dict_args)
np.savez(feat_fpath, kpts, desc)
return kpts, desc
def precompute_hesaff(rchip_fpath, feat_fpath, dict_args):
return precompute(rchip_fpath, feat_fpath, dict_args, compute_hesaff)
#---------------------------------------
# Work functions which call the external feature detectors
# Helper function to call commands
try:
from hstpl.extern_feat import pyhesaff
def detect_kpts_new(rchip_fpath, dict_args):
kpts, desc = pyhesaff.detect_kpts(rchip_fpath, **dict_args)
return kpts, desc
print('[extern_feat] new hessaff is available')
except ImportError as ex:
print('[extern_feat] new hessaff is not available: %r' % ex)
if '--strict' in sys.argv:
raise
try:
from hstpl.extern_feat import pyhesaffexe
def detect_kpts_old(rchip_fpath, dict_args):
kpts, desc = pyhesaffexe.detect_kpts(rchip_fpath, **dict_args)
return kpts, desc
print('[extern_feat] old hessaff is available')
except ImportError as ex:
print('[extern_feat] old hessaff is not available: %r' % ex)
if '--strict' in sys.argv:
raise
if OLD_HESAFF:
detect_kpts = detect_kpts_old
print('[extern_feat] using: old hessian affine')
else:
detect_kpts = detect_kpts_new
print('[extern_feat] using: new pyhesaff')
#----
def compute_hesaff(rchip_fpath, dict_args):
return detect_kpts(rchip_fpath, dict_args)
The np.savez statement in precompute (in extern_feat) is what creates the .npz feature file. When the file is empty later, I think that something went wrong during parallelization so that the kpts and desc were either not computed or computed incorrectly.
Matt noticed that sometimes the two items in this statement (kpts and desc) get saved individually rather than in a zip file. In this case, the program fails with ValueError: Seed must be between 0 and 2**32 - 1
parallel_compute is in hscom/Parallelize.py. I'm fairly certain that this is where the issue lies, since the program sometimes fails when trying to compute multiple chip features at once, and sometimes fails when trying to compute multiple chips at once.
def parallel_compute(func, arg_list, num_procs=None, lazy=True, args=None, comm$
if args is not None and num_procs is None:
num_procs = args.num_procs
elif num_procs is None:
num_procs = max(1, int(multiprocessing.cpu_count() / 2))
task_list = make_task_list(func, arg_list, lazy=lazy, common_args=common_ar$
nTasks = len(task_list)
if nTasks == 0:
print('[parallel] ... No %s tasks left to compute!' % func.func_name)
return None
# Do not execute small tasks in parallel
if nTasks < num_procs / 2 or nTasks == 1:
num_procs = 1
num_procs = min(num_procs, nTasks)
task_lbl = func.func_name + ': '
try:
ret = parallelize_tasks(task_list, num_procs, task_lbl)
except Exception as ex:
sys.stdout.flush()
print('[parallel!] Problem while parallelizing task: %r' % ex)
print('[parallel!] task_list: ')
for task in task_list:
print(' %r' % (task,))
break
print('[parallel!] common_args = %r' % common_args)
print('[parallel!] num_procs = %r ' % (num_procs,))
print('[parallel!] task_lbl = %r ' % (task_lbl,))
sys.stdout.flush()
raise
return ret
The first time a query is run on a new database with 3 chips, some sort of error occurs in the multiprocessing of precomupte_hesaff (located in the extern_feat file)
============================= The second time you open HotSpotter and try to run a query on the same database, the following error occurs:
If you look at /home/noah/Desktop/attempt9/_hsdb/computed/feats/cid1_FEAT(hesaff+sift,0_9001)_CHIP(sz750).npz the file exists, but is empty. I think that what is happening is that the file is created before the multiprocessing/precompute_hesaff error occurs, but never written to. The second time through the program, it sees that the file exists, so it doesn't try to rerun precompute_hesaff, but is still unable to continue.