Error during copying video file

[acomrie]

I am getting an error (see below) during nwb generation that seems to be permissions-related at the stage of copying over video files. After trying to investigate with the debugger, the src and dest files and containing directories all have rwx permissions for everyone. Any thoughts greatly appreciated! Thanks in advance.

In case useful, here are a few of the variables from the end of the traceback - st: os.stat_result(st_mode=33279, st_ino=25182299, st_dev=59, st_nlink=1, st_uid=39495, st_gid=39495, st_size=107605743, st_atime=1679965970, st_mtime=1645842377, st_ctime=1666115234)

dst: '/cumulus/amankili/Banner/nwbout/video/20220225_Banner_01_sleepBan77mWnostim.1.h264'

src: '/cumulus/amankili/Banner/raw/20220225/20220225_Banner_01_sleepBan77mWnostim.1.h264'

self.video_files_to_copy: ['20220225_Banner_01_sleepBan77mWnostim.1.h264', '20220225_Banner_03_sleepBan77mWnostim.1.h264', '20220225_Banner_05_sleepBan77mWnostim.1.h264', '20220225_Banner_07_sleepBan77mWnostim.1.h264', '20220225_Banner_09_sleepBan77mWnostim.1.h264', '20220225_Banner_11_sleepBan77mWnostim.1.h264', '20220225_Banner_13_sleepBan77mWnostim.1.h264', '20220225_Banner_15_sleepBan77mWnostim.1.h264', '20220225_Banner_17_sleepBan77mWnostim.1.h264', '20220225_Banner_02_wtrackBan77mWlockout80mstheta90.1.h264', '20220225_Banner_04_wtrackBan77mWlockout80mstheta90.1.h264', '20220225_Banner_06_wtrackBan77mWlockout80mstheta90.1.h264', '20220225_Banner_08_wtrackBan77mWlockout80mstheta90.1.h264', '20220225_Banner_10_wtrackBan77mWlockout80mstheta90.1.h264', '20220225_Banner_12_wtrackBan77mWlockout80mstheta90.1.h264', '20220225_Banner_14_wtrackBan77mWlockout80mstheta90.1.h264', '20220225_Banner_16_wtrackBan77mWlockout80mstheta90.1.h264']

---------------------------------------------------------------------------
PermissionError                           Traceback (most recent call last)
/tmp/ipykernel_3511942/1858210567.py in <module>
     37                               video_path=video_path,
     38                               trodes_rec_export_args = trodes_rec_export_args)
---> 39         content = builder.build_nwb()
     40         print(content) #just to look
     41         # Automatically delete preprocessing files? Not for this guy given old pipeline use still

~/Src/rec_to_nwb/rec_to_nwb/processing/builder/raw_to_nwb_builder.py in build_nwb(self, run_preprocessing, process_mda_valid_time, process_mda_invalid_time, process_pos_valid_time, process_pos_invalid_time)
    230             process_mda_invalid_time=process_mda_invalid_time,
    231             process_pos_valid_time=process_pos_valid_time,
--> 232             process_pos_invalid_time=process_pos_invalid_time)
    233 
    234         logger.info('Done...\n')

~/Src/rec_to_nwb/rec_to_nwb/processing/builder/raw_to_nwb_builder.py in __build_nwb_file(self, process_mda_valid_time, process_mda_invalid_time, process_pos_valid_time, process_pos_invalid_time)
    244         for date in self.dates:
    245             logger.info('Date: {}'.format(date))
--> 246             nwb_builder = self.get_nwb_builder(date)
    247             content = nwb_builder.build()
    248             nwb_builder.write(content)

~/Src/rec_to_nwb/rec_to_nwb/processing/builder/raw_to_nwb_builder.py in get_nwb_builder(self, date)
    303             reconfig_header=self.__get_header_path(),
    304             # reconfig_header=self.__is_rec_config_valid()
--> 305             **old_dataset_kwargs
    306         )
    307 

~/Src/rec_to_nwb/rec_to_nwb/processing/tools/beartype/beartype.py in func_beartyped(__beartype_func, *args, **kwargs)

~/Src/rec_to_nwb/rec_to_nwb/processing/builder/nwb_file_builder.py in __init__(self, data_path, animal_name, date, nwb_metadata, process_dio, process_mda, process_analog, process_pos_timestamps, preprocessing_path, video_path, output_file, reconfig_header, is_old_dataset, session_start_time)
    294                              'raw', self.date),
    295                 self.video_path,
--> 296                 self.metadata["associated_video_files"],
    297             )
    298 

~/Src/rec_to_nwb/rec_to_nwb/processing/builder/originators/video_files_originator.py in __init__(self, raw_data_path, video_path, video_files_metadata, convert_timestamps, return_timestamps)
     16             raw_data_path, video_path, video_files_metadata,
     17             convert_timestamps=convert_timestamps,
---> 18             return_timestamps=return_timestamps)
     19 
     20     def make(self, nwb_content):

~/Src/rec_to_nwb/rec_to_nwb/processing/tools/beartype/beartype.py in func_beartyped(__beartype_func, *args, **kwargs)

~/Src/rec_to_nwb/rec_to_nwb/processing/nwb/components/video_files/fl_video_files_manager.py in __init__(self, raw_data_path, video_path, video_files_metadata, convert_timestamps, return_timestamps)
     21         self.video_files_copy_maker = VideoFilesCopyMaker(
     22             [video_files['name'] for video_files in video_files_metadata])
---> 23         self.video_files_copy_maker.copy(raw_data_path, video_path)
     24         self.fl_video_files_extractor = FlVideoFilesExtractor(
     25             raw_data_path, video_files_metadata,

~/Src/rec_to_nwb/rec_to_nwb/processing/nwb/components/video_files/video_files_copy_maker.py in copy(self, src, dst)
     15             raise InvalidPathException(dst + ' is not valid path')
     16         for video_file in self.video_files_to_copy:
---> 17             copy_file(os.path.join(src,  video_file), dst)

~/anaconda3/envs/rec_to_nwb/lib/python3.7/shutil.py in copy(src, dst, follow_symlinks)
    247         dst = os.path.join(dst, os.path.basename(src))
    248     copyfile(src, dst, follow_symlinks=follow_symlinks)
--> 249     copymode(src, dst, follow_symlinks=follow_symlinks)
    250     return dst
    251 

~/anaconda3/envs/rec_to_nwb/lib/python3.7/shutil.py in copymode(src, dst, follow_symlinks)
    142 
    143     st = stat_func(src)
--> 144     chmod_func(dst, stat.S_IMODE(st.st_mode))
    145 
    146 if hasattr(os, 'listxattr'):

PermissionError: [Errno 1] Operation not permitted: '/cumulus/amankili/Banner/nwbout/video/20220225_Banner_01_sleepBan77mWnostim.1.h264'

[edeno]

Can you try a manual copy and confirm it works?

[lfrank]

Could you try this again now? I added write permissions throughout the folder.

[acomrie]

Thanks both. This is resolved now. A note for anyone that runs into this in the future - the solution I ended up with was deleting all the previously copied video files for this date in the destination dir and re-running nwb generation. This also worked a second time when the same error came up on another date of data.