search

KudryashevLab / TomoBEAR

TomoBEAR is a configurable and customizable modular pipeline for streamlined processing of cryo-electron tomographic data for subtomogram averaging.
https://github.com/KudryashevLab/TomoBEAR/wiki
Other
26 stars 5 forks source link

GPU out of memory issue #22

Closed rickhooy closed 1 year ago

rickhooy commented 1 year ago

Describe the bug

Pipeline crashes at CreateStacks step due to GPU memory issue

Project JSON file

"general": {
        "project_name": "sampleProject",
        "project_description": "Practice",
        "data_path": "/data/tb/Frames/*.tif",
        "processing_path": "/data/tb/processing",
        "expected_symmetrie": "C1",
        "template_matching_binning": 8,
        "gold_bead_size_in_nm": 10,
        "reconstruction_thickness": 3000,
        "rotation_tilt_axis": 79,
        "aligned_stack_binning": 2,
        "pre_aligned_stack_binning": 2,
        "binnings": [2, 4, 8]
    },
    "MetaData": {
    },
    "SortFiles": {
    },
    "MotionCor2": {
    },
    "CreateStacks": {
    },
    "DynamoTiltSeriesAlignment": {
    },
    "DynamoCleanStacks": {
    },
    "BatchRunTomo": {
        "skip_steps": [4],
        "ending_step": 6
    },
    "StopPipeline": {

Error message

Error using parallel.internal.queue.Future/fetchOutputs One or more futures resulted in an error.

Error in LocalPipeline/execution_in_order (line 636) [dynamic_configuration_tmp, status_tmp] = fetchOutputs(f, 'UniformOutput', false);

Error in LocalPipeline/execute (line 400) [dynamic_configuration_out, tomogramstatus{i - 1}] = obj.("execution" + execution_method)(merged_configuration, obj.pipeline_definition{i}, previous_tomogram_status);

Error in runPipeline (line 150) pipeline.execute(starting_tomogram, ending_tomogram, step, gpu);

Error in runTomoBear (line 34) runPipeline(compute_environment, configuration_path, default_configuration_path, starting_tomogram, ending_tomogram, step, gpu);

Caused by: Error using gpuArray/imgaussfilt>frequencyGaussianFilter Out of memory on device. To view more detail about available memory on the GPU, use 'gpuDevice()'. If the problem persists, reset the GPU by calling 'gpuDevice(1)'.

To Reproduce Steps to reproduce the behavior:

  1. runTomoBear("local","/data/tb/template.json")

Expected behavior How should I configure the settings/compute environment to avoid GPU memory issues?

Screenshots If applicable, add screenshots to help explain your problem.

Desktop (please complete the following information):

Additional context Workstation has 4 x 3070, 8Gb GPU RAM each Running TomoBear out of Matlab

defaults.json

{
    "general": {
        "gpu": -1,
        "jobs_per_node": 1,
        "gpus_per_node": 2,
        "nodes": [],
        "slurm_execute": true,
        "slurm_node_list": "",
        "slurm_nodes": 0,
        "slurm_nice": 0,
        "slurm_partition": "",
        "slurm_gres": "",
        "slurm_qos": "",
        "slurm_constraint": "",
        "slurm_time": "",
        "slurm_exclusive": true,
        "slurm_flags": "",
        "slurm_gpus": 0,
        "slurm_mem_per_gpu_in_gb": 0,
        "slurm_gpus_per_task": 0,
        "slurm": false,
        "gpu_worker_multiplier": 1,
        "pipeline_location": "/usr/local/TomoBEAR",
        "pipeline_executable": "run_TomoBEAR.sh",
        "sbatch_wrapper": "sbatch.sh",
        "cuda_forward_compatibility": true,
        "random_number_generator_seed": 0,
        "tomogram_output_prefix": "tomogram",
        "tomogram_input_prefix": "",
        "data_path": "",
        "processing_path": "",
        "debug": false,
        "project_name": "TomoBEAR",
        "project_description": "TomoBEAR project description",
        "wipe_cache": false,
        "motion_correction_command": "motioncor2", 
        "ctf_correction_command": "Gctf",
        "ctffind_heredoc_path": "configurations/ctffind_heredoc.sh",
        "double_numbering": true,
        "prefix_position": 0,
        "tomogram_number_position": -1,
        "tilt_number_position": -1,
        "angle_position": -1,
        "date_position": -1,
        "time_position": -1,
        "output_folder": "output",
        "scratch_folder": "scratch",
        "remove_folders": false,
        "tilt_scheme": "dose_symmetric",
        "tilting_step": 3,
        "gold_bead_size_in_nm": 10,
        "rotation_tilt_axis": 85,
        "pid_wait_time": 5,
        "raw_files_folder": "raw_files",
        "fid_files_folder": "fiducial_models",
        "motion_corrected_files_folder": "motion_corrected_mrcs",
        "aligned_tilt_stacks_folder": "aligned_tilt_stacks",
        "ctf_corrected_aligned_tilt_stacks_folder": "ctf_corrected_aligned_tilt_stacks",
        "tilt_stacks_folder": "tilt_stacks",
        "binned_tilt_stacks_folder": "binned_tilt_stacks",
        "binned_aligned_tilt_stacks_folder": "binned_aligned_tilt_stacks",
        "ctf_corrected_binned_aligned_tilt_stacks_folder": "ctf_corrected_binned_aligned_tilt_stacks",
        "tomograms_folder": "tomograms",
        "ctf_corrected_tomograms_folder": "ctf_corrected_tomograms",
        "exact_filtered_tomograms_folder": "exact_filtered_tomograms",
        "binned_tomograms_folder": "binned_tomograms",
        "ctf_corrected_binned_tomograms_folder": "ctf_corrected_binned_tomograms",
        "exact_filtered_ctf_corrected_binned_tomograms_folder": "exact_filtered_ctf_corrected_binned_tomograms",
        "binned_exact_filtered_tomograms_folder": "binned_exact_filtered_tomograms",
        "particles_folder": "particles",
        "particles_table_folder": "particles_table",
        "particles_susan_info_folder": "particles_susan_info",
        "meta_data_folder": "meta_data",
        "dynamo_folder": "dynamo",
        "templates_folder": "templates",
        "gain_correction_folder": "gain_correction",
        "binarized_stacks_folder": "binarized_stacks",
        "ctf_corrected_stack_suffix": "ctfc",
        "ignore_success_files": false,
        "keV": 300,
        "spherical_aberation": 2.7,
        "skip_n_first_projections": 3,
        "nominal_defocus_in_nm": 0,
        "link_files_threshold_in_mb": 1,
        "tilt_type": "single",
        "minimum_files": 15,
        "increase_folder_numbers": false,
        "cpu_fraction": 0.5,
        "tomogram_indices": [],
        "tomogram_begin": 0,
        "tomogram_end": 0,
        "tomogram_step": 1,
        "aligned_stack_binning": 4,
        "pre_aligned_stack_binning": 4,
        "template_matching_binning": 16,
        "reconstruction_thickness": 4000,
        "binnings": [16],
        "ignore_file_system_time_stamps": true,
        "automatic_filename_parts_recognition": true,
        "angle_regex": "_([+-]*\\d+[.]+\\d+)[_.]*",
        "name_regex": "([A-Za-z\\d_]+[A-Za-z]+)",
        "number_regex": "[_]*([\\d]+[_[\\d]*]*)_",
        "name_number_regex": "([A-Za-z\\d_]+[A-Za-z]+)[_]*([\\d]+[_[\\d]*]*)",
        "name_number_regex_backup": "([A-Za-z\\d_]+[A-Za-z]+)[_]*([\\d]+[_[\\d]*]*)_",
        "month_date_time_regex": "_([A-Z][a-z]{2}[\\d]{2})_([\\d]{2}.[\\d]{2}.[\\d]{2})[._]",
        "tomogram_acquisition_time_in_minutes": 70,
        "keep_intermediates": true,
        "reconstruct": "binned",
        "em_clarity_path": "/usr/local/emClarity",
        "dynamo_path": "/usr/local/dynamo",
        "astra_path": "",
        "modules": [],
        "duplicated_tilts": "last",
        "ft_bin": 1,
        "expected_symmetrie": "C1",
        "checkpoint_module": false,
        "propagate_failed_stacks": true,
        "first_tilt_angle": "",
        "execute": true,
        "box_size": 1.0,
        "citation": "",
        "susan_particle_batch": 1000,
        "direct_copy": 0,
        "as_boxes": 1,
        "SUSAN_path": "",
        "mask_bandpass": [0, 2, 2],
        "ratio_mask_pixels_based_on_unbinned_pixels": 0.05,
        "use_dose_weighted_particles": false,
        "dose": 0,
        "b_factor_per_projection": 4, 
        "dose_order": [],
        "dt_crop_max_mb": 250000,
        "dt_crop_in_memory": 1,
        "dynamo_allow_padding": true,
        "regenerate_load_modules_file": true,
        "ge_flags": "",
        "qsub_wrapper": "qsub.sh",
        "email": "",
        "ge_queue": "",
        "ge_processing_environment": "smp",
        "ge_time": "",
        "ge_memory_per_processing_element_in_gb": "",
        "ge_gpus": 4,
        "ge_processing_elements": 4,
        "python_envs": "",
        "use_conda": true,
        "mcr_location": "",
        "defocus_limit_factor": 2.5,
        "skip_data_check": false,
        "ld_library_path":"",
        "additional_shell_initialization_script": "",
        "mask_path": "",
        "exclude_projections": 0,
        "dose_weight_first_image": true,
        "even_tilt_stacks_folder": "even_tilt_stacks",
        "odd_tilt_stacks_folder": "odd_tilt_stacks",
        "aligned_even_tilt_stacks_folder": "aligned_even_tilt_stacks",
        "aligned_odd_tilt_stacks_folder": "aligned_odd_tilt_stacks",
        "aligned_dose_weighted_tilt_stacks_folder": "aligned_dose_weighted_tilt_stacks",
        "aligned_dose_weighted_sum_tilt_stacks_folder": "aligned_dose_weighted_sum_tilt_stacks",
        "even_motion_corrected_files_folder": "even_motion_corrected_files",
        "odd_motion_corrected_files_folder": "odd_motion_corrected_files",
        "dose_weighted_motion_corrected_files_folder": "dose_weighted_motion_corrected_files",
        "dose_weighted_sum_motion_corrected_files_folder": "dose_weighted_sum_motion_corrected_files",
        "apply_patch_based_correction_to_all_projections": true,
        "binned_even_tilt_stacks_folder": "binned_even_tilt_stacks",
        "binned_odd_tilt_stacks_folder": "binned_odd_tilt_stacks",
        "binned_aligned_even_tilt_stacks_folder": "binned_aligned_even_tilt_stacks",
        "binned_aligned_odd_tilt_stacks_folder": "binned_aligned_odd_tilt_stacks",
        "binned_aligned_dose_weighted_tilt_stacks_folder": "binned_aligned_dose_weighted_tilt_stacks",
        "binned_aligned_dose_weighted_sum_tilt_stacks_folder": "binned_aligned_dose_weighted_sum_tilt_stacks",
        "even_tomograms_folder": "even_tomograms",
        "odd_tomograms_folder": "odd_tomograms",
        "dose_weighted_tomograms_folder": "dose_weighted_tomograms",
        "dose_weighted_sum_tomograms_folder": "dose_weighted_sum_tomograms",
        "binned_even_tomograms_folder": "binned_even_tomograms",
        "binned_odd_tomograms_folder": "binned_odd_tomograms",
        "ctf_corrected_binned_even_tomograms_folder": "ctf_corrected_binned_even_tomograms",
        "ctf_corrected_binned_odd_tomograms_folder": "ctf_corrected_binned_odd_tomograms",
        "binned_dose_weighted_tomograms_folder": "binned_dose_weighted_tomograms",
        "binned_dose_weighted_sum_tomograms_folder": "binned_dose_weighted_sum_tomograms",
        "ctf_corrected_binned_dose_weighted_tomograms_folder": "ctf_corrected_binned_dose_weighted_tomograms",
        "ctf_corrected_binned_dose_weighted_sum_tomograms_folder": "ctf_corrected_binned_dose_weighted_sum_tomograms",
        "ctf_corrected_binned_aligned_even_tilt_stacks_folder": "ctf_corrected_binned_aligned_even_tilt_stacks",
        "ctf_corrected_binned_aligned_odd_tilt_stacks_folder": "ctf_corrected_binned_aligned_odd_tilt_stacks",
        "ctf_corrected_binned_aligned_dose_weighted_tilt_stacks_folder": "ctf_corrected_binned_aligned_dose_weighted_tilt_stacks",
        "ctf_corrected_binned_aligned_dose_weighted_sum_tilt_stacks_folder": "ctf_corrected_binned_aligned_dose_weighted_sum_tilt_stacks",
        "conda_path": "",
        "matlab_shell": "matlab_shell.sh",
        "denoised_ctf_corrected_tomograms_folder": "denoised_ctf_corrected_tomograms",
        "denoised_ctf_corrected_binned_tomograms_folder": "denoised_ctf_corrected_binned_tomograms",
        "denoised_tomograms_folder": "denoised_tomograms",
        "dose_weighted_tilt_stacks_folder": "dose_weighted_tilt_stacks",
        "dose_weighted_sum_tilt_stacks_folder": "dose_weighted_sum_tilt_stacks",
        "denoised_binned_tomograms_folder": "denoised_binned_tomograms",
        "denoised_tilt_stacks_folder": "denoised_tilt_stacks",
        "aretomo_command": "aretomo",
        "listening_time_threshold_in_minutes": 15
    },
    "StopPipeline": {
        "execution_method": "control"
    },
    "TomoAlign": {
        "use_tomowarpalign": true,
        "warp_image_range": "",
        "show_transformed_fiducial_model": true,
        "use_aligned_stack": false,
        "leave_one_out_test": true,
        "sample_thickness": true,
        "use_splines": true,
        "use_3d_motion": true,
        "polynomial_order_in_x": 0,
        "polynomial_order_in_y": 0,
        "polynomial_order_in_z": 0,
        "polynomial_order_for_x_motion": 2,
        "polynomial_order_for_y_motion": 2,
        "polynomial_order_for_z_motion": 2,
        "remove_mixed_polynomial_terms": "z",
        "optimize_alignment": "",
        "reference_image_in_degreese": 0,
        "reconstruct": false
    },
    "Reconstruct": {
        "execution_method": "in_order",
        "reconstruct": "binned",
        "use_ctf_corrected_stack": true,
        "generate_exact_filtered_tomograms": false,
        "exact_filter_size": 1500,
        "generate_nad_filtered_tomograms": false,
        "nad_filter_output_iterations_list": [3],
        "nad_filter_number_of_iterations": -1,
        "nad_filter_sigma_for_smoothing": -1,
        "nad_filter_threshold_for_gradients": -1,
        "use_rawtlt": true,
        "correct_angles": "center",
        "skip": false,
        "citation": "",
        "citation_link": "",
        "doi": ""
    },
    "MetaData":{
        "execution_method": "once",
        "parallel_execution": true,
        "do_statistics": false,
        "skip": false,
        "citation": ""
    },
    "SortFiles": {
        "execution_method": "parallel",
        "starting_tomogram": 1,
        "use_link": true,
        "fixed_number": 0,
        "skip": false,
        "citation": ""
    },
    "MotionCor2": {
        "method": "MotionCor2",
        "execution_method": "in_order",
        "output_postfix": "motioncor2",
        "iterations": 50,
        "tolerance": 0.5,
        "gpu_memory": 0.5,
        "patch": "7 5 15",
        "group": 2,
        "outstack": 0,
        "gain": "",
        "defect": "",
        "dark": "",
        "b_factor": [500, 150],
        "in_fm_motion": 1,
        "apply_dose_weighting": false,
        "fm_dose": 0,
        "split_sum": false,
        "tilt": "",
        "magnification_anisotropy_major_scale": 1,
        "magnification_anisotropy_minor_scale": 1,
        "magnification_anisotropy_major_axis_angle": 360,
        "magnification_anisotropy_major_scale_tmp": 1,
        "magnification_anisotropy_minor_scale_tmp": 1,
        "magnification_anisotropy_major_axis_angle_tmp": 360,
        "split_sum": "false",
        "out_aln": "",
        "initial_dose": "",
        "align": 1,
        "throw": "0",
        "trunc": "0",
        "sum_range": [],
        "fm_ref": "",
        "skip": false,
        "citation": "",
        "citation_link": "",
        "doi": "",
        "eer_sampling": 2,
        "fm_int_file": "",
        "eer_total_number_of_fractions": 576,
        "eer_fraction_grouping": 16,
        "eer_exposure_per_fraction": 0.006945
    },
    "CreateStacks": {
        "execution_method": "in_order",
        "slice_suffix": "view",
        "normalization_method": "frames",
        "normalized_postfix": "norm",
        "stack_name": "tiltstack",
        "pixel_intensity_average": 128,
        "pixel_intensity_standard_deviation": 4,
        "border_pixels": 75,
        "skip": false,
        "citation": "",
        "citation_link": "",
        "doi": ""
    },
    "GCTFCtfphaseflipCTFCorrection": {
        "execution_method": "parallel",
        "slice_suffix": "view",
        "slice_folder": "slices",
        "gctf_correction_log_file": "gctf.log",
        "ctf_correction_log_file": "ctf_correction.log",
        "exact_filter_suffix": "ef",
        "tomogram_suffix": "full",
        "rotated_tomogram_suffix": "rotx",
        "generate_exact_filtered_tomograms": true,
        "exact_filter_size": 1500,
        "reconstruction_thickness": 2000,
        "use_aligned_stack": false,
        "do_phase_flip": false,
        "run_ctf_phase_flip": false,
        "reconstruct_tomograms": false,
        "defocus_tolerance": 20,
        "iWidth": 15,
        "maximum_strip_width": 1000,
        "ampContrast": 0.1,
        "defocus_file_version": 3,
        "defocus_file_version_3_flag": 1,
        "use_rawtlt": true,
        "skip": false,
        "citation": "",
        "citation_link": "",
        "doi": "",
        "do_EPA": true,
        "estimated_astigmatism": 1000,
        "ctf_estimation_method": "gctf",
        "fourier_spectrum_size": 1024,
        "resL": 50,
        "resH": 4,
        "defL": 5000,
        "defH": 90000,
        "ctf_preestim_defS": 500,
        "ctf_estim_defS": 100,
        "ctffind_output_suffix": "ctfestim"
    },
    "RefineFiducialModel": {
        "execution_method": "sequential",
        "skip": false,
        "citation": ""
    },
    "BatchRunTomo": {
        "execution_method": "parallel",
        "take_fiducials_from_dynamo": true,
        "generate_seed_model_with_all_fiducials_from_dynamo": true,
        "reconstruct_binned_stacks": false,
        "gold_erasing_extra_diameter": 30,
        "maximum_strip_width": 1000,
        "cpu_machine_list": "",
        "template_file": "configurations/batchruntomo-template.adoc",
        "directive_file_name": "DirectiveFile",
        "starting_step": 0,
        "ending_step": 21,
        "exit_on_error": true,
        "batchruntomo_description": "0: Setup
                 1: Preprocessing
                 2: Cross-correlation alignment
                 3: Prealigned stack
                 4: Patch tracking, autoseeding, or RAPTOR
                 5: Bead tracking
                 6: Alignment 
                 7: Positioning
                 8: Aligned stack generation
                 9: CTF plotting
                 10: 3D gold detection
                 11: CTF correction
                 12: Gold erasing after transforming fiducial model or
                     projecting 3D model
                 13: 2D filtering
                 14: Reconstruction
                 14.5: Postprocessing on a/b axis reconstruction
                 15: Combine setup
                 16: Solvematch
                 17: Initial matchvol;
                 18: Autopatchfit
                 19: Volcombine
                 20: Postprocessing with Trimvol
                 21: NAD (Nonlinear anistropic diffusion)",
        "directives_description": {
            "path": "/path/to/imod/com/directives.csv",
            "link": "https://bio3d.colorado.edu/imod/doc/directives.html"
        },
        "directives": {
            "setupset.copyarg.focus": 0,
            "setupset.copyarg.bfocus": 0,
            "setupset.copyarg.dual": 0,
            "setupset.copyarg.montage": 0,
            "runtime.Preprocessing.any.archiveOriginal": 0,
            "setupset.copyarg.pixel": "<REPLACE>",
            "setupset.copyarg.gold": "<REPLACE>",
            "setupset.copyarg.rotation": "<REPLACE>",
            "setupset.copyarg.userawtlt": 1,
            "setupset.copyarg.extract": 0,
            "setupset.copyarg.voltage": "<REPLACE>",
            "setupset.copyarg.Cs": "<REPLACE>",
            "setupset.copyarg.defocus": "<REPLACE>",
            "runtime.Preprocessing.any.removeXrays": 1,
            "comparam.eraser.ccderaser.LineObjects": 2,
            "comparam.eraser.ccderaser.BoundaryObjects": 3,
            "comparam.eraser.ccderaser.AllSectionObjects": "1-3",
            "comparam.prenewst.newstack.BinByFactor": "<REPLACE>",
            "runtime.RAPTOR.any.useAlignedStack": 1,
            "runtime.RAPTOR.any.numberOfMarkers": 50,
            "runtime.Fiducials.any.trackingMethod": 0,
            "runtime.Fiducials.any.seedingMethod": 1,
            "comparam.track.beadtrack.LightBeads": 0,
            "comparam.track.beadtrack.RoundsOfTracking": 5,
            "runtime.BeadTracking.any.numberOfRuns": 2,
            "comparam.track.beadtrack.SobelFilterCentering": 1,
            "comparam.track.beadtrack.KernelSigmaForSobel": 1.5,
            "comparam.autofidseed.autofidseed.TwoSurfaces": 0,
            "comparam.autofidseed.autofidseed.TargetNumberOfBeads": 20,
            "comparam.autofidseed.autofidseed.AdjustSizes": 1,
            "comparam.align.tiltalign.MagOption": 0,
            "comparam.align.tiltalign.TiltOption": 0,
            "comparam.align.tiltalign.RotOption": "-1",
            "comparam.align.tiltalign.XTiltOption": 0,
            "comparam.align.tiltalign.BeamTiltOption": 0,
            "comparam.newst.newstack.AntialiasFilter": -1,
            "runtime.AlignedStack.any.binByFactor": "<REPLACE>",
            "runtime.AlignedStack.any.correctCTF": 1,
            "runtime.AlignedStack.any.eraseGold": 1,
            "comparam.align.tiltalign.RobustFitting": 1,
            "comparam.tilt.tilt.THICKNESS": "<REPLACE>",
            "runtime.Reconstruction.any.useSirt": 0,
            "runtime.Reconstruction.any.doBackprojAlso": 1,
            "runtime.Postprocess.any.doTrimvol": 1,
            "runtime.Trimvol.any.reorient": 2,
            "runtime.Preprocessing.any.removeExcludedViews": 0,
            "setupset.copyarg.twodir": 0,
            "setupset.scanHeader": 0,
            "comparam.tilt.tilt.LOG": "",
            "comparam.tilt.tilt.SCALE": "0.0 1.0",
            "comparam.tilt.tilt.RADIAL": "0.5 0.0",
            "comparam.tilt.tilt.XAXISTILT": "0.0",
            "comparam.tilt.tilt.AdjustOrigin": 0,
            "comparam.align.tiltalign.AngleOffset": "0.0",
            "comparam.align.tiltalign.SeparateGroup": "<REPLACE>",
            "comparam.align.tiltalign.SurfacesToAnalyze": 1,
            "runtime.Positioning.any.centerOnGold": 1,
            "runtime.Positioning.any.sampleType": 0,
            "runtime.Positioning.any.wholeTomogram": 0,
            "runtime.Positioning.any.hasGoldBeads": 1,
            "comparam.xcorr.tiltxcorr.SearchMagChanges": 0,
            "comparam.align.tiltalign.LocalAlignments": 0,
            "runtime.GoldErasing.any.extraDiameter": "<REPLACE>",
            "runtime.GoldErasing.any.thickness": "<REPLACE>"
        },
        "skip": false,
        "citation": "",
        "citation_link": "",
        "doi": ""
    },
    "BinStacks":{
        "execution_method": "parallel",
        "use_ctf_corrected_aligned_stack": true,
        "use_aligned_stack": false,
        "antialias_filter": 6,
        "defocus_tolerance": 20,
        "iWidth": 2,
        "ampContrast": 0.1,
        "run_ctf_phaseflip": false,
        "ctf_corrected_stack_suffix": "ctfc",
        "skip": false,
        "citation": "",
        "citation_link": "",
        "doi": ""
    },
    "TemplateGenerationFromFile": {
        "execution_method": "once",
        "template_path": "",
        "mask_path": "",
        "use_ellipsoid": true,
        "radii_ratio": [0.33, 0.33, 0.5],
        "ellipsoid_smoothing_ratio": 0.16,
        "template_bandpass_cut_on_fourier_pixel": 2,
        "template_bandpass_cut_off_resolution_in_angstrom": 20,
        "template_bandpass_smoothing_pixels": 3,
        "use_bandpassed_template": true,
        "use_smoothed_mask": true,
        "invert_density": false,
        "skip": false,
        "citation": ""
    },
    "EMDTemplateGeneration": {
        "execution_method": "once",
        "template_emd_number": "",
        "mask_emd_filename": "",
        "template_bandpass_cut_on_fourier_pixel": 2,
        "template_bandpass_cut_off_resolution_in_angstrom": 20,
        "template_bandpass_smoothing_pixels": 3,
        "use_half_template_size": false,
        "mask_cut_off": 0.05,
        "template_cut_off": 0.75,
        "type": "dynamo",
        "use_bandpassed_template": true,
        "use_smoothed_mask": true,
        "dark_density": true,
        "skip": false,
        "flip_handedness": false,
        "citation": ""
    },
    "DynamoTemplateMatching": {
        "execution_method": "in_order",
        "use_ctf_corrected_tomograms": true,
        "show_table": false,
        "show_cross_correlations": false,
        "show_generated_template": false,
        "randomize_angles": false,
        "size_of_chunk": [512, 720, 500],
        "auto_detect_sampling": false,
        "auto_detect_sampling_multiplication_factor": 5,
        "cone_range": 360,
        "cone_sampling": 15,
        "matlab_workers": 1,
        "symmetry_opearator": 19,
        "template_transform": "none",
        "in_plane_range": 360,
        "in_plane_sampling": 7.5,
        "sampling": 15,
        "threshold_standard_deviation": 3,
        "ellipsoid_smoothing_pixels": 5,
        "skip": false,
        "citation": "",
        "use_denoised_tomograms": false
    },
    "TemplateMatchingPostProcessing": {
        "execution_method": "once",
        "parallel_execution": false,
        "checkpoint_module": true,
        "all_in_one_folder": true,
        "particle_count": 0,
        "particles_to_be_cropped": 0,
        "cc_std": 2.5,
        "crop_particles": true,
        "bandpass_cc_volume": false,
        "bandpass_cc_high_pass": 0,
        "bandpass_cc_low_pass": 0,
        "bandpass_cc_smoothing": 0,
        "keep_binned": true,
        "keep_unbinned": true,
        "mask_gaussian_fall_off": false,
        "precision": 4,
        "cross_correlation_mask": false,
        "remove_large_correlation_clusters": false,
        "use_mask_for_cluster_removal": false,
        "cluster_std":2,
        "mask_non_zero_voxels_ratio": 1.5,
        "non_zero_voxels_threshold": 0.05,
        "exclusion_radius_box_size_ratio": 0.5,
        "keep_binned": false,
        "keep_unbinned": true,
        "skip": false,
        "citation": "",
        "use_denoised_tomograms": false,
        "use_SUSAN": false,
        "ctf_correction_method": "defocus_file",
        "susan_padding": 200,
        "per_particle_ctf_correction": "phase_flip",
        "padding_policy": "zero",
        "normalization": "zm"
    },
    "DynamoImportTomograms": {
        "execution_method": "once",
        "import_tomograms": "both",
        "skip": false,
        "citation": "",
        "citation_link": "",
        "doi": ""
    },
    "GainCorrection": {
        "execution_method": "once",
        "parallel_execution": true,
        "method": "median",
        "median_filter": [3, 3],
        "skip": false,
        "citation": ""
    },
    "DynamoTiltSeriesAlignment": {
        "execution_method": "parallel",
        "use_newstack_for_binning": false,
        "checkpoint_module": true,
        "method": "rms",
        "generate_fiducial_files": true,
        "dynamo_tilt_stack_alignment_template_file": "configurations/global.doc",
        "config_file_name": "config.doc",
        "skip_ctf_estimation": true,
        "mask_radius_factor": 1.5,
        "template_side_length_factor": 4,
        "max_shift_ratio": 0.25,
        "gold_bead_size_in_nm_testing_range": 1,
        "test_range": true,
        "test_whole_range": false,
        "take_defaults": true,
        "detection_binning_factor": 4,
        "original_parameters": {
            "settings.computing.cpus": "1", 
            "settings.computing.gpuSet": "1",
            "settings.computing.gpuUse": 1, 
            "settings.computing.parallelCPUUse": 0, 
            "settings.general.amplitudeContrast": 0.9,
            "settings.general.apix": 1.701,
            "settings.general.nominalDefocus": -2.5, 
            "settings.general.sphericalAberration": 2.7,
            "settings.general.voltage": 300,
            "steps.alignWorkingStack.alignmentBinLevel": 4,
            "steps.binnedReconstruction.reconstructBinnedSIRT": 0,
            "steps.binnedReconstruction.reconstructBinnedWBP": 0,
            "steps.binnedReconstruction.reconstructBinnedWBPCTF": 0, 
            "steps.binnedReconstruction.reconstructionBinnedHeight": 500,
            "steps.binner.workingBinningFactor": 3,
            "steps.chainSelector.minimumMarkerDistance": 100,
            "steps.chainSelector.minimumMarkersPerTilt": 3, 
            "steps.chainSelector.minimumOccupancy": 15,
            "steps.chainSelector.relaxedMinimumOccupancy": 5,
            "steps.chainSelector.skipMarkedIndices": 1, 
            "steps.correctCTF.imodPhaseFlipExecutable": "ctfphaseflip", 
            "steps.correctCTF.phaseflipDefocusTolerance": 250, 
            "steps.correctCTF.phaseflipInterpolationWidth": 4, 
            "steps.correctCTF.phaseflipMaximumStripWidth": 1024,
            "steps.correctCTF.useImodPhaseFlip": 1,
            "steps.detectPeaks.beadRadius": 30, 
            "steps.detectPeaks.detectionBinningFactor": 4, 
            "steps.detectPeaks.maskRadius": 36,
            "steps.detectPeaks.templateSidelength": 72,
            "steps.estimateCTF.ctffind4": "ctffind",
            "steps.estimateCTF.ctffind4Card": "$DYNAMO_ROOT/examples/ctffind4Card.doc",
            "steps.estimateCTF.ctffind4Use": 0,
            "steps.estimateCTF.ctffind4UseCard": 0, 
            "steps.finalSelection.maximalResidualPerObservation": "Inf", 
            "steps.finalSelection.maximalResidualPerTrace": "Inf", 
            "steps.finalSelection.minimumAmountOfMarkersPerMicrograph": 2,
            "steps.fittingModel.psi": "single",
            "steps.fittingModel.psiRange": 2,
            "steps.fixAlignmentMarkers.stackZshift": 0, 
            "steps.fullReconstruction.centerBinnedCoordinatesValue": [0, 0, 0], 
            "steps.fullReconstruction.reconstructFullSIRT": 0, 
            "steps.fullReconstruction.reconstructFullWBP": 0, 
            "steps.fullReconstruction.reconstructFullWBPCTF": 0, 
            "steps.fullReconstruction.reconstructionFullSize": [400, 400, 400], 
            "steps.fullReconstruction.reconstructionShiftCenter": [0, 0, 0], 
            "steps.fullReconstruction.useCenterOnbinnedCoordinates": 0, 
            "steps.independentMarkerRefinement.gaussfiltOutlierDetectionCC": 1, 
            "steps.independentMarkerRefinement.iterationsRefineAverages": 1, 
            "steps.independentMarkerRefinement.recenterAverages": 1, 
            "steps.peakFeatures.symmetryOrder": 9, 
            "steps.peakSelector.useSobelForSelection": 1, 
            "steps.peakSelector.useSymmetryOrderForSelection": 1, 
            "steps.reindexer.excludeMultipleMatches": 1, 
            "steps.reindexer.exclusionRadiusMultipleMatches": 30, 
            "steps.reindexer.minimumOccupancy": 10, 
            "steps.reindexer.proximityThreshold3DThinning": 20, 
            "steps.reindexer.proximityThresholdReprojection": 10, 
            "steps.shifter.maximalHysteresis": 20, 
            "steps.shifter.maximalShift": 1000, 
            "steps.shifter.shiftInterval": 20, 
            "steps.shifter.skipManualDiscardsInShifts": 1, 
            "steps.tiltExtensor.knotGridSeparation": 20, 
            "steps.tiltExtensor.maximumOverlapProjections": 50, 
            "steps.tiltExtensor.minimalKnotContributions": 15, 
            "steps.tiltExtensor.rerunIterativeReindexingInExtensor": 1, 
            "steps.tiltExtensor.rerunTiltGapFillingInExtensor": 1, 
            "steps.tiltExtensor.separationYStripe": 20, 
            "steps.tiltExtensor.thresholdKnotDistance": 40, 
            "steps.tiltExtensor.widthYStripe": 40, 
            "steps.tiltGapFiller.estimateResidualsThreshold": 0, 
            "steps.tiltGapFiller.increaseDistanceThreshold": 10, 
            "steps.tiltGapFiller.initialDistanceThreshold": 10, 
            "steps.tiltGapFiller.maximalDistanceThreshold": 40, 
            "steps.tiltGapFiller.maximumMarkersDefiningGap": 4, 
            "steps.tiltGapFiller.minimumMarkersTargeted": 4, 
            "steps.tiltGapFiller.minimumOccupancyContributingChain": 4, 
            "steps.tiltGapFiller.residualsThreshold": 4, 
            "steps.tiltGapFiller.targetedOccupancyContributingChain": 10, 
            "steps.traceGapFiller.exclusionRadius": 30, 
            "steps.traceGapFiller.fastCC": 1, 
            "steps.traceGapFiller.maximalDeviationFromReprojection": 5, 
            "steps.trimMarkers.maximalMedianResidualMarker": 10, 
            "steps.trimMarkers.maximalResidualObservation": 10,
            "steps.trimMarkers.minimumOccupancy": 15, 
            "steps.trimMarkers.proximityDeletionThreshold": 80, 
            "steps.trimMarkers.proximityFusionThreshold": 10
        },
        "skip": false,
        "citation": "",
        "citation_link": "",
        "doi": ""
    },
    "DynamoCleanStacks": {
        "execution_method": "parallel",
        "show_truncated_stacks": false,
        "skip": false,
        "citation": "",
        "citation_link": "",
        "doi": ""
    },
    "DynamoAlignmentProject": {
        "randomize_angles": false,
        "bf": 4,
        "split_by_y": true,
        "atand_factor": 2,
        "use_SUSAN": false,
        "cone_flip": 0,
        "checkpoint_module": true,
        "noise": 1,
        "noise_scaling_factor": 0.7,
        "use_noise_classes": true,
        "susan_lowpass": 65,
        "dynamo_lowpass_factor": 0.5,
        "threshold_mode": 5,
        "threshold": 0.5,
        "area_search_mode": 1,
        "susan_padding": 200,
        "per_particle_ctf_correction": "phase_flip",
        "ssnr": [],
        "padding_policy": "zero",
        "normalization": "zm",
        "use_symmetry": true,
        "execution_method": "once",
        "reference": "average",
        "iterations": 0,
        "swap_particles": true,
        "refine_factor": 2,
        "show_results": false,
        "last_classification_binning":4,
        "projects_per_binning": 2,
        "alignment_method": "mra",
        "parallel_execution": true,
        "use_elliptic_mask": true,
        "radii_ratio": [0.5, 0.5, 0.5],
        "ellipsoid_smoothing_ratio": 0.16,
        "shift_limit_factor": 0.25,
        "discretization_bias": 0.33,
        "project_name": "mraProject",
        "destination": "matlab_gpu",
        "classes": 0,
        "selected_classes": [],
        "sampling": 0,
        "ite_r1": 1,    
        "nref_r1": 0,      
        "cone_range_r1": 0,    
        "cr_r1": 0,    
        "cone_sampling_r1": 0,     
        "cs_r1": 0,     
        "cone_flip_r1": 0,     
        "cone_check_peak_r1": 0,  
        "cone_freeze_reference_r1": 0,
        "inplane_range_r1": 0,     
        "inplane_sampling_r1": 0,     
        "inplane_flip_r1": 0,                       
        "inplane_check_peak_r1": 0,        
        "inplane_freeze_reference_r1": 0,       
        "refine_r1": 1,       
        "refine_factor_r1": 2,  
        "rf_r1": 0,       
        "rff_r1": 2,
        "rf_r2": 0,       
        "rff_r2": 2,
        "rf_r3": 0,       
        "rff_r3": 2,  
        "rf_r4": 0,       
        "rff_r4": 2,
        "rf_r5": 0,       
        "rff_r5": 2,
        "rf_r6": 0,       
        "rff_r6": 2,
        "rf_r7": 0,       
        "rff_r7": 2,  
        "rf_r8": 0,       
        "rff_r8": 2,
        "high_r1": 0,           
        "low_r1": 0,     
        "sym_r1": 0,                                
        "dim_r1": 0,                                
        "area_search_r1": 0,
        "area_search_modus_r1": 0,                  
        "separation_in_tomogram_r1": 0,             
        "limit_xy_check_peak_r1": 0,                
        "limit_z_check_peak_r1": 0,                 
        "use_CC_r1": 0,                             
        "localnc_r1": 0,                            
        "mra_r1": 0,                                
        "threshold_r1": 0,                          
        "threshold_modus_r1": 0,                    
        "threshold2_r1": 0,                         
        "threshold2_modus_r1": 0,                   
        "ccmatrix_r1": 0,     
        "ccmatrix_type_r1": 0,                      
        "ccmatrix_batch_r1": 0,                     
        "Xmatrix_r1": 0,                            
        "Xmatrix_maxMb_r1": 0,                      
        "PCA_r1": 0,                                
        "PCA_neigs_r1": 0,                          
        "kmeans_r1": "",                             
        "kmeans_ncluster_r1": 0,                    
        "kmeans_ncoefficients_r1": 0,               
        "nclass_r1": 0,                             
        "plugin_align_r1": 0,                       
        "plugin_post_r1": 0,                        
        "plugin_iter_r1": 0,                        
        "plugin_align_order_r1": 0,                 
        "plugin_post_order_r1": 0,                  
        "plugin_iter_order_r1": 0,                  
        "flags_r1": 0,                              
        "convergence_type_r1": 0,                   
        "convergence_r1": 0,                        
        "rings_r1": 0,                              
        "rings_random_r1": 0,                       
        "dynamic_mask_r1": 0,
        "citation": "",
        "mask_path": "",
        "mask_apix": 1,
        "SUSAN_defocus_min": 10000,
        "SUSAN_defocus_max": 50000,
        "SUSAN_ctf_box_size":400,
        "SUSAN_binning": 0,
        "ctf_correction_method": "defocus_file",
        "bandpass_method": "angles",
        "exclude_projections": 0
    },
    "PolishParticlesAndEstimateResolution": {
        "execution_method": "once",
        "particle_count": 0,
        "binning": 0,
        "sampling_iterations": 10,
        "gaussian_filter_std": 0.5,
        "use_SUSAN": false,
        "use_SUSAN_symmetry": true,
        "SUSAN_use_imod_style_ctf_correction": false,
        "per_particle_ctf_correction": "phase_flip",
        "ssnr": [1, 0],
        "padding_policy": "noise",
        "normalization": "zm",
        "use_symmetry": true,
        "use_SUSAN_averager": true,
        "show_figures": false,
        "save_figures": true
    },
    "CryoCARE": {
        "execution_method": "once",
        "repository_path": "",
        "cryoCARE_env": "",
        "binning": 0,
        "train_split": 0.9,
        "neighbouring_projections": 1,
        "tensorboard": true,
        "epochs": 200,
        "steps_per_epoch": 200,
        "batch_size": 16,
        "unet_kern_size": 3,
        "unet_n_depth": 3,
        "unet_n_first": 16,
        "learning_rate": 0.0004,
        "patch_shape": 512,
        "tomograms_to_train_on": 0
    },
    "DeepFinder": {
        "execution_method": "once",
        "checkpoint_module": true,
        "net_weights": "",
        "deep_finder_env": "",
        "repository_path": "",
        "binning": 0,
        "use_ctf_corrected_tomograms": true,
        "use_denoised_tomograms": true,
        "strategy": "spheres",
        "classes": 2,
        "batch_size": 20,
        "number_of_epochs": 150,
        "steps_per_epoch": 100,
        "random_shift_in_voxels": 0,
        "shift_factor": 0.25,
        "boot_strap": true,
        "direct_read": false,
        "steps_per_validation": 10,
        "trainings_set_fraction": 0.8,
        "patch_size": 0
    },
    "AreTomo": {
        "execution_method": "in_order",
        "input_stack_binning": 1,
        "reconstruction": false,
        "weighted_back_projection": true,
        "tilt_axis_refine_flag": 1,
        "correct_tilt_axis_offset": 0,
        "apply_given_tilt_axis_offset": false,
        "tilt_axis_offset": 0,
        "align_height_ratio": 0.75,
        "apply_dose_weighting": false,
        "sart": "20 5",
        "roi": "0 0",
        "roi_file": "",
        "patch": "0 0",
        "flip_volume": 1,
        "flip_intensity": 0,
        "citation": ""
    },
    "RelionAlignmentProject": {
        "execution_method": "once",
        "checkpoint_module": true,
        "binning": 0
    },
    "GridEdgeEraser": {
        "execution_method": "in_order",
        "detection_binning": 4,
        "grid_hole_diameter_in_um": 2,
        "output_shift_user": [0, 0],
        "output_shift_kernel_factor": [0, 0],
        "binarize_threshold_in_std": 3,
        "grid_detection_threshold_in_std": 3,
        "smooth_mask_border": true,
        "smooth_to_mean": true,
        "smoothing_exp_decay": -40,
        "cleaned_postfix": "gef",
        "relink_as_previous_output": false
    },
    "GenerateParticles": {
        "execution_method": "once",
        "generate_particles_method": "",
        "particles_table_path": "",
        "box_size": 1.0,
        "particles_binning": -1,
        "use_SUSAN": false,
        "ctf_correction_method": "defocus_file",
        "susan_padding": 200,
        "per_particle_ctf_correction": "phase_flip",
        "padding_policy": "zero",
        "normalization": "zm"
    },
    "IsoNet": {
        "execution_method": "once",
        "isonet_env": "source_isonet",
        "repository_path": "/usr/local/IsoNet",
        "tomograms_to_use": [],
        "steps_to_execute": {},
        "steps_to_execute_defaults": {
            "prepare_star": {
                "use_ctf_corrected_tomograms": false,
                "add_defocus_to_star": true,
                "tomograms_binning": -1,
                "folder_name": "tomograms",
                "output_star": "tomograms.star",
                "number_subtomos": -1
            },
            "deconv": {
                "star_file": "tomograms.star",
                "deconv_folder": "deconv",
                "snrfalloff": 0.7,
                "deconvstrength": -1,
                "highpassnyquist": -1,
                "chunk_size": -1,
                "overlap_rate": -1,
                "ncpu": -1
            },
            "make_mask": {
                "star_file": "tomograms.star",
                "mask_folder": "mask",
                "patch_size": -1,
                "mask_boundary": "",
                "density_percentage": 50,
                "std_percentage": 50,
                "z_crop": -1,
                "use_deconv_tomo": true
            },
            "extract": {
                "star_file": "tomograms.star",
                "subtomo_folder": "subtomo",
                "subtomo_star": "subtomo.star",
                "cube_size": -1,
                "crop_size": -1,
                "use_deconv_tomo": true
            },
            "refine": {
                "subtomo_star": "subtomo.star",
                "iterations": 30,
                "data_dir": "",
                "pretrained_model": "",
                "result_dir": "results",
                "preprocessing_ncpus": -1,
                "continue_from": "",
                "epochs": -1,
                "batch_size": -1,
                "steps_per_epoch": -1,
                "noise_level": [0.05,0.1,0.15,0.2],
                "noise_start_iter": [10,15,20,25],
                "noise_mode": "",
                "noise_dir": "",
                "learning_rate": -1,
                "drop_out": -1,
                "convs_per_depth": -1,
                "kernel": [],
                "unet_depth": -1,
                "filter_base": -1,
                "batch_normalization": -1,
                "normalize_percentile": -1
            },
            "predict": {
                "star_file": "tomograms.star",
                "model": "results/model_iter30.h5",
                "output_dir": "corrected_tomos",
                "cube_size": -1,
                "crop_size": -1,
                "batch_size": -1,
                "normalize_percentile": -1
            }
        }
    }
}
rickhooy commented 1 year ago

Figured it out. My frames are in superres; bin0 TS (>30GB) was likely causing the memory error. I set the 'ft_bin' (which I surmise is the motioncor2 flag) variable in the defaults.json file to '2' and reran. Motion corrected tilts are now bin1 (unbinned) and stack is expected size (3.4GB). Pipeline proceeded through DynamoAlignment step before next error (I'll post a separate issue).

ArtsemiY commented 1 year ago

Hi @rickhooy , thanks for writing! Sorry that it took longer to get to your issue, but I am happy that you was able to proceed further! I will take a look at the new issue soon.