search

denisecailab / minian

miniscope analysis pipeline with interactive visualizations
GNU General Public License v3.0
91 stars 35 forks source link

Storing video requires too much memory #268

Closed jordiabante closed 5 months ago

jordiabante commented 5 months ago

Hi,

We are using minian to correct motion in some calcium imaging data. We first generate HIS files which we then convert to AVI to use as input with minian. The input videos weigh about 8 GB and we're finding that ffmpeg requires an absurd amount of memory when executing either

Y_fm_chk = save_minian(Y.astype(float).rename("Y_fm_chk"), intpath, overwrite=True)

or

write_video(Y_fm_chk, "minian_mc.avi", minian_ds_path)

We have tested the code with a smaller AVI file, and it seemed to work well, although it also required a ridiculous amount of RAM. However, with the real videos, the process ends up getting killed (the machine we're using has 64 GB of RAM). We have also tried using dusk to limit the memory, but that doesn't seem to work either.

I'm attaching the code and the dependencies for your reference. Any help would be greatly appreciated.

Thanks, jordi

# load dependancies
import itertools as itt
import os
import sys

import holoviews as hv
import numpy as np
import xarray as xr
from dask.distributed import Client, LocalCluster
from holoviews.operation.datashader import datashade, regrid
from holoviews.util import Dynamic
from IPython.core.display import display

##############################################################################
# Our functions
##############################################################################

from datetime import datetime
def _print(*args, **kw):
    print("[%s]" % (datetime.now()),*args, **kw)

##############################################################################
# Configuration
##############################################################################

_print("Configuration...")

# Set up Initial Basic Parameters#
minian_path = "/media/HDD_2TB/jordi/repos/minian/minian"
dpath = "/media/HDD_4TB_1/jordi/calcium_imaging/hdrep_primary_cultures/uncorrected/"
minian_ds_path = "/media/HDD_4TB_1/jordi/calcium_imaging/hdrep_primary_cultures/minian/"
intpath = "/media/HDD_4TB_1/jordi/calcium_imaging/hdrep_primary_cultures/intermediate/"
subset = dict(frame=slice(0, None))
subset_mc = None
interactive = False
output_size = 100
n_workers = int(os.getenv("MINIAN_NWORKERS", 1))
param_save_minian = {
    "dpath": minian_ds_path,
    "meta_dict": dict(session=-1, animal=-2),
    "overwrite": True,
}

# Pre-processing Parameters#
param_load_videos = {
    "pattern": r"test_PRIM20231117_GrayScale_AVI.avi",
    "dtype": np.uint8,
    "downsample": dict(frame=1, height=1, width=1),
    "downsample_strategy": "subset",
}
param_denoise = {"method": "median", "ksize": 7}
param_background_removal = {"method": "tophat", "wnd": 15}

# Motion Correction Parameters#
subset_mc = None
param_estimate_motion = {"dim": "frame"}

# Initialization Parameters#
param_seeds_init = {
    "wnd_size": 1000,
    "method": "rolling",
    "stp_size": 500,
    "max_wnd": 15,
    "diff_thres": 3,
}
param_pnr_refine = {"noise_freq": 0.06, "thres": 1}
param_ks_refine = {"sig": 0.05}
param_seeds_merge = {"thres_dist": 10, "thres_corr": 0.8, "noise_freq": 0.06}
param_initialize = {"thres_corr": 0.8, "wnd": 10, "noise_freq": 0.06}
param_init_merge = {"thres_corr": 0.8}

# CNMF Parameters#
# param_get_noise = {"noise_range": (0.06, 0.5)}
# param_first_spatial = {
#     "dl_wnd": 10,
#     "sparse_penal": 0.01,
#     "size_thres": (25, None),
# }
# param_first_temporal = {
#     "noise_freq": 0.06,
#     "sparse_penal": 1,
#     "p": 1,
#     "add_lag": 20,
#     "jac_thres": 0.2,
# }
# param_first_merge = {"thres_corr": 0.8}
# param_second_spatial = {
#     "dl_wnd": 10,
#     "sparse_penal": 0.01,
#     "size_thres": (25, None),
# }
# param_second_temporal = {
#     "noise_freq": 0.06,
#     "sparse_penal": 1,
#     "p": 1,
#     "add_lag": 20,
#     "jac_thres": 0.4,
# }

os.environ["OMP_NUM_THREADS"] = "1"
os.environ["MKL_NUM_THREADS"] = "1"
os.environ["OPENBLAS_NUM_THREADS"] = "1"
os.environ["MINIAN_INTERMEDIATE"] = intpath

##############################################################################
# Import minian
##############################################################################

_print("Importing minian...")

sys.path.append(minian_path)
from minian.cnmf import (
    compute_AtC,
    compute_trace,
    get_noise_fft,
    smooth_sig,
    unit_merge,
    update_spatial,
    update_temporal,
    update_background,
)
from minian.initialization import (
    gmm_refine,
    initA,
    initC,
    intensity_refine,
    ks_refine,
    pnr_refine,
    seeds_init,
    seeds_merge,
)
from minian.motion_correction import apply_transform, estimate_motion
from minian.preprocessing import denoise, remove_background
from minian.utilities import (
    TaskAnnotation,
    get_optimal_chk,
    load_videos,
    open_minian,
    save_minian,
)
from minian.visualization import (
    CNMFViewer,
    VArrayViewer,
    generate_videos,
    visualize_gmm_fit,
    visualize_motion,
    visualize_preprocess,
    visualize_seeds,
    visualize_spatial_update,
    visualize_temporal_update,
    write_video,
)

##############################################################################
# module initialization
##############################################################################

_print("Initialiazing...")

dpath = os.path.abspath(dpath)
# hv.notebook_extension("bokeh", width=100)

##############################################################################
# start cluster
##############################################################################

_print("Starting cluster...")

# cluster = LocalCluster(
#     n_workers=n_workers,
#     memory_limit="30GB",
#     resources={"MEM": 1},
#     threads_per_worker=4,
#     dashboard_address=":8585",
# )
# annt_plugin = TaskAnnotation()
# cluster.scheduler.add_plugin(annt_plugin)
# client = Client(cluster)

##############################################################################
# load videos and initialize
##############################################################################

_print("Loading videos...")

# load and get optimal chunks
varr = load_videos(dpath, **param_load_videos)
chk, _ = get_optimal_chk(varr, dtype=float)

# subset video (default:none)
varr_ref = varr.sel(subset)

##############################################################################
# corrections
##############################################################################

_print("Glow removal...")

# glow removal
varr_min = varr_ref.min("frame").compute()
varr_ref = varr_ref - varr_min

_print("Denoising...")

# denoise
varr_ref = denoise(varr_ref, **param_denoise)

_print("Background removal...")

# background removal
varr_ref = remove_background(varr_ref, **param_background_removal)

_print("Motion correction...")

# motion correction
motion = estimate_motion(varr_ref.sel(subset_mc), **param_estimate_motion)

# _print("Saving estimated motion...")

# # save
# motion = save_minian(
#     motion.rename("motion").chunk({"frame": chk["frame"]}), **param_save_minian
# )

_print("Applying transformation...")

# apply transform
Y = apply_transform(varr_ref, motion, fill=0)

_print("Saving corrected object...")

# save transformed object
Y_fm_chk = save_minian(Y.astype(float).rename("Y_fm_chk"), intpath, overwrite=True)
# Y_hw_chk = save_minian(
#     Y_fm_chk.rename("Y_hw_chk"),
#     intpath,
#     overwrite=True,
#     chunks={"frame": -1, "height": chk["height"], "width": chk["width"]},
# )

_print("Saving corrected video...")

# store corrected video
# vid_arr = xr.concat([varr_ref, Y_fm_chk], "width").chunk({"width": -1})
write_video(Y_fm_chk, "minian_mc.avi", minian_ds_path)

##############################################################################
# close cluster
##############################################################################

# _print("Closing cluster client...")

# client.close()
# cluster.close()
# packages in environment at /media/HDD_4TB_1/jordi/miniconda3/envs/minian:
#
# Name                    Version                   Build  Channel
_libgcc_mutex             0.1                 conda_forge    conda-forge
_openmp_mutex             4.5                       2_gnu    conda-forge
anyio                     3.7.1              pyhd8ed1ab_0    conda-forge
argon2-cffi               23.1.0             pyhd8ed1ab_0    conda-forge
argon2-cffi-bindings      21.2.0           py38h01eb140_4    conda-forge
asciitree                 0.3.3                      py_2    conda-forge
asttokens                 2.4.1              pyhd8ed1ab_0    conda-forge
attrs                     23.2.0             pyh71513ae_0    conda-forge
backcall                  0.2.0              pyh9f0ad1d_0    conda-forge
beautifulsoup4            4.12.3             pyha770c72_0    conda-forge
bleach                    6.1.0              pyhd8ed1ab_0    conda-forge
blosc                     1.21.5               h0f2a231_0    conda-forge
bokeh                     1.4.0            py38h32f6830_1    conda-forge
brotli                    1.0.9                h166bdaf_9    conda-forge
brotli-bin                1.0.9                h166bdaf_9    conda-forge
brotli-python             1.0.9            py38hfa26641_9    conda-forge
brunsli                   0.1                  h9c3ff4c_0    conda-forge
bzip2                     1.0.8                hd590300_5    conda-forge
c-ares                    1.27.0               hd590300_0    conda-forge
c-blosc2                  2.13.2               hb4ffafa_0    conda-forge
ca-certificates           2024.2.2             hbcca054_0    conda-forge
cairo                     1.16.0            h9f066cc_1006    conda-forge
certifi                   2024.2.2           pyhd8ed1ab_0    conda-forge
cffi                      1.16.0           py38h6d47a40_0    conda-forge
cfitsio                   3.470                hb418390_7    conda-forge
cftime                    1.6.1            py38h71d37f0_0    conda-forge
charls                    2.2.0                h9c3ff4c_0    conda-forge
charset-normalizer        3.3.2              pyhd8ed1ab_0    conda-forge
click                     7.1.2              pyh9f0ad1d_0    conda-forge
cloudpickle               3.0.0              pyhd8ed1ab_0    conda-forge
colorama                  0.4.6              pyhd8ed1ab_0    conda-forge
colorcet                  3.1.0              pyhd8ed1ab_0    conda-forge
comm                      0.2.2              pyhd8ed1ab_0    conda-forge
curl                      7.76.1               h979ede3_1    conda-forge
cvxpy                     1.2.1            py38h578d9bd_0    conda-forge
cvxpy-base                1.2.1            py38h47df419_0    conda-forge
cycler                    0.12.1             pyhd8ed1ab_0    conda-forge
cytoolz                   0.12.3           py38h01eb140_0    conda-forge
dask                      2021.2.0           pyhd8ed1ab_0    conda-forge
dask-core                 2021.2.0           pyhd8ed1ab_0    conda-forge
datashader                0.12.1             pyh44b312d_1    conda-forge
datashape                 0.5.4                      py_1    conda-forge
dbus                      1.13.6               h5008d03_3    conda-forge
debugpy                   1.8.1            py38h17151c0_0    conda-forge
decorator                 5.1.1              pyhd8ed1ab_0    conda-forge
defusedxml                0.7.1              pyhd8ed1ab_0    conda-forge
distributed               2021.2.0         py38h578d9bd_0    conda-forge
ecos                      2.0.10           py38h71d37f0_1    conda-forge
eigen                     3.4.0                h00ab1b0_0    conda-forge
entrypoints               0.4                pyhd8ed1ab_0    conda-forge
exceptiongroup            1.2.0              pyhd8ed1ab_2    conda-forge
executing                 2.0.1              pyhd8ed1ab_0    conda-forge
expat                     2.6.2                h59595ed_0    conda-forge
fasteners                 0.17.3             pyhd8ed1ab_0    conda-forge
ffmpeg                    4.2.3                h167e202_0    conda-forge
ffmpeg-python             0.2.0                      py_0    conda-forge
fftw                      3.3.10          nompi_hc118613_108    conda-forge
fontconfig                2.14.2               h14ed4e7_0    conda-forge
freetype                  2.12.1               h267a509_2    conda-forge
fsspec                    2024.3.1           pyhca7485f_0    conda-forge
future                    1.0.0              pyhd8ed1ab_0    conda-forge
gettext                   0.21.1               h27087fc_0    conda-forge
giflib                    5.2.1                h0b41bf4_3    conda-forge
glib                      2.80.0               hf2295e7_1    conda-forge
glib-tools                2.80.0               hde27a5a_1    conda-forge
gmp                       6.3.0                h59595ed_1    conda-forge
gnutls                    3.6.13               h85f3911_1    conda-forge
graphite2                 1.3.13            h58526e2_1001    conda-forge
gst-plugins-base          1.14.5               h0935bb2_2    conda-forge
gstreamer                 1.18.5               h9f60fe5_3    conda-forge
harfbuzz                  2.7.2                ha5b49bf_1    conda-forge
hdf4                      4.2.15               h9772cbc_5    conda-forge
hdf5                      1.10.6          nompi_h6a2412b_1114    conda-forge
holoviews                 1.12.7             pyh9f0ad1d_1    conda-forge
icu                       67.1                 he1b5a44_0    conda-forge
idna                      3.6                pyhd8ed1ab_0    conda-forge
imagecodecs               2021.8.26        py38hb5ce8f7_1    conda-forge
imageio                   2.34.0             pyh4b66e23_0    conda-forge
importlib-metadata        7.0.2              pyha770c72_0    conda-forge
importlib_resources       6.3.2              pyhd8ed1ab_0    conda-forge
ipykernel                 6.29.3             pyhd33586a_0    conda-forge
ipython                   8.12.2             pyh41d4057_0    conda-forge
ipython_genutils          0.2.0                      py_1    conda-forge
ipywidgets                8.1.2              pyhd8ed1ab_0    conda-forge
jasper                    1.900.1           h07fcdf6_1006    conda-forge
jbig                      2.1               h7f98852_2003    conda-forge
jedi                      0.19.1             pyhd8ed1ab_0    conda-forge
jinja2                    2.11.3             pyhd8ed1ab_2    conda-forge
joblib                    1.3.2              pyhd8ed1ab_0    conda-forge
jpeg                      9e                   h0b41bf4_3    conda-forge
jsonschema                4.21.1             pyhd8ed1ab_0    conda-forge
jsonschema-specifications 2023.12.1          pyhd8ed1ab_0    conda-forge
jupyter                   1.0.0             pyhd8ed1ab_10    conda-forge
jupyter_client            7.3.4              pyhd8ed1ab_0    conda-forge
jupyter_console           6.6.3              pyhd8ed1ab_0    conda-forge
jupyter_core              5.7.2            py38h578d9bd_0    conda-forge
jupyter_server            1.24.0             pyhd8ed1ab_0    conda-forge
jupyterlab_pygments       0.3.0              pyhd8ed1ab_1    conda-forge
jupyterlab_widgets        3.0.10             pyhd8ed1ab_0    conda-forge
jxrlib                    1.1                  hd590300_3    conda-forge
kiwisolver                1.4.5            py38h7f3f72f_1    conda-forge
krb5                      1.17.2               h926e7f8_0    conda-forge
lame                      3.100             h166bdaf_1003    conda-forge
lcms2                     2.12                 hddcbb42_0    conda-forge
ld_impl_linux-64          2.40                 h41732ed_0    conda-forge
lerc                      3.0                  h9c3ff4c_0    conda-forge
libaec                    1.1.2                h59595ed_1    conda-forge
libblas                   3.9.0           20_linux64_openblas    conda-forge
libbrotlicommon           1.0.9                h166bdaf_9    conda-forge
libbrotlidec              1.0.9                h166bdaf_9    conda-forge
libbrotlienc              1.0.9                h166bdaf_9    conda-forge
libcblas                  3.9.0           20_linux64_openblas    conda-forge
libclang                  11.1.0          default_ha53f305_1    conda-forge
libcurl                   7.76.1               hc4aaa36_1    conda-forge
libdeflate                1.8                  h7f98852_0    conda-forge
libedit                   3.1.20191231         he28a2e2_2    conda-forge
libev                     4.33                 hd590300_2    conda-forge
libevent                  2.1.10               h9b69904_4    conda-forge
libexpat                  2.6.2                h59595ed_0    conda-forge
libffi                    3.4.2                h7f98852_5    conda-forge
libgcc-ng                 13.2.0               h807b86a_5    conda-forge
libgfortran-ng            13.2.0               h69a702a_5    conda-forge
libgfortran5              13.2.0               ha4646dd_5    conda-forge
libglib                   2.80.0               hf2295e7_1    conda-forge
libgomp                   13.2.0               h807b86a_5    conda-forge
libiconv                  1.17                 hd590300_2    conda-forge
libitk                    5.1.2                h86d312b_0    conda-forge
libitk-devel              5.1.2                h9c3ff4c_0    conda-forge
liblapack                 3.9.0           20_linux64_openblas    conda-forge
liblapacke                3.9.0           20_linux64_openblas    conda-forge
libllvm10                 10.0.1               he513fc3_3    conda-forge
libllvm11                 11.1.0               he0ac6c6_5    conda-forge
libnetcdf                 4.8.0           nompi_hcd642e3_103    conda-forge
libnghttp2                1.51.0               hdcd2b5c_0    conda-forge
libnsl                    2.0.1                hd590300_0    conda-forge
libopenblas               0.3.25          pthreads_h413a1c8_0    conda-forge
libopencv                 4.2.0                    py38_7    conda-forge
libpng                    1.6.43               h2797004_0    conda-forge
libpq                     12.3                 h255efa7_3    conda-forge
libsodium                 1.0.18               h36c2ea0_1    conda-forge
libsqlite                 3.45.2               h2797004_0    conda-forge
libssh2                   1.10.0               haa6b8db_3    conda-forge
libstdcxx-ng              13.2.0               h7e041cc_5    conda-forge
libtiff                   4.3.0                h6f004c6_2    conda-forge
libuuid                   2.38.1               h0b41bf4_0    conda-forge
libwebp                   1.2.2                h3452ae3_0    conda-forge
libwebp-base              1.2.2                h7f98852_1    conda-forge
libxcb                    1.13              h7f98852_1004    conda-forge
libxkbcommon              1.0.3                he3ba5ed_0    conda-forge
libxml2                   2.9.10               h68273f3_2    conda-forge
libzip                    1.9.2                hc869a4a_1    conda-forge
libzlib                   1.2.13               hd590300_5    conda-forge
libzopfli                 1.0.3                h9c3ff4c_0    conda-forge
llvmlite                  0.35.0           py38h4630a5e_1    conda-forge
locket                    1.0.0              pyhd8ed1ab_0    conda-forge
lz4-c                     1.9.3                h9c3ff4c_1    conda-forge
markdown                  3.6                pyhd8ed1ab_0    conda-forge
markupsafe                2.0.1            py38h497a2fe_1    conda-forge
matplotlib-base           3.2.2            py38h5d868c9_1    conda-forge
matplotlib-inline         0.1.6              pyhd8ed1ab_0    conda-forge
medpy                     0.4.0                    pypi_0    pypi
metis                     5.1.0             h59595ed_1007    conda-forge
minian                    1.2.1            py38ha770c72_1    conda-forge
mistune                   0.8.4           pyh1a96a4e_1006    conda-forge
msgpack-python            1.0.7            py38h7f3f72f_0    conda-forge
multipledispatch          0.6.0                      py_0    conda-forge
mysql-common              8.0.32               h14678bc_0    conda-forge
mysql-libs                8.0.32               h54cf53e_0    conda-forge
natsort                   8.4.0              pyhd8ed1ab_0    conda-forge
nbclassic                 1.0.0              pyhb4ecaf3_1    conda-forge
nbclient                  0.5.13             pyhd8ed1ab_0    conda-forge
nbconvert                 6.4.5              pyhd8ed1ab_2    conda-forge
nbconvert-core            6.4.5              pyhd8ed1ab_2    conda-forge
nbconvert-pandoc          6.4.5              pyhd8ed1ab_2    conda-forge
nbformat                  5.10.3             pyhd8ed1ab_0    conda-forge
ncurses                   6.4.20240210         h59595ed_0    conda-forge
nest-asyncio              1.6.0              pyhd8ed1ab_0    conda-forge
netcdf4                   1.5.7           nompi_py38h5e9db54_100    conda-forge
nettle                    3.6                  he412f7d_0    conda-forge
networkx                  2.4                        py_1    conda-forge
notebook                  6.5.6              pyha770c72_0    conda-forge
notebook-shim             0.2.4              pyhd8ed1ab_0    conda-forge
nspr                      4.35                 h27087fc_0    conda-forge
nss                       3.98                 h1d7d5a4_0    conda-forge
numba                     0.52.0           py38h51da96c_0    conda-forge
numcodecs                 0.12.1           py38h17151c0_0    conda-forge
numpy                     1.20.2           py38h9894fe3_0    conda-forge
opencv                    4.2.0                    py38_7    conda-forge
openh264                  2.1.1                h780b84a_0    conda-forge
openjpeg                  2.5.0                h7d73246_0    conda-forge
openssl                   1.1.1w               hd590300_0    conda-forge
osqp                      0.6.2.post0      py38h43a58ef_3    conda-forge
packaging                 24.0               pyhd8ed1ab_0    conda-forge
pandas                    1.2.3            py38h51da96c_0    conda-forge
pandoc                    2.19.2               h32600fe_2    conda-forge
pandocfilters             1.5.0              pyhd8ed1ab_0    conda-forge
panel                     0.8.0                    pypi_0    pypi
param                     1.9.3                      py_0    conda-forge
parso                     0.8.3              pyhd8ed1ab_0    conda-forge
partd                     1.4.1              pyhd8ed1ab_0    conda-forge
patsy                     0.5.6              pyhd8ed1ab_0    conda-forge
pcre2                     10.43                hcad00b1_0    conda-forge
pexpect                   4.9.0              pyhd8ed1ab_0    conda-forge
pickleshare               0.7.5                   py_1003    conda-forge
pillow                    9.1.1            py38h0ee0e06_1    conda-forge
pip                       24.0               pyhd8ed1ab_0    conda-forge
pixman                    0.43.2               h59595ed_0    conda-forge
pkgutil-resolve-name      1.3.10             pyhd8ed1ab_1    conda-forge
platformdirs              4.2.0              pyhd8ed1ab_0    conda-forge
pooch                     1.8.1              pyhd8ed1ab_0    conda-forge
prometheus_client         0.20.0             pyhd8ed1ab_0    conda-forge
prompt-toolkit            3.0.42             pyha770c72_0    conda-forge
prompt_toolkit            3.0.42               hd8ed1ab_0    conda-forge
psutil                    5.9.8            py38h01eb140_0    conda-forge
pthread-stubs             0.4               h36c2ea0_1001    conda-forge
ptyprocess                0.7.0              pyhd3deb0d_0    conda-forge
pure_eval                 0.2.2              pyhd8ed1ab_0    conda-forge
py-opencv                 4.2.0            py38h23f93f0_7    conda-forge
pycparser                 2.21               pyhd8ed1ab_0    conda-forge
pyct                      0.4.6                      py_0    conda-forge
pyct-core                 0.4.6                      py_0    conda-forge
pyfftw                    0.12.0           py38h9e8fb0f_3    conda-forge
pygments                  2.17.2             pyhd8ed1ab_0    conda-forge
pymetis                   2020.1           py38h69a4267_5    conda-forge
pyparsing                 3.1.2              pyhd8ed1ab_0    conda-forge
pysocks                   1.7.1              pyha2e5f31_6    conda-forge
python                    3.8.15          h257c98d_0_cpython    conda-forge
python-dateutil           2.9.0              pyhd8ed1ab_0    conda-forge
python-fastjsonschema     2.19.1             pyhd8ed1ab_0    conda-forge
python_abi                3.8                      4_cp38    conda-forge
pytz                      2024.1             pyhd8ed1ab_0    conda-forge
pyviz_comms               3.0.1              pyhd8ed1ab_0    conda-forge
pywavelets                1.3.0            py38h71d37f0_1    conda-forge
pyyaml                    6.0.1            py38h01eb140_1    conda-forge
pyzmq                     24.0.1           py38hfc09fa9_1    conda-forge
qdldl-python              0.1.5            py38h43a58ef_2    conda-forge
qt                        5.12.9               h763d07f_1    conda-forge
qtconsole-base            5.5.1              pyha770c72_0    conda-forge
qtpy                      2.4.1              pyhd8ed1ab_0    conda-forge
readline                  8.2                  h8228510_1    conda-forge
rechunker                 0.3.3              pyhd8ed1ab_0    conda-forge
referencing               0.34.0             pyhd8ed1ab_0    conda-forge
requests                  2.31.0             pyhd8ed1ab_0    conda-forge
rpds-py                   0.18.0           py38h0cc4f7c_0    conda-forge
scikit-image              0.18.1           py38h51da96c_0    conda-forge
scikit-learn              0.22.1           py38hcdab131_1    conda-forge
scipy                     1.9.1            py38hea3f02b_0    conda-forge
scs                       3.2.0            py38hc65120a_1    conda-forge
send2trash                1.8.2              pyh41d4057_0    conda-forge
setuptools                59.8.0           py38h578d9bd_1    conda-forge
simpleitk                 2.0.2            py38hb8ccc7d_1    conda-forge
six                       1.16.0             pyh6c4a22f_0    conda-forge
sk-video                  1.1.10             pyh24bf2e0_4    conda-forge
snappy                    1.1.10               h9fff704_0    conda-forge
sniffio                   1.3.1              pyhd8ed1ab_0    conda-forge
sortedcontainers          2.4.0              pyhd8ed1ab_0    conda-forge
soupsieve                 2.5                pyhd8ed1ab_1    conda-forge
sparse                    0.11.2                     py_0    conda-forge
sqlite                    3.45.2               h2c6b66d_0    conda-forge
stack_data                0.6.2              pyhd8ed1ab_0    conda-forge
statsmodels               0.13.2           py38h71d37f0_0    conda-forge
tblib                     3.0.0              pyhd8ed1ab_0    conda-forge
terminado                 0.18.1             pyh0d859eb_0    conda-forge
testpath                  0.6.0              pyhd8ed1ab_0    conda-forge
tifffile                  2021.11.2          pyhd8ed1ab_0    conda-forge
tk                        8.6.13          noxft_h4845f30_101    conda-forge
toolz                     0.12.1             pyhd8ed1ab_0    conda-forge
tornado                   6.1              py38h0a891b7_3    conda-forge
tqdm                      4.66.2             pyhd8ed1ab_0    conda-forge
traitlets                 5.14.2             pyhd8ed1ab_0    conda-forge
typing-extensions         4.10.0               hd8ed1ab_0    conda-forge
typing_extensions         4.10.0             pyha770c72_0    conda-forge
urllib3                   2.2.1              pyhd8ed1ab_0    conda-forge
wcwidth                   0.2.13             pyhd8ed1ab_0    conda-forge
webencodings              0.5.1              pyhd8ed1ab_2    conda-forge
websocket-client          1.7.0              pyhd8ed1ab_0    conda-forge
wheel                     0.42.0             pyhd8ed1ab_0    conda-forge
widgetsnbextension        4.0.10             pyhd8ed1ab_0    conda-forge
x264                      1!152.20180806       h14c3975_0    conda-forge
xarray                    0.16.2             pyhd8ed1ab_0    conda-forge
xorg-kbproto              1.0.7             h7f98852_1002    conda-forge
xorg-libice               1.1.1                hd590300_0    conda-forge
xorg-libsm                1.2.4                h7391055_0    conda-forge
xorg-libx11               1.8.4                h0b41bf4_0    conda-forge
xorg-libxau               1.0.11               hd590300_0    conda-forge
xorg-libxdmcp             1.1.3                h7f98852_0    conda-forge
xorg-libxext              1.3.4                h0b41bf4_2    conda-forge
xorg-libxrender           0.9.10            h7f98852_1003    conda-forge
xorg-renderproto          0.11.1            h7f98852_1002    conda-forge
xorg-xextproto            7.3.0             h0b41bf4_1003    conda-forge
xorg-xproto               7.0.31            h7f98852_1007    conda-forge
xz                        5.2.6                h166bdaf_0    conda-forge
yaml                      0.2.5                h7f98852_2    conda-forge
zarr                      2.17.1             pyhd8ed1ab_0    conda-forge
zeromq                    4.3.5                h59595ed_1    conda-forge
zfp                       0.5.5                h9c3ff4c_8    conda-forge
zict                      3.0.0              pyhd8ed1ab_0    conda-forge
zipp                      3.17.0             pyhd8ed1ab_0    conda-forge
zlib                      1.2.13               hd590300_5    conda-forge
zlib-ng                   2.0.7                h0b41bf4_0    conda-forge
zstd                      1.5.5                hfc55251_0    conda-forge
GrowlingM1ke commented 5 months ago

If you're confident it's an ffmpeg issue then it shouldn't be save_minian as that only saves the data structure to a zarr array and as far as I understand it doesn't make any calls to ffmpeg https://github.com/denisecailab/minian/blob/f64c456ca027200e19cf40a80f0596106918fd09/minian/utilities.py#L440.

This is a bit of a long shot but since you didn't have any issues with loading the video into an xarray I would investigate what datatype is currently stored in the Y array and maybe converting that into something more memory efficient before calling write_video. Another solution would be to slice the array into smaller chunks (in halves or quarters?) generating the video files and then combining them.

Edit 1: Forget my suggestion of changing the datatype, it seems that this is already accounted for in the code, as they convert it to uint8.

Edit 2: From what I can see from the code it should be only saving it in chunks: https://github.com/denisecailab/minian/blob/f64c456ca027200e19cf40a80f0596106918fd09/minian/visualization.py#L1275C5-L1276C53

Might be worth investigating how your data is chunked and maybe chunking it into smaller pieces.

austinbaggetta commented 5 months ago

Hi Jordiabante,

8GB is quite small, so I'm surprised you're having an issue converting Y to Y_fm_chk and saving the motion correction video. You could try converting to float32 instead of float64, but I suspect this isn't the issue.

Looking at your Dask cluster, you are initializing the cluster with 1 worker that then gets allocated a memory limit of 30GB:

## Your current cluster configuration
n_workers = int(os.getenv("MINIAN_NWORKERS", 1))

cluster = LocalCluster(
     n_workers=n_workers,
     memory_limit="30GB",
     resources={"MEM": 1},
     threads_per_worker=4,
     dashboard_address=":8585",
)

I have never tried to use Minian with one worker, and I suspect this may be the issue (even with the 4 threads, or 4 cores per worker). I would recommend doing anywhere from 5-7 workers and adjusting their memory limit accordingly, while also decreasing the threads_per_worker to 2 cores:

## Potential cluster configuration
n_workers = int(os.getenv("MINIAN_NWORKERS", 5))

cluster = LocalCluster(
     n_workers=n_workers,
     memory_limit="7GB",
     resources={"MEM": 1},
     threads_per_worker=2,
     dashboard_address=":8585",
)

The configuration above would work with your computer's 64GB of RAM, as it would only utilize (at most) 35GB.

Let me know if this works!

jordiabante commented 5 months ago

Hi all,

Thanks for your suggestions.

@GrowlingM1ke, I believe save_minian does call ffmpeg (I see a call in the stderr that corresponds to that part of the code in principle). Regarding the chunks we are using function get_optimal_chk to obtain the optimal size. Shouldn't that take care of it?

@austinbaggetta I modified the cluster set up to:

## Potential cluster configuration
n_workers = int(os.getenv("MINIAN_NWORKERS", 5))

cluster = LocalCluster(
     n_workers=n_workers,
     memory_limit="7GB",
     resources={"MEM": 1},
     threads_per_worker=2,
     dashboard_address=":8585"
)

annt_plugin = TaskAnnotation()
cluster.scheduler.add_plugin(annt_plugin)
client = Client(cluster)

and I get the following error:

Task exception was never retrieved
future: <Task finished name='Task-13' coro=<_wrap_awaitable() done, defined at /media/HDD_4TB_1/jordi/miniconda3/envs/minian/lib/python3.8/asyncio/tasks.py:688> exception=RuntimeError('\n        An attempt has been made to start a new process before the\n        current process has finished its bootstrapping phase.\n\n        This probably means that you are not using fork to start your\n        child processes and you have forgotten to use the proper idiom\n        in the main module:\n\n            if __name__ == \'__main__\':\n                freeze_support()\n                ...\n\n        The "freeze_support()" line can be omitted if the program\n        is not going to be frozen to produce an executable.')>
Traceback (most recent call last):
  File "/media/HDD_4TB_1/jordi/miniconda3/envs/minian/lib/python3.8/asyncio/tasks.py", line 695, in _wrap_awaitable
    return (yield from awaitable.__await__())
  File "/media/HDD_4TB_1/jordi/miniconda3/envs/minian/lib/python3.8/site-packages/distributed/core.py", line 284, in _
    await self.start()
  File "/media/HDD_4TB_1/jordi/miniconda3/envs/minian/lib/python3.8/site-packages/distributed/nanny.py", line 295, in start
    response = await self.instantiate()
  File "/media/HDD_4TB_1/jordi/miniconda3/envs/minian/lib/python3.8/site-packages/distributed/nanny.py", line 378, in instantiate
    result = await self.process.start()
  File "/media/HDD_4TB_1/jordi/miniconda3/envs/minian/lib/python3.8/site-packages/distributed/nanny.py", line 575, in start
    await self.process.start()
  File "/media/HDD_4TB_1/jordi/miniconda3/envs/minian/lib/python3.8/site-packages/distributed/process.py", line 33, in _call_and_set_future
    res = func(*args, **kwargs)
  File "/media/HDD_4TB_1/jordi/miniconda3/envs/minian/lib/python3.8/site-packages/distributed/process.py", line 203, in _start
    process.start()
  File "/media/HDD_4TB_1/jordi/miniconda3/envs/minian/lib/python3.8/multiprocessing/process.py", line 121, in start
    self._popen = self._Popen(self)
  File "/media/HDD_4TB_1/jordi/miniconda3/envs/minian/lib/python3.8/multiprocessing/context.py", line 284, in _Popen
    return Popen(process_obj)
  File "/media/HDD_4TB_1/jordi/miniconda3/envs/minian/lib/python3.8/multiprocessing/popen_spawn_posix.py", line 32, in __init__
    super().__init__(process_obj)
  File "/media/HDD_4TB_1/jordi/miniconda3/envs/minian/lib/python3.8/multiprocessing/popen_fork.py", line 19, in __init__
    self._launch(process_obj)
  File "/media/HDD_4TB_1/jordi/miniconda3/envs/minian/lib/python3.8/multiprocessing/popen_spawn_posix.py", line 42, in _launch
    prep_data = spawn.get_preparation_data(process_obj._name)
  File "/media/HDD_4TB_1/jordi/miniconda3/envs/minian/lib/python3.8/multiprocessing/spawn.py", line 154, in get_preparation_data
    _check_not_importing_main()
  File "/media/HDD_4TB_1/jordi/miniconda3/envs/minian/lib/python3.8/multiprocessing/spawn.py", line 134, in _check_not_importing_main
    raise RuntimeError('''
RuntimeError: 
        An attempt has been made to start a new process before the
        current process has finished its bootstrapping phase.

        This probably means that you are not using fork to start your
        child processes and you have forgotten to use the proper idiom
        in the main module:

            if __name__ == '__main__':
                freeze_support()
                ...

        The "freeze_support()" line can be omitted if the program
        is not going to be frozen to produce an executable.

Do you have any ideas? I am not familiar at all with Dask, so this could very well be something very basic.

Thanks!

jordiabante commented 5 months ago

Hi all,

I modified the cluster definition to:

if __name__ == '__main__':

    ## Potential cluster configuration
    n_workers = int(os.getenv("MINIAN_NWORKERS", 7))

    cluster = LocalCluster(
         n_workers=n_workers,
         memory_limit="5GB",
         resources={"MEM": 1},
         threads_per_worker=2,
         dashboard_address=":8585",
        start_method='fork'  # or 'spawn'
    )

    annt_plugin = TaskAnnotation()
    cluster.scheduler.add_plugin(annt_plugin)

    try:
        client = Client(cluster)
        # Your Dask computations using the client
    finally:
        # Close the client and cluster explicitly
        client.close()
        cluster.close()

This doesn't produce the same error. However, the code seems to get stuck (using 5 and 7 workers) at the same point it does when I don't use a cluster. At this point the 64Gb are all being used (ignoring the restriction by the cluster) and the swp memory starts increasing until it maxes out and then the process gets killed.

[2024-04-18 12:03:04.356140] Configuration...
[2024-04-18 12:03:04.356336] Importing minian...
[2024-04-18 12:03:39.139878] Initialiazing...
[2024-04-18 12:03:39.139929] Starting cluster...
[2024-04-18 12:03:40.667591] Configuration...
[2024-04-18 12:03:40.667641] Importing minian...
[2024-04-18 12:03:40.676275] Configuration...
[2024-04-18 12:03:40.676334] Importing minian...
[2024-04-18 12:03:40.676715] Configuration...
[2024-04-18 12:03:40.676764] Importing minian...
[2024-04-18 12:03:40.685165] Configuration...
[2024-04-18 12:03:40.685230] Importing minian...
[2024-04-18 12:03:40.691049] Configuration...
[2024-04-18 12:03:40.691113] Importing minian...
[2024-04-18 12:03:40.692652] Configuration...
[2024-04-18 12:03:40.692708] Importing minian...
[2024-04-18 12:03:40.717103] Configuration...
[2024-04-18 12:03:40.717164] Importing minian...
[2024-04-18 12:03:43.871927] Initialiazing...
[2024-04-18 12:03:43.872141] Starting cluster...
[2024-04-18 12:03:43.872180] Loading videos...
[2024-04-18 12:03:43.894702] Initialiazing...
[2024-04-18 12:03:43.894741] Starting cluster...
[2024-04-18 12:03:43.894747] Loading videos...
loading 1 videos in folder /media/HDD_4TB_1/jordi/calcium_imaging/hdrep_primary_cultures/uncorrected
loading 1 videos in folder /media/HDD_4TB_1/jordi/calcium_imaging/hdrep_primary_cultures/uncorrected
[2024-04-18 12:03:43.922585] Initialiazing...
[2024-04-18 12:03:43.922622] Starting cluster...
[2024-04-18 12:03:43.922628] Loading videos...
[2024-04-18 12:03:43.922760] Initialiazing...
[2024-04-18 12:03:43.922790] Starting cluster...
[2024-04-18 12:03:43.922796] Loading videos...
loading 1 videos in folder /media/HDD_4TB_1/jordi/calcium_imaging/hdrep_primary_cultures/uncorrected
loading 1 videos in folder /media/HDD_4TB_1/jordi/calcium_imaging/hdrep_primary_cultures/uncorrected
[2024-04-18 12:03:43.924899] Initialiazing...
[2024-04-18 12:03:43.924936] Starting cluster...
[2024-04-18 12:03:43.924942] Loading videos...
loading 1 videos in folder /media/HDD_4TB_1/jordi/calcium_imaging/hdrep_primary_cultures/uncorrected
[2024-04-18 12:03:43.954685] Initialiazing...
[2024-04-18 12:03:43.954718] Starting cluster...
[2024-04-18 12:03:43.954725] Loading videos...
loading 1 videos in folder /media/HDD_4TB_1/jordi/calcium_imaging/hdrep_primary_cultures/uncorrected
[2024-04-18 12:03:43.959545] Initialiazing...
[2024-04-18 12:03:43.959578] Starting cluster...
[2024-04-18 12:03:43.959584] Loading videos...
loading 1 videos in folder /media/HDD_4TB_1/jordi/calcium_imaging/hdrep_primary_cultures/uncorrected
[2024-04-18 12:03:44.535566] Glow removal...
[2024-04-18 12:03:44.535610] Glow removal...
[2024-04-18 12:03:44.535615] Glow removal...
[2024-04-18 12:03:44.535614] Glow removal...
[2024-04-18 12:03:44.535648] Glow removal...
[2024-04-18 12:03:44.535660] Glow removal...
[2024-04-18 12:03:44.535673] Glow removal...
ffmpeg version 4.2.3ffmpeg version 4.2.3 Copyright (c) 2000-2020 the FFmpeg developers
 Copyright (c) 2000-2020 the FFmpeg developers
  built with gcc 7.5.0 (crosstool-NG 1.24.0.123_1667d2b)
  configuration: --prefix=/home/conda/feedstock_root/build_artifacts/ffmpeg_1590573566052/_h_env_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_plac --cc=/home/conda/feedstock_root/build_artifacts/ffmpeg_1590573566052/_build_env/bin/x86_64-conda_cos6-linux-gnu-cc --disable-doc --disable-openssl --enable-avresample --enable-gnutls --enable-gpl --enable-hardcoded-tables --enable-libfreetype --enable-libopenh264 --enable-libx264 --enable-pic --enable-pthreads --enable-shared --enable-static --enable-version3 --enable-zlib --enable-libmp3lame
  built with gcc 7.5.0 (crosstool-NG 1.24.0.123_1667d2b)
  configuration: --prefix=/home/conda/feedstock_root/build_artifacts/ffmpeg_1590573566052/_h_env_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_plac --cc=/home/conda/feedstock_root/build_artifacts/ffmpeg_1590573566052/_build_env/bin/x86_64-conda_cos6-linux-gnu-cc --disable-doc --disable-openssl --enable-avresample --enable-gnutls --enable-gpl --enable-hardcoded-tables --enable-libfreetype --enable-libopenh264 --enable-libx264 --enable-pic --enable-pthreads --enable-shared --enable-static --enable-version3 --enable-zlib --enable-libmp3lame
ffmpeg version 4.2.3  libavutil      56. 31.100 / 56. 31.100
  libavcodec     58. 54.100 / 58. 54.100
 Copyright (c) 2000-2020 the FFmpeg developers  libavutil      56. 31.100 / 56. 31.100

  built with gcc 7.5.0 (crosstool-NG 1.24.0.123_1667d2b)
  configuration: --prefix=/home/conda/feedstock_root/build_artifacts/ffmpeg_1590573566052/_h_env_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_plac --cc=/home/conda/feedstock_root/build_artifacts/ffmpeg_1590573566052/_build_env/bin/x86_64-conda_cos6-linux-gnu-cc --disable-doc --disable-openssl --enable-avresample --enable-gnutls --enable-gpl --enable-hardcoded-tables --enable-libfreetype --enable-libopenh264 --enable-libx264 --enable-pic --enable-pthreads --enable-shared --enable-static --enable-version3 --enable-zlib --enable-libmp3lame
ffmpeg version 4.2.3 Copyright (c) 2000-2020 the FFmpeg developers
  built with gcc 7.5.0 (crosstool-NG 1.24.0.123_1667d2b)
  configuration: --prefix=/home/conda/feedstock_root/build_artifacts/ffmpeg_1590573566052/_h_env_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_plac --cc=/home/conda/feedstock_root/build_artifacts/ffmpeg_1590573566052/_build_env/bin/x86_64-conda_cos6-linux-gnu-cc --disable-doc --disable-openssl --enable-avresample --enable-gnutls --enable-gpl --enable-hardcoded-tables --enable-libfreetype --enable-libopenh264 --enable-libx264 --enable-pic --enable-pthreads --enable-shared --enable-static --enable-version3 --enable-zlib --enable-libmp3lame
  libavutil      56. 31.100 / 56. 31.100
  libavcodec     58. 54.100 / 58. 54.100
  libavformat    58. 29.100 / 58. 29.100
  libavdevice    58.  8.100 / 58.  8.100
  libavfilter     7. 57.100 /  7. 57.100
  libavresample   4.  0.  0 /  4.  0.  0
  libavutil      56. 31.100 / 56. 31.100
  libswscale      5.  5.100 /  5.  5.100
  libavcodec     58. 54.100 / 58. 54.100
  libswresample   3.  5.100 /  3.  5.100
  libavformat    58. 29.100 / 58. 29.100
  libpostproc    55.  5.100 / 55.  5.100
  libavdevice    58.  8.100 / 58.  8.100
  libavfilter     7. 57.100 /  7. 57.100
  libavresample   4.  0.  0 /  4.  0.  0
  libswscale      5.  5.100 /  5.  5.100
  libswresample   3.  5.100 /  3.  5.100
  libpostproc    55.  5.100 / 55.  5.100
ffmpeg version 4.2.3 Copyright (c) 2000-2020 the FFmpeg developers
  built with gcc 7.5.0 (crosstool-NG 1.24.0.123_1667d2b)
  configuration: --prefix=/home/conda/feedstock_root/build_artifacts/ffmpeg_1590573566052/_h_env_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_plac --cc=/home/conda/feedstock_root/build_artifacts/ffmpeg_1590573566052/_build_env/bin/x86_64-conda_cos6-linux-gnu-cc --disable-doc --disable-openssl --enable-avresample --enable-gnutls --enable-gpl --enable-hardcoded-tables --enable-libfreetype --enable-libopenh264 --enable-libx264 --enable-pic --enable-pthreads --enable-shared --enable-static --enable-version3 --enable-zlib --enable-libmp3lame
  libavutil      56. 31.100 / 56. 31.100
  libavcodec     58. 54.100 / 58. 54.100
  libavformat    58. 29.100 / 58. 29.100
  libavdevice    58.  8.100 / 58.  8.100
  libavfilter     7. 57.100 /  7. 57.100
  libavresample   4.  0.  0 /  4.  0.  0
  libswscale      5.  5.100 /  5.  5.100
  libswresample   3.  5.100 /  3.  5.100
  libpostproc    55.  5.100 / 55.  5.100
Input #0, avi, from '/media/HDD_4TB_1/jordi/calcium_imaging/hdrep_primary_cultures/uncorrected/test_PRIM20231117_GrayScale_AVI.avi':
  Duration: Input #0, avi, from '/media/HDD_4TB_1/jordi/calcium_imaging/hdrep_primary_cultures/uncorrected/test_PRIM20231117_GrayScale_AVI.avi':
00:11:00.20  Duration: , start: 00:11:00.200.000000, start: , bitrate: 0.000000104203 kb/s, bitrate: 
104203 kb/s
    Stream #0:0    Stream #0:0: Video: rawvideo, pal8, 806x806, 104207 kb/s: Video: rawvideo, pal8, 806x806, 104207 kb/s, , 20 fps, 20 fps, 20 tbr, 20 tbr, 20 tbn, 20 tbn, 20 tbc20 tbc

Stream mapping:
Stream mapping:
  Stream #0:0 -> #0:0  Stream #0:0 -> #0:0 (rawvideo (native) -> rawvideo (native)) (rawvideo (native) -> rawvideo (native))

Press [q] to stop, [?] for help
Press [q] to stop, [?] for help
  libavformat    58. 29.100 / 58. 29.100
  libavdevice    58.  8.100 / 58.  8.100
  libavfilter     7. 57.100 /  7. 57.100
  libavresample   4.  0.  0 /  4.  0.  0
  libswscale      5.  5.100 /  5.  5.100
  libswresample   3.  5.100 /  3.  5.100
  libpostproc    55.  5.100 / 55.  5.100
  libavcodec     58. 54.100 / 58. 54.100
  libavformat    58. 29.100 / 58. 29.100
  libavdevice    58.  8.100 / 58.  8.100
  libavfilter     7. 57.100 /  7. 57.100
  libavresample   4.  0.  0 /  4.  0.  0
  libswscale      5.  5.100 /  5.  5.100
  libswresample   3.  5.100 /  3.  5.100
  libpostproc    55.  5.100 / 55.  5.100
ffmpeg version 4.2.3 Copyright (c) 2000-2020 the FFmpeg developers
  built with gcc 7.5.0 (crosstool-NG 1.24.0.123_1667d2b)
  configuration: --prefix=/home/conda/feedstock_root/build_artifacts/ffmpeg_1590573566052/_h_env_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_plac --cc=/home/conda/feedstock_root/build_artifacts/ffmpeg_1590573566052/_build_env/bin/x86_64-conda_cos6-linux-gnu-cc --disable-doc --disable-openssl --enable-avresample --enable-gnutls --enable-gpl --enable-hardcoded-tables --enable-libfreetype --enable-libopenh264 --enable-libx264 --enable-pic --enable-pthreads --enable-shared --enable-static --enable-version3 --enable-zlib --enable-libmp3lame
  libavutil      56. 31.100 / 56. 31.100
  libavcodec     58. 54.100 / 58. 54.100
  libavformat    58. 29.100 / 58. 29.100
  libavdevice    58.  8.100 / 58.  8.100
  libavfilter     7. 57.100 /  7. 57.100
  libavresample   4.  0.  0 /  4.  0.  0
  libswscale      5.  5.100 /  5.  5.100
  libswresample   3.  5.100 /  3.  5.100
  libpostproc    55.  5.100 / 55.  5.100
Input #0, avi, from '/media/HDD_4TB_1/jordi/calcium_imaging/hdrep_primary_cultures/uncorrected/test_PRIM20231117_GrayScale_AVI.avi':
  Duration: 00:11:00.20, start: 0.000000, bitrate: 104203 kb/s
ffmpeg version 4.2.3    Stream #0:0 Copyright (c) 2000-2020 the FFmpeg developers: Video: rawvideo, pal8, 806x806, 104207 kb/s
,   built with gcc 7.5.0 (crosstool-NG 1.24.0.123_1667d2b)
  configuration: --prefix=/home/conda/feedstock_root/build_artifacts/ffmpeg_1590573566052/_h_env_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_plac --cc=/home/conda/feedstock_root/build_artifacts/ffmpeg_1590573566052/_build_env/bin/x86_64-conda_cos6-linux-gnu-cc --disable-doc --disable-openssl --enable-avresample --enable-gnutls --enable-gpl --enable-hardcoded-tables --enable-libfreetype --enable-libopenh264 --enable-libx264 --enable-pic --enable-pthreads --enable-shared --enable-static --enable-version3 --enable-zlib --enable-libmp3lame
20 fps, 20 tbr, 20 tbn, 20 tbc
  libavutil      56. 31.100 / 56. 31.100
  libavcodec     58. 54.100 / 58. 54.100
  libavformat    58. 29.100 / 58. 29.100
  libavdevice    58.  8.100 / 58.  8.100
  libavfilter     7. 57.100 /  7. 57.100
  libavresample   4.  0.  0 /  4.  0.  0
  libswscale      5.  5.100 /  5.  5.100
  libswresample   3.  5.100 /  3.  5.100
  libpostproc    55.  5.100 / 55.  5.100
Stream mapping:
  Stream #0:0 -> #0:0 (rawvideo (native) -> rawvideo (native))
Press [q] to stop, [?] for help
Input #0, avi, from '/media/HDD_4TB_1/jordi/calcium_imaging/hdrep_primary_cultures/uncorrected/test_PRIM20231117_GrayScale_AVI.avi':
  Duration: 00:11:00.20, start: 0.000000, bitrate: 104203 kb/s
    Stream #0:0: Video: rawvideo, pal8, 806x806, 104207 kb/s, 20 fps, 20 tbr, 20 tbn, 20 tbc
Input #0, avi, from '/media/HDD_4TB_1/jordi/calcium_imaging/hdrep_primary_cultures/uncorrected/test_PRIM20231117_GrayScale_AVI.avi':
  Duration: 00:11:00.20, start: 0.000000, bitrate: 104203 kb/s
    Stream #0:0: Video: rawvideo, pal8, 806x806, 104207 kb/s, 20 fps, 20 tbr, 20 tbn, 20 tbc
Stream mapping:
  Stream #0:0 -> #0:0 (rawvideo (native) -> rawvideo (native))
Press [q] to stop, [?] for help
Stream mapping:
  Stream #0:0 -> #0:0 (rawvideo (native) -> rawvideo (native))
Press [q] to stop, [?] for help
Input #0, avi, from '/media/HDD_4TB_1/jordi/calcium_imaging/hdrep_primary_cultures/uncorrected/test_PRIM20231117_GrayScale_AVI.avi':
  Duration: 00:11:00.20, start: 0.000000, bitrate: 104203 kb/s
    Stream #0:0: Video: rawvideo, pal8, 806x806, 104207 kb/s, 20 fps, 20 tbr, 20 tbn, 20 tbc
Input #0, avi, from '/media/HDD_4TB_1/jordi/calcium_imaging/hdrep_primary_cultures/uncorrected/test_PRIM20231117_GrayScale_AVI.avi':
  Duration: 00:11:00.20, start: 0.000000, bitrate: 104203 kb/s
    Stream #0:0: Video: rawvideo, pal8, 806x806, 104207 kb/sStream mapping:
,   Stream #0:0 -> #0:020 fps,  (rawvideo (native) -> rawvideo (native))20 tbr, 
20 tbn, Press [q] to stop, [?] for help
20 tbc
Stream mapping:
  Stream #0:0 -> #0:0 (rawvideo (native) -> rawvideo (native))
Press [q] to stop, [?] for help
Output #0, rawvideo, to 'pipe:':
  Metadata:
    encoder         : Lavf58.29.100
Output #0, rawvideo, to 'pipe:':
Output #0, rawvideo, to 'pipe:':
  Metadata:
    encoder         : Lavf58.29.100  Metadata:
    Stream #0:0    encoder         : Output #0, rawvideo, to 'pipe:':
: Video: rawvideo (Y800 / 0x30303859), gray, 806x806, q=2-31, 103941 kb/sLavf58.29.100, 
20 fps,   Metadata:
20 tbn, 20 tbc    encoder         : 
Lavf58.29.100    Metadata:

      encoder         : Lavc58.54.100 rawvideo
    Stream #0:0: Video: rawvideo (Y800 / 0x30303859), gray, 806x806, q=2-31, 103941 kb/s, 20 fps, 20 tbn, 20 tbc
    Metadata:
    Stream #0:0      encoder         : : Video: rawvideo (Y800 / 0x30303859), gray, 806x806, q=2-31, 103941 kb/sLavc58.54.100 rawvideo, 
20 fps, 20 tbn, 20 tbc
    Metadata:
      encoder         : Lavc58.54.100 rawvideo
Output #0, rawvideo, to 'pipe:':
  Metadata:
    encoder         : Lavf58.29.100
Output #0, rawvideo, to 'pipe:':
  Metadata:
    encoder         : Lavf58.29.100    Stream #0:0Output #0, rawvideo, to 'pipe:':

: Video: rawvideo (Y800 / 0x30303859), gray, 806x806, q=2-31, 103941 kb/s,   Metadata:
20 fps,     encoder         : 20 tbn, Lavf58.29.10020 tbc
    Stream #0:0: Video: rawvideo (Y800 / 0x30303859), gray, 806x806, q=2-31, 103941 kb/s
, 20 fps, 20 tbn, 20 tbc
    Metadata:
      encoder         : Lavc58.54.100 rawvideo
    Stream #0:0
    Stream #0:0: Video: rawvideo (Y800 / 0x30303859), gray, 806x806, q=2-31, 103941 kb/s: Video: rawvideo (Y800 / 0x30303859), gray, 806x806, q=2-31, 103941 kb/s, ,     Metadata:
20 fps, 20 fps,       encoder         : 20 tbn, 20 tbn, Lavc58.54.100 rawvideo20 tbc20 tbc

    Metadata:
    Metadata:
      encoder         :       encoder         : Lavc58.54.100 rawvideoLavc58.54.100 rawvideo

frame=13204 fps=122 q=-0.0 Lsize= 8376752kB time=00:11:00.20 bitrate=103941.8kbits/s speed= 6.1x    
video:8376752kB audio:0kB subtitle:0kB other streams:0kB global headers:0kB muxing overhead: 0.000000%
frame=13204 fps=122 q=-0.0 Lsize= 8376752kB time=00:11:00.20 bitrate=103941.8kbits/s speed=6.09x    
video:8376752kB audio:0kB subtitle:0kB other streams:0kB global headers:0kB muxing overhead: 0.000000%
frame=13204 fps=122 q=-0.0 Lsize= 8376752kB time=00:11:00.20 bitrate=103941.8kbits/s speed=6.09x    
video:8376752kB audio:0kB subtitle:0kB other streams:0kB global headers:0kB muxing overhead: 0.000000%
frame=13204 fps=122 q=-0.0 Lsize= 8376752kB time=00:11:00.20 bitrate=103941.8kbits/s speed=6.09x    
video:8376752kB audio:0kB subtitle:0kB other streams:0kB global headers:0kB muxing overhead: 0.000000%
frame=13204 fps=122 q=-0.0 Lsize= 8376752kB time=00:11:00.20 bitrate=103941.8kbits/s speed=6.09x    
video:8376752kB audio:0kB subtitle:0kB other streams:0kB global headers:0kB muxing overhead: 0.000000%
frame=13204 fps=122 q=-0.0 Lsize= 8376752kB time=00:11:00.20 bitrate=103941.8kbits/s speed=6.09x    
video:8376752kB audio:0kB subtitle:0kB other streams:0kB global headers:0kB muxing overhead: 0.000000%
frame=13204 fps=122 q=-0.0 Lsize= 8376752kB time=00:11:00.20 bitrate=103941.8kbits/s speed=6.09x    
video:8376752kB audio:0kB subtitle:0kB other streams:0kB global headers:0kB muxing overhead: 0.000000%
[2024-04-18 12:05:41.062809] Denoising...
[2024-04-18 12:05:41.064407] Denoising...
[2024-04-18 12:05:41.066194] Denoising...
[2024-04-18 12:05:41.066354] Denoising...
[2024-04-18 12:05:41.067592] Denoising...
[2024-04-18 12:05:41.076994] Denoising...
[2024-04-18 12:05:41.082689] Denoising...
[2024-04-18 12:05:41.635887] Background removal...
[2024-04-18 12:05:41.639948] Background removal...
[2024-04-18 12:05:41.641641] Background removal...
[2024-04-18 12:05:41.642357] Background removal...
[2024-04-18 12:05:41.644510] Motion correction...
[2024-04-18 12:05:41.646212] Background removal...
[2024-04-18 12:05:41.648473] Motion correction...
[2024-04-18 12:05:41.649984] Background removal...
[2024-04-18 12:05:41.650904] Motion correction...
[2024-04-18 12:05:41.651986] Motion correction...
[2024-04-18 12:05:41.655712] Motion correction...
[2024-04-18 12:05:41.658694] Motion correction...
[2024-04-18 12:05:41.663167] Background removal...
[2024-04-18 12:05:41.671169] Motion correction...
[2024-04-18 12:05:41.751098] Applying transformation...
[2024-04-18 12:05:41.751383] Applying transformation...
[2024-04-18 12:05:41.751802] Applying transformation...
[2024-04-18 12:05:41.751962] Applying transformation...
[2024-04-18 12:05:41.752343] Applying transformation...
[2024-04-18 12:05:41.752393] Applying transformation...
[2024-04-18 12:05:41.752673] Applying transformation...
[2024-04-18 12:05:41.758730] Saving corrected object...
[2024-04-18 12:05:41.758849] Saving corrected object...
[2024-04-18 12:05:41.759283] Saving corrected object...
[2024-04-18 12:05:41.759840] Saving corrected object...
[2024-04-18 12:05:41.759849] Saving corrected object...
[2024-04-18 12:05:41.760173] Saving corrected object...
[2024-04-18 12:05:41.760474] Saving corrected object...
ffmpeg version 4.2.3ffmpeg version 4.2.3ffmpeg version 4.2.3ffmpeg version 4.2.3ffmpeg version 4.2.3 Copyright (c) 2000-2020 the FFmpeg developers Copyright (c) 2000-2020 the FFmpeg developersffmpeg version 4.2.3 Copyright (c) 2000-2020 the FFmpeg developers Copyright (c) 2000-2020 the FFmpeg developers

 Copyright (c) 2000-2020 the FFmpeg developers Copyright (c) 2000-2020 the FFmpeg developers
ffmpeg version 4.2.3  built with gcc 7.5.0 (crosstool-NG 1.24.0.123_1667d2b)
  built with gcc 7.5.0 (crosstool-NG 1.24.0.123_1667d2b)
  built with gcc 7.5.0 (crosstool-NG 1.24.0.123_1667d2b)

  built with gcc 7.5.0 (crosstool-NG 1.24.0.123_1667d2b)
  configuration: --prefix=/home/conda/feedstock_root/build_artifacts/ffmpeg_1590573566052/_h_env_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_plac --cc=/home/conda/feedstock_root/build_artifacts/ffmpeg_1590573566052/_build_env/bin/x86_64-conda_cos6-linux-gnu-cc --disable-doc --disable-openssl --enable-avresample --enable-gnutls --enable-gpl --enable-hardcoded-tables --enable-libfreetype --enable-libopenh264 --enable-libx264 --enable-pic --enable-pthreads --enable-shared --enable-static --enable-version3 --enable-zlib --enable-libmp3lame
  configuration: --prefix=/home/conda/feedstock_root/build_artifacts/ffmpeg_1590573566052/_h_env_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_plac --cc=/home/conda/feedstock_root/build_artifacts/ffmpeg_1590573566052/_build_env/bin/x86_64-conda_cos6-linux-gnu-cc --disable-doc --disable-openssl --enable-avresample --enable-gnutls --enable-gpl --enable-hardcoded-tables --enable-libfreetype --enable-libopenh264 --enable-libx264 --enable-pic --enable-pthreads --enable-shared --enable-static --enable-version3 --enable-zlib --enable-libmp3lame
 Copyright (c) 2000-2020 the FFmpeg developers  configuration: --prefix=/home/conda/feedstock_root/build_artifacts/ffmpeg_1590573566052/_h_env_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_plac --cc=/home/conda/feedstock_root/build_artifacts/ffmpeg_1590573566052/_build_env/bin/x86_64-conda_cos6-linux-gnu-cc --disable-doc --disable-openssl --enable-avresample --enable-gnutls --enable-gpl --enable-hardcoded-tables --enable-libfreetype --enable-libopenh264 --enable-libx264 --enable-pic --enable-pthreads --enable-shared --enable-static --enable-version3 --enable-zlib --enable-libmp3lame
  built with gcc 7.5.0 (crosstool-NG 1.24.0.123_1667d2b)
  configuration: --prefix=/home/conda/feedstock_root/build_artifacts/ffmpeg_1590573566052/_h_env_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_plac --cc=/home/conda/feedstock_root/build_artifacts/ffmpeg_1590573566052/_build_env/bin/x86_64-conda_cos6-linux-gnu-cc --disable-doc --disable-openssl --enable-avresample --enable-gnutls --enable-gpl --enable-hardcoded-tables --enable-libfreetype --enable-libopenh264 --enable-libx264 --enable-pic --enable-pthreads --enable-shared --enable-static --enable-version3 --enable-zlib --enable-libmp3lame

  configuration: --prefix=/home/conda/feedstock_root/build_artifacts/ffmpeg_1590573566052/_h_env_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_plac --cc=/home/conda/feedstock_root/build_artifacts/ffmpeg_1590573566052/_build_env/bin/x86_64-conda_cos6-linux-gnu-cc --disable-doc --disable-openssl --enable-avresample --enable-gnutls --enable-gpl --enable-hardcoded-tables --enable-libfreetype --enable-libopenh264 --enable-libx264 --enable-pic --enable-pthreads --enable-shared --enable-static --enable-version3 --enable-zlib --enable-libmp3lame
  built with gcc 7.5.0 (crosstool-NG 1.24.0.123_1667d2b)
  built with gcc 7.5.0 (crosstool-NG 1.24.0.123_1667d2b)
  configuration: --prefix=/home/conda/feedstock_root/build_artifacts/ffmpeg_1590573566052/_h_env_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_plac --cc=/home/conda/feedstock_root/build_artifacts/ffmpeg_1590573566052/_build_env/bin/x86_64-conda_cos6-linux-gnu-cc --disable-doc --disable-openssl --enable-avresample --enable-gnutls --enable-gpl --enable-hardcoded-tables --enable-libfreetype --enable-libopenh264 --enable-libx264 --enable-pic --enable-pthreads --enable-shared --enable-static --enable-version3 --enable-zlib --enable-libmp3lame
  configuration: --prefix=/home/conda/feedstock_root/build_artifacts/ffmpeg_1590573566052/_h_env_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_plac --cc=/home/conda/feedstock_root/build_artifacts/ffmpeg_1590573566052/_build_env/bin/x86_64-conda_cos6-linux-gnu-cc --disable-doc --disable-openssl --enable-avresample --enable-gnutls --enable-gpl --enable-hardcoded-tables --enable-libfreetype --enable-libopenh264 --enable-libx264 --enable-pic --enable-pthreads --enable-shared --enable-static --enable-version3 --enable-zlib --enable-libmp3lame
  libavutil      56. 31.100 / 56. 31.100
  libavcodec     58. 54.100 / 58. 54.100
  libavformat    58. 29.100 / 58. 29.100
  libavdevice    58.  8.100 / 58.  8.100
  libavfilter     7. 57.100 /  7. 57.100
  libavresample   4.  0.  0 /  4.  0.  0
  libswscale      5.  5.100 /  5.  5.100
  libavutil      56. 31.100 / 56. 31.100
  libswresample   3.  5.100 /  3.  5.100
  libavutil      56. 31.100 / 56. 31.100
  libavcodec     58. 54.100 / 58. 54.100
  libpostproc    55.  5.100 / 55.  5.100
  libavcodec     58. 54.100 / 58. 54.100
  libavformat    58. 29.100 / 58. 29.100
  libavformat    58. 29.100 / 58. 29.100
  libavdevice    58.  8.100 / 58.  8.100
  libavdevice    58.  8.100 / 58.  8.100
  libavfilter     7. 57.100 /  7. 57.100
  libavfilter     7. 57.100 /  7. 57.100
  libavresample   4.  0.  0 /  4.  0.  0
  libavresample   4.  0.  0 /  4.  0.  0
  libavutil      56. 31.100 / 56. 31.100
  libswscale      5.  5.100 /  5.  5.100
  libavutil      56. 31.100 / 56. 31.100
  libavutil      56. 31.100 / 56. 31.100
  libswscale      5.  5.100 /  5.  5.100
  libswresample   3.  5.100 /  3.  5.100
  libavcodec     58. 54.100 / 58. 54.100
  libavcodec     58. 54.100 / 58. 54.100
  libavcodec     58. 54.100 / 58. 54.100
  libswresample   3.  5.100 /  3.  5.100
  libpostproc    55.  5.100 / 55.  5.100
  libavformat    58. 29.100 / 58. 29.100
  libavformat    58. 29.100 / 58. 29.100
  libavformat    58. 29.100 / 58. 29.100
  libpostproc    55.  5.100 / 55.  5.100
  libavdevice    58.  8.100 / 58.  8.100
  libavdevice    58.  8.100 / 58.  8.100
  libavdevice    58.  8.100 / 58.  8.100
  libavfilter     7. 57.100 /  7. 57.100
  libavfilter     7. 57.100 /  7. 57.100
  libavresample   4.  0.  0 /  4.  0.  0
  libswscale      5.  5.100 /  5.  5.100
  libavresample   4.  0.  0 /  4.  0.  0
  libswresample   3.  5.100 /  3.  5.100
  libswscale      5.  5.100 /  5.  5.100
  libpostproc    55.  5.100 / 55.  5.100
  libswresample   3.  5.100 /  3.  5.100
  libpostproc    55.  5.100 / 55.  5.100
  libavfilter     7. 57.100 /  7. 57.100
  libavutil      56. 31.100 / 56. 31.100
  libavcodec     58. 54.100 / 58. 54.100
  libavformat    58. 29.100 / 58. 29.100
  libavresample   4.  0.  0 /  4.  0.  0
  libavdevice    58.  8.100 / 58.  8.100
  libavfilter     7. 57.100 /  7. 57.100
  libswscale      5.  5.100 /  5.  5.100
  libavresample   4.  0.  0 /  4.  0.  0
  libswresample   3.  5.100 /  3.  5.100
  libswscale      5.  5.100 /  5.  5.100
  libpostproc    55.  5.100 / 55.  5.100
  libswresample   3.  5.100 /  3.  5.100
  libpostproc    55.  5.100 / 55.  5.100
Input #0, avi, from '/media/HDD_4TB_1/jordi/calcium_imaging/hdrep_primary_cultures/uncorrected/test_PRIM20231117_GrayScale_AVI.avi':
  Duration: 00:11:00.20, start: 0.000000, bitrate: 104203 kb/s
    Stream #0:0: Video: rawvideo, pal8, 806x806, 104207 kb/s, 20 fps, Input #0, avi, from '/media/HDD_4TB_1/jordi/calcium_imaging/hdrep_primary_cultures/uncorrected/test_PRIM20231117_GrayScale_AVI.avi':
20 tbr,   Duration: 20 tbn, Input #0, avi, from '/media/HDD_4TB_1/jordi/calcium_imaging/hdrep_primary_cultures/uncorrected/test_PRIM20231117_GrayScale_AVI.avi':
Input #0, avi, from '/media/HDD_4TB_1/jordi/calcium_imaging/hdrep_primary_cultures/uncorrected/test_PRIM20231117_GrayScale_AVI.avi':
20 tbc00:11:00.20Input #0, avi, from '/media/HDD_4TB_1/jordi/calcium_imaging/hdrep_primary_cultures/uncorrected/test_PRIM20231117_GrayScale_AVI.avi':
  Duration: Input #0, avi, from '/media/HDD_4TB_1/jordi/calcium_imaging/hdrep_primary_cultures/uncorrected/test_PRIM20231117_GrayScale_AVI.avi':
  Duration: 
  Duration: 00:11:00.20, start: , start:   Duration: 00:11:00.2000:11:00.20, start: 0.0000000.00000000:11:00.20, bitrate: , start: 0.000000, bitrate: , start: 0.000000, bitrate: 104203 kb/s104203 kb/s
0.000000, bitrate: 
, bitrate: 104203 kb/s104203 kb/s104203 kb/s
    Stream #0:0    Stream #0:0
: Video: rawvideo, pal8, 806x806, 104207 kb/s: Video: rawvideo, pal8, 806x806, 104207 kb/s    Stream #0:0, 
20 fps, 20 tbr, 20 tbn, , 20 tbc
20 fps, 20 tbr,     Stream #0:020 tbn, : Video: rawvideo, pal8, 806x806, 104207 kb/s20 tbc, 
: Video: rawvideo, pal8, 806x806, 104207 kb/s20 fps, 20 tbr, 20 tbn, Stream mapping:
20 tbc, 
  Stream #0:0 -> #0:0 (rawvideo (native) -> rawvideo (native))20 fps, 
20 tbr, Press [q] to stop, [?] for help
20 tbn, 20 tbc
    Stream #0:0: Video: rawvideo, pal8, 806x806, 104207 kb/s, 20 fps, 20 tbr, 20 tbn, 20 tbc
Stream mapping:
  Stream #0:0 -> #0:0Stream mapping:
 (rawvideo (native) -> rawvideo (native))
  Stream #0:0 -> #0:0Press [q] to stop, [?] for help
 (rawvideo (native) -> rawvideo (native))Stream mapping:

Press [q] to stop, [?] for help
  Stream #0:0 -> #0:0 (rawvideo (native) -> rawvideo (native))
Press [q] to stop, [?] for help
Stream mapping:
  Stream #0:0 -> #0:0 (rawvideo (native) -> rawvideo (native))
Press [q] to stop, [?] for help
Stream mapping:
  Stream #0:0 -> #0:0 (rawvideo (native) -> rawvideo (native))
Press [q] to stop, [?] for help
Input #0, avi, from '/media/HDD_4TB_1/jordi/calcium_imaging/hdrep_primary_cultures/uncorrected/test_PRIM20231117_GrayScale_AVI.avi':
  Duration: 00:11:00.20, start: 0.000000, bitrate: 104203 kb/s
    Stream #0:0: Video: rawvideo, pal8, 806x806, 104207 kb/s, 20 fps, 20 tbr, 20 tbn, 20 tbc
Stream mapping:
  Stream #0:0 -> #0:0 (rawvideo (native) -> rawvideo (native))
Press [q] to stop, [?] for help
Output #0, rawvideo, to 'pipe:':
  Metadata:
    encoder         : Output #0, rawvideo, to 'pipe:':
  Metadata:
    encoder         : Lavf58.29.100
    Stream #0:0: Video: rawvideo (Y800 / 0x30303859), gray, 806x806, q=2-31, 103941 kb/s, 20 fps, 20 tbn, 20 tbc
    Metadata:
      encoder         : Lavc58.54.100 rawvideo
Lavf58.29.100
    Stream #0:0: Video: rawvideo (Y800 / 0x30303859), gray, 806x806, q=2-31, 103941 kb/s, 20 fps, 20 tbn, 20 tbc
    Metadata:
      encoder         : Lavc58.54.100 rawvideo
Output #0, rawvideo, to 'pipe:':
  Metadata:
    encoder         : Lavf58.29.100
    Stream #0:0: Video: rawvideo (Y800 / 0x30303859), gray, 806x806, q=2-31, 103941 kb/s, 20 fps, 20 tbn, 20 tbc
    Metadata:
      encoder         : Output #0, rawvideo, to 'pipe:':
Lavc58.54.100 rawvideo
  Metadata:
    encoder         : Lavf58.29.100
    Stream #0:0: Video: rawvideo (Y800 / 0x30303859), gray, 806x806, q=2-31, 103941 kb/s, 20 fps, 20 tbn, 20 tbc
    Metadata:
      encoder         : Lavc58.54.100 rawvideo
Output #0, rawvideo, to 'pipe:':
  Metadata:
    encoder         : Lavf58.29.100
    Stream #0:0Output #0, rawvideo, to 'pipe:':
  Metadata:
    encoder         : Lavf58.29.100
    Stream #0:0: Video: rawvideo (Y800 / 0x30303859), gray, 806x806, q=2-31, 103941 kb/s, 20 fps, 20 tbn, 20 tbc
    Metadata:
      encoder         : Lavc58.54.100 rawvideo
Output #0, rawvideo, to 'pipe:':
  Metadata:
    encoder         : Lavf58.29.100
    Stream #0:0: Video: rawvideo (Y800 / 0x30303859), gray, 806x806, q=2-31, 103941 kb/s, 20 fps, 20 tbn, 20 tbc
    Metadata:
      encoder         : Lavc58.54.100 rawvideo
: Video: rawvideo (Y800 / 0x30303859), gray, 806x806, q=2-31, 103941 kb/s, 20 fps, 20 tbn, 20 tbc
    Metadata:
      encoder         : Lavc58.54.100 rawvideo
frame=13204 fps=141 q=-0.0 Lsize= 8376752kB time=00:11:00.20 bitrate=103941.8kbits/s speed=7.07x    
video:8376752kB audio:0kB subtitle:0kB other streams:0kB global headers:0kB muxing overhead: 0.000000%
frame=13204 fps=138 q=-0.0 Lsize= 8376752kB time=00:11:00.20 bitrate=103941.8kbits/s speed=6.92x    
frame=13204 fps=138 q=-0.0 Lsize= 8376752kB time=00:11:00.20 bitrate=103941.8kbits/s speed=6.92x    
frame=13204 fps=138 q=-0.0 Lsize= 8376752kB time=00:11:00.20 bitrate=103941.8kbits/s speed=6.92x    
video:8376752kB audio:0kB subtitle:0kB other streams:0kB global headers:0kB muxing overhead: video:8376752kB audio:0kB subtitle:0kB other streams:0kB global headers:0kB muxing overhead: 0.000000%0.000000%video:8376752kB audio:0kB subtitle:0kB other streams:0kB global headers:0kB muxing overhead: 

0.000000%
frame=13204 fps=138 q=-0.0 Lsize= 8376752kB time=00:11:00.20 bitrate=103941.8kbits/s speed=6.92x    
frame=13204 fps=138 q=-0.0 Lsize= 8376752kB time=00:11:00.20 bitrate=103941.8kbits/s speed=6.92x    
video:8376752kB audio:0kB subtitle:0kB other streams:0kB global headers:0kB muxing overhead: 0.000000%video:8376752kB audio:0kB subtitle:0kB other streams:0kB global headers:0kB muxing overhead: 0.000000%

frame=13204 fps=138 q=-0.0 Lsize= 8376752kB time=00:11:00.20 bitrate=103941.8kbits/s speed=6.92x    
video:8376752kB audio:0kB subtitle:0kB other streams:0kB global headers:0kB muxing overhead: 0.000000%

Also, I was wondering whether there's any point in using multiple workers to process a single video? Is it possible each worker is just doing the same in this case?

austinbaggetta commented 5 months ago

Hi Jordi,

I have also used if name == __main__: to fix the same error when trying to run Minian using a .py file in the past, so I am glad you found that fix.

Is the code that you are running the code from your original post? It's unclear to me why each step (denoising, background removal, etc.) is being output multiple times when you have one line that says print.

One minor detail is that I would recommend closing the cluster after all your computations are completed, so start the cluster at the beginning of your script and close it at the end, all within the if name == __main__ statement. In other words your whole Minian pipeline, if you have it as a .py file, should be within the if name == __main__ statement. You don't have to put the settings and imports within this statement, just from the cluster creation and loading video portion onward. You may already be doing this though!

I've never seen the start_method argument in LocalCluster, and it's unclear from the Dask documentation (https://docs.dask.org/en/stable/deploying-python.html) what this argument is doing, so I recommend removing it as none of us in the lab have used it and have no issues.

The point of using multiple workers for Dask is that each worker will handle a chunk of your data - Minian, using Dask arrays built into xarray, will chunk your xarray.DataArray into (by default) 256MB chunks of data (determined through the get_optimal_chk function, and the variable chk will be used when temporally or spatially chunking your data in future computations). Dask then will create a task graph when the save_minian function is used since the actual computations are only occuring when .compute is called, where it will figure out the optimal way for each worker to process chunks in parallel. More workers means more processes running simultaneously and reduces run time, though only up to a point.

I'm not exactly sure why it's getting stuck from this thread. Reach out to me privately if you want to set up a Zoom call if none of the above recommendations apply to your situation, would love to help you get this working - hopefully we can find an answer that will help the community as well.

jordiabante commented 5 months ago

Hi Austin,

Yes, everything was being output multiple times because I did not include the rest of the code under the if name == __main__ statement. This fixed that!

Re the start_method, I think I tried this as a way to fix the issues generated by the previous problem. I think I was starting multiple clusters and that was just messing things further. After putting the code under the if name == __main__ statement, I was able to remove it.

I also found that ffmpeg freezing could be related to not being able to write to its output pipe any longer and that reducing the amount of stream in prints could ameliorate this issue (1,2). This was a long shot and I don't think it's making any difference. But, this explains the extra arguments in write_video.

I have tried with and without the use of a Dask cluster and the code still gets stuck at the end.

  1. No dask: script hangs after canvassing all the frames within the write_video function (400 fps) and RAM memory spikes up progressively eventually maxing out. Here, I do not include the save_minian command since there is really no advantage in this case as far as I understand.
  2. Dask: with the current set up, the memory stays under control. However, I get the following error message when trying to save the video: Unable to find a suitable output format for ''.

I will go ahead and message you privately. Thanks, Austin!

No dask

# load dependancies
import os
import sys
import numpy as np

##############################################################################
# Our functions
##############################################################################

from datetime import datetime
def _print(*args, **kw):
    print("[%s]" % (datetime.now()),*args, **kw)

##############################################################################
# Configuration
##############################################################################

_print("Configuration...")

# Set up Initial Basic Parameters#
minian_path = "/media/HDD_2TB/jordi/repos/minian/minian"
dpath = "/media/HDD_4TB_1/jordi/calcium_imaging/hdrep_primary_cultures/uncorrected/"
minian_ds_path = "/media/HDD_4TB_1/jordi/calcium_imaging/hdrep_primary_cultures/minian/"
intpath = "/media/HDD_4TB_1/jordi/calcium_imaging/hdrep_primary_cultures/intermediate/"
subset = dict(frame=slice(0, None))
subset_mc = None
interactive = False
output_size = 100
param_save_minian = {
    "dpath": minian_ds_path,
    "meta_dict": dict(session=-1, animal=-2),
    "overwrite": True,
}

# Pre-processing Parameters#
param_load_videos = {
    "pattern": r"test_PRIM20231117_GrayScale_AVI.avi",
    "dtype": np.uint8,
    "downsample": dict(frame=1, height=1, width=1),
    "downsample_strategy": "subset",
}
param_denoise = {"method": "median", "ksize": 7}
param_background_removal = {"method": "tophat", "wnd": 15}

# Motion Correction Parameters#
subset_mc = None
param_estimate_motion = {"dim": "frame"}

# Initialization Parameters#
param_seeds_init = {
    "wnd_size": 1000,
    "method": "rolling",
    "stp_size": 500,
    "max_wnd": 15,
    "diff_thres": 3,
}
param_pnr_refine = {"noise_freq": 0.06, "thres": 1}
param_ks_refine = {"sig": 0.05}
param_seeds_merge = {"thres_dist": 10, "thres_corr": 0.8, "noise_freq": 0.06}
param_initialize = {"thres_corr": 0.8, "wnd": 10, "noise_freq": 0.06}
param_init_merge = {"thres_corr": 0.8}

os.environ["OMP_NUM_THREADS"] = "1"
os.environ["MKL_NUM_THREADS"] = "1"
os.environ["OPENBLAS_NUM_THREADS"] = "1"
os.environ["MINIAN_INTERMEDIATE"] = intpath

##############################################################################
# Import minian
##############################################################################

_print("Importing minian...")

sys.path.append(minian_path)
from minian.cnmf import (
    compute_AtC,
    compute_trace,
    get_noise_fft,
    smooth_sig,
    unit_merge,
    update_spatial,
    update_temporal,
    update_background,
)
from minian.initialization import (
    gmm_refine,
    initA,
    initC,
    intensity_refine,
    ks_refine,
    pnr_refine,
    seeds_init,
    seeds_merge,
)
from minian.motion_correction import apply_transform, estimate_motion
from minian.preprocessing import denoise, remove_background
from minian.utilities import (
    TaskAnnotation,
    get_optimal_chk,
    load_videos,
    open_minian,
    save_minian,
)
from minian.visualization import (
    CNMFViewer,
    VArrayViewer,
    generate_videos,
    visualize_gmm_fit,
    visualize_motion,
    visualize_preprocess,
    visualize_seeds,
    visualize_spatial_update,
    visualize_temporal_update,
    write_video,
)

##############################################################################
# module initialization
##############################################################################

_print("Initialiazing...")

dpath = os.path.abspath(dpath)

##############################################################################
# load videos and initialize
##############################################################################

_print("Loading videos...")

# load and get optimal chunks
varr = load_videos(dpath, **param_load_videos)
chk, _ = get_optimal_chk(varr, dtype=float)

# chunks
varr = varr.chunk({"frame": 200, "height": -1, "width": -1}).rename("varr")

# subset video (default:none)
varr_ref = varr.sel(subset)

##############################################################################
# corrections
##############################################################################

# glow removal
_print("Glow removal...")
varr_min = varr_ref.min("frame").compute()
varr_ref = varr_ref - varr_min

# denoise
_print("Denoising...")
varr_ref = denoise(varr_ref, **param_denoise)

# background removal
_print("Background removal...")
varr_ref = remove_background(varr_ref, **param_background_removal)

# motion correction
_print("Motion correction...")
motion = estimate_motion(varr_ref.sel(subset_mc), **param_estimate_motion)

# apply transform
_print("Applying transformation...")
Y = apply_transform(varr_ref, motion, fill=0)

# store corrected video
_print("Saving corrected video...")
_ = write_video(Y.astype(float).rename("Y_fm_chk"), "minian_mc.avi", minian_ds_path, norm=True, 
                options={"loglevel":"error","hide_banner":"","nostats":""})

Dask

Code

# load dependancies
import numpy as np
import os
import sys
from dask.distributed import Client, LocalCluster

##############################################################################
# Our functions
##############################################################################

from datetime import datetime
def _print(*args, **kw):
    print("[%s]" % (datetime.now()),*args, **kw)

##############################################################################
# Configuration
##############################################################################

# Set up Initial Basic Parameters#
minian_path = "/media/HDD_2TB/jordi/repos/minian/minian"
dpath = "/media/HDD_4TB_1/jordi/calcium_imaging/hdrep_primary_cultures/uncorrected/"
minian_ds_path = "/media/HDD_4TB_1/jordi/calcium_imaging/hdrep_primary_cultures/minian/"
intpath = "/media/HDD_4TB_1/jordi/calcium_imaging/hdrep_primary_cultures/intermediate/"
subset = dict(frame=slice(0, None))
subset_mc = None
interactive = False
output_size = 100
param_save_minian = {
    "dpath": minian_ds_path,
    "meta_dict": dict(session=-1, animal=-2),
    "overwrite": True,
}

# Pre-processing Parameters#
param_load_videos = {
    "pattern": r"test_PRIM20231117_GrayScale_AVI.avi",# r"test_hdrep.avi",
    "dtype": np.uint8,
    "downsample": dict(frame=1, height=1, width=1),
    "downsample_strategy": "subset",
}
param_denoise = {"method": "median", "ksize": 7}
param_background_removal = {"method": "tophat", "wnd": 15}

# Motion Correction Parameters#
subset_mc = None
param_estimate_motion = {"dim": "frame"}

# Initialization Parameters#
param_seeds_init = {
    "wnd_size": 1000,
    "method": "rolling",
    "stp_size": 500,
    "max_wnd": 15,
    "diff_thres": 3,
}
param_pnr_refine = {"noise_freq": 0.06, "thres": 1}
param_ks_refine = {"sig": 0.05}
param_seeds_merge = {"thres_dist": 10, "thres_corr": 0.8, "noise_freq": 0.06}
param_initialize = {"thres_corr": 0.8, "wnd": 10, "noise_freq": 0.06}
param_init_merge = {"thres_corr": 0.8}

os.environ["OMP_NUM_THREADS"] = "1"
os.environ["MKL_NUM_THREADS"] = "1"
os.environ["OPENBLAS_NUM_THREADS"] = "1"
os.environ["MINIAN_INTERMEDIATE"] = intpath

##############################################################################
# Import minian
##############################################################################

sys.path.append(minian_path)
from minian.cnmf import (
    compute_AtC,
    compute_trace,
    get_noise_fft,
    smooth_sig,
    unit_merge,
    update_spatial,
    update_temporal,
    update_background,
)
from minian.initialization import (
    gmm_refine,
    initA,
    initC,
    intensity_refine,
    ks_refine,
    pnr_refine,
    seeds_init,
    seeds_merge,
)
from minian.motion_correction import apply_transform, estimate_motion
from minian.preprocessing import denoise, remove_background
from minian.utilities import (
    TaskAnnotation,
    get_optimal_chk,
    load_videos,
    open_minian,
    save_minian,
)
from minian.visualization import (
    CNMFViewer,
    VArrayViewer,
    generate_videos,
    visualize_gmm_fit,
    visualize_motion,
    visualize_preprocess,
    visualize_seeds,
    visualize_spatial_update,
    visualize_temporal_update,
    write_video,
)

##############################################################################
# module initialization
##############################################################################

if __name__ == '__main__':

    _print("Initialiazing...")

    dpath = os.path.abspath(dpath)

    _print("Starting cluster...")

    ## Potential cluster configuration
    n_workers = int(os.getenv("MINIAN_NWORKERS", 5))

    cluster = LocalCluster(
         n_workers=n_workers,
         memory_limit="10GB",
         resources={"MEM": 1},
         threads_per_worker=2,
         dashboard_address=":8585"
        # start_method='fork'  # or 'spawn'
    )

    annt_plugin = TaskAnnotation()
    cluster.scheduler.add_plugin(annt_plugin)

    try:

        client = Client(cluster)

        ##############################################################################
        # load videos and initialize
        ##############################################################################

        _print("Loading videos...")

        # load and get optimal chunks
        varr = load_videos(dpath, **param_load_videos)
        chk, _ = get_optimal_chk(varr, dtype=float)

        # chunks
        # varr = varr.chunk({"frame": chk["frame"], "height": -1, "width": -1}).rename("varr")
        # varr = varr.chunk({"frame": 200, "height": -1, "width": -1}).rename("varr")
        varr = save_minian(
            varr.chunk({"frame": chk["frame"], "height": -1, "width": -1}).rename("varr"),
            intpath,
            overwrite=True,
            mem_limit='10GB'
        )
        # varr = open_minian(intpath)

        # subset video (default:none)
        varr_ref = varr.sel(subset)

        ##############################################################################
        # corrections
        ##############################################################################

        _print("Glow removal...")

        # glow removal
        varr_min = varr_ref.min("frame").compute()
        varr_ref = varr_ref - varr_min

        _print("Denoising...")

        # denoise
        varr_ref = denoise(varr_ref, **param_denoise)

        _print("Background removal...")

        # background removal
        varr_ref = remove_background(varr_ref, **param_background_removal)

        _print("Motion correction...")

        # motion correction
        motion = estimate_motion(varr_ref.sel(subset_mc), **param_estimate_motion)

        # _print("Saving estimated motion...")

        # save
        # motion = save_minian(
        #     motion.rename("motion").chunk({"frame": chk["frame"]}), **param_save_minian
        # )

        _print("Applying transformation...")

        # apply transform
        Y = apply_transform(varr_ref, motion, fill=0)

        # _print("Saving corrected object...")

        # save transformed object
        # Y_fm_chk = save_minian(Y.astype(float).rename("Y_fm_chk"), intpath, overwrite=True)
        # Y_hw_chk = save_minian(Y_fm_chk.rename("Y_hw_chk"), intpath,overwrite=True,
        #     chunks={"frame": -1, "height": chk["height"], "width": chk["width"]},
        # )

        _print("Saving corrected video...")

        # store corrected video
        print(Y)
        _ = write_video(Y.astype(float).rename("Y_fm_chk"), "minian_mc.avi", minian_ds_path, norm=True, 
                        options={"loglevel":"error","hide_banner":"","nostats":""})

    finally:

        ##############################################################################
        # close cluster
        ##############################################################################

        _print("Closing cluster client...")
        # Close the client and cluster explicitly
        client.close()
        cluster.close()

Log

[2024-04-21 13:03:11.411790] Initialiazing...
[2024-04-21 13:03:11.411819] Starting cluster...
[2024-04-21 13:03:15.313842] Loading videos...
loading 1 videos in folder /media/HDD_4TB_1/jordi/calcium_imaging/hdrep_primary_cultures/uncorrected
ffmpeg version 4.2.3 Copyright (c) 2000-2020 the FFmpeg developers
  built with gcc 7.5.0 (crosstool-NG 1.24.0.123_1667d2b)
  configuration: --prefix=/home/conda/feedstock_root/build_artifacts/ffmpeg_1590573566052/_h_env_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_plac --cc=/home/conda/feedstock_root/build_artifacts/ffmpeg_1590573566052/_build_env/bin/x86_64-conda_cos6-linux-gnu-cc --disable-doc --disable-openssl --enable-avresample --enable-gnutls --enable-gpl --enable-hardcoded-tables --enable-libfreetype --enable-libopenh264 --enable-libx264 --enable-pic --enable-pthreads --enable-shared --enable-static --enable-version3 --enable-zlib --enable-libmp3lame
  libavutil      56. 31.100 / 56. 31.100
  libavcodec     58. 54.100 / 58. 54.100
  libavformat    58. 29.100 / 58. 29.100
  libavdevice    58.  8.100 / 58.  8.100
  libavfilter     7. 57.100 /  7. 57.100
  libavresample   4.  0.  0 /  4.  0.  0
  libswscale      5.  5.100 /  5.  5.100
  libswresample   3.  5.100 /  3.  5.100
  libpostproc    55.  5.100 / 55.  5.100
Input #0, avi, from '/media/HDD_4TB_1/jordi/calcium_imaging/hdrep_primary_cultures/uncorrected/test_PRIM20231117_GrayScale_AVI.avi':
  Duration: 00:11:00.20, start: 0.000000, bitrate: 104203 kb/s
    Stream #0:0: Video: rawvideo, pal8, 806x806, 104207 kb/s, 20 fps, 20 tbr, 20 tbn, 20 tbc
Stream mapping:
  Stream #0:0 -> #0:0 (rawvideo (native) -> rawvideo (native))
Press [q] to stop, [?] for help
Output #0, rawvideo, to 'pipe:':
  Metadata:
    encoder         : Lavf58.29.100
    Stream #0:0: Video: rawvideo (Y800 / 0x30303859), gray, 806x806, q=2-31, 103941 kb/s, 20 fps, 20 tbn, 20 tbc
    Metadata:
      encoder         : Lavc58.54.100 rawvideo
frame=13204 fps=387 q=-0.0 Lsize= 8376752kB time=00:11:00.20 bitrate=103941.8kbits/s speed=19.3x    
video:8376752kB audio:0kB subtitle:0kB other streams:0kB global headers:0kB muxing overhead: 0.000000%
[2024-04-21 13:04:06.507305] Glow removal...
[2024-04-21 13:04:09.475022] Denoising...
[2024-04-21 13:04:09.479128] Background removal...
[2024-04-21 13:04:09.481288] Motion correction...
[2024-04-21 13:04:11.306973] Applying transformation...
[2024-04-21 13:04:11.312548] Saving corrected video...
[NULL @ 0x55db65d909c0] Unable to find a suitable output format for ''
: Invalid argument
[2024-04-21 13:43:42.374401] Closing cluster client...
Traceback (most recent call last):
  File "hdrep_minian_pipeline_dask.py", line 254, in <module>
    _ = write_video(Y, "minian_mc.avi", minian_ds_path, norm=True, options={"loglevel":"error","hide_banner":"","nostats":""})
  File "/media/HDD_4TB_1/jordi/miniconda3/envs/minian/lib/python3.8/site-packages/minian/visualization.py", line 1276, in write_video
    process.stdin.write(np.array(blk).tobytes())
BrokenPipeError: [Errno 32] Broken pipe
austinbaggetta commented 5 months ago

Issue was fixed - the issue was due to commenting out the intermediate save steps, which you can see in the code shown above!