Using Pymovements with MIT1003

[nisar2]

Question

Hi. I'm trying to analyze the MIT1003 dataset and perform some processing on it. I get an error when detecting microsaccades: threshold does not provide enough variance as required by min_threshold ([0. 0.] < 1e-10). The code runs for several files but eventually breaks. I think I'm probably setting up things incorrectly.

Sample Code

experiment = pm.gaze.Experiment( # parameters from some code provided by dataset creators

    screen_width_px=1024,
    screen_height_px=768,
    screen_width_cm=40.5,
    screen_height_cm=30,
    distance_cm=75,
    origin='upper left',
    sampling_rate=240,
)

filename_format = r'{filename}.csv'

filename_format_dtypes = {
    'filename': str,
}

custom_read_kwargs = {
    'separator': ',',
}

time_unit = 'step'
pixel_columns = ['x', 'y']

dataset_definition = pm.DatasetDefinition(
    name='my_dataset',
    experiment=experiment,
    filename_format=filename_format,
    filename_format_dtypes=filename_format_dtypes,
    custom_read_kwargs=custom_read_kwargs,
    time_column=time_column,
    time_unit=time_unit,
    pixel_columns=pixel_columns,
)

dataset = pm.Dataset(
    definition=dataset_definition,
    path='/shared/project-storage/users/nisar2/projects/pymovement/data/MIT1003',
)

dataset.load()

dataset.pix2deg()
dataset.pos2vel(method='savitzky_golay', degree=2, window_length=7)
dataset.apply('pos2acc', degree=2, window_length=7)
dataset.detect_events('ivt')
dataset.detect_events('microsaccades') # breaks here after processing several of the files (though not sure if processing correctly)

Please include relevant code snippets or files that provide context for your question.

One of the data files is attached: ajs_i05june05_static_street_boston_p1010764.csv

[SiQube]

could you show the error you're getting?

[nisar2]

Here's the error:

threshold does not provide enough variance as required by min_threshold ([0. 0.] < 1e-10)

[SiQube]

can you show the entire output?

[nisar2]

7525it [00:05, 1310.10it/s]

{
    "name": "ValueError",
    "message": "threshold does not provide enough variance as required by min_threshold ([0. 0.] < 1e-10)",
    "stack": "---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
Cell In[70], line 9
      1 # dataset.pix2deg()
      2 # print('here1')
      3 # dataset.pos2vel(method='savitzky_golay', degree=2, window_length=7)
   (...)
      6 # print('here3')
      7 # dataset.detect_events('ivt', name='fixation.ivt')
      8 print('here4')
----> 9 dataset.detect_events('microsaccades', minimum_duration=12)
     10 print('here5')
     11 # dataset.compute_event_properties(\"dispersion\")
     12 # print('here6')
     13 # dataset.compute_event_properties(['amplitude', 'peak_velocity'])
     14 # print('here7')
     15 # pm.plotting.main_sequence_plot(dataset.events[0])
     16 # print('here8')

File ~/anaconda3/envs/diffts/lib/python3.9/site-packages/pymovements/dataset/dataset.py:537, in Dataset.detect_events(self, method, eye, clear, verbose, **kwargs)
    500 def detect_events(
    501         self,
    502         method: Callable[..., EventDataFrame] | str,
   (...)
    507         **kwargs: Any,
    508 ) -> Dataset:
    509     \"\"\"Detect events by applying a specific event detection method.
    510 
    511     Parameters
   (...)
    535         If gaze files have not been loaded yet or gaze files do not contain the right columns.
    536     \"\"\"
--> 537     return self.detect(
    538         method=method,
    539         eye=eye,
    540         clear=clear,
    541         verbose=verbose,
    542         **kwargs,
    543     )

File ~/anaconda3/envs/diffts/lib/python3.9/site-packages/pymovements/dataset/dataset.py:593, in Dataset.detect(self, method, eye, clear, verbose, **kwargs)
    589 disable_progressbar = not verbose
    590 for file_id, (gaze, fileinfo_row) in tqdm(
    591         enumerate(zip(self.gaze, self.fileinfo.to_dicts())), disable=disable_progressbar,
    592 ):
--> 593     gaze.detect(method, eye=eye, clear=clear, **kwargs)
    594     # workaround until events are fully part of the GazeDataFrame
    595     gaze.events.frame = dataset_files.add_fileinfo(
    596         definition=self.definition,
    597         df=gaze.events.frame,
    598         fileinfo=fileinfo_row,
    599     )

File ~/anaconda3/envs/diffts/lib/python3.9/site-packages/pymovements/gaze/gaze_dataframe.py:668, in GazeDataFrame.detect(self, method, eye, clear, **kwargs)
    659 if self.trial_columns is None:
    660     method_kwargs = self._fill_event_detection_kwargs(
    661         method,
    662         gaze=self.frame,
   (...)
    665         **kwargs,
    666     )
--> 668     new_events = method(**method_kwargs)
    670     self.events.frame = pl.concat(
    671         [self.events.frame, new_events.frame],
    672         how='diagonal',
    673     )
    674 else:

File ~/anaconda3/envs/diffts/lib/python3.9/site-packages/pymovements/events/detection/_microsaccades.py:108, in microsaccades(velocities, timesteps, minimum_duration, threshold, threshold_factor, minimum_threshold, include_nan, name)
    105     threshold = np.array(threshold)
    107 if (threshold < minimum_threshold).any():
--> 108     raise ValueError(
    109         'threshold does not provide enough variance as required by min_threshold'
    110         f' ({threshold} < {minimum_threshold})',
    111     )
    113 # Radius of elliptic threshold.
    114 radius = threshold * threshold_factor

ValueError: threshold does not provide enough variance as required by min_threshold ([0. 0.] < 1e-10)"
}

[SiQube]

it seems that in your dataframe there are only 0. velocities, maybe you can loop over the files while extracting the events and check whether the dataframe is empty or just 0 velocities?