Open jlashner opened 2 years ago
After following the call stack, the simple fix appears to be increasing the timeout from 30 seconds. The call is fairly low-level, so changing it might negatively impact other parts of pysmurf. Also, 30 seconds is already quite long, which might indicate some underlying issue. Looking further into it.
Curious to see if this fixes it. Have you been able to recreate this problem on a SLAC system?
Currently having this issue in the most recent version of pysmurf.
ExceptionTraceback (most recent call last)
<ipython-input-21-155fd56fe9a0> in <module>
1 uxm_setup.uxm_setup(
----> 2 S, cfg, bands=bands, skip_estimate_attens=True, skip_setup_amps=True,
3 )
4
/usr/local/src/pysmurf/python/pysmurf/client/util/pub.py in wrapper(S, pub_action, *args, **kwargs)
48 S.pub._action_ts = S.get_timestamp()
49
---> 50 rv = func(S, *args, **kwargs)
51
52 finally:
/sodetlib/sodetlib/operations/uxm_setup.py in uxm_setup(S, cfg, bands, show_plots, update_cfg, skip_estimate_attens, skip_phase_delay, skip_setup_amps)
475 sdl.set_session_data(S, 'timestamps', summary['timestamps'])
476 success, summary['setup_phase_delay'] = setup_phase_delay(
--> 477 S, cfg, bands, update_cfg=update_cfg)
478 if not success:
479 S.log("UXM Setup failed on setup phase delay step")
/usr/local/src/pysmurf/python/pysmurf/client/util/pub.py in wrapper(S, pub_action, *args, **kwargs)
48 S.pub._action_ts = S.get_timestamp()
49
---> 50 rv = func(S, *args, **kwargs)
51
52 finally:
/sodetlib/sodetlib/operations/uxm_setup.py in setup_phase_delay(S, cfg, bands, update_cfg, modify_attens)
217 #S.set_att_dc(b, 30)
218 summary['bands'].append(int(b))
--> 219 band_delay_us, _ = S.estimate_phase_delay(b, make_plot=True, show_plot=False)
220 band_delay_us = float(band_delay_us)
221 summary['band_delay_us'].append(band_delay_us)
/usr/local/src/pysmurf/python/pysmurf/client/util/pub.py in wrapper(S, pub_action, *args, **kwargs)
48 S.pub._action_ts = S.get_timestamp()
49
---> 50 rv = func(S, *args, **kwargs)
51
52 finally:
/usr/local/src/pysmurf/python/pysmurf/client/util/smurf_util.py in estimate_phase_delay(self, band, nsamp, make_plot, show_plot, save_plot, save_data, n_scan, timestamp, uc_att, dc_att, freq_min, freq_max)
290 freq_cable, resp_cable = self.full_band_resp(
291 band, nsamp=nsamp, make_plot=make_plot,
--> 292 save_data=save_data, n_scan=n_scan)
293
294 idx_cable = np.where( (freq_cable > freq_min) & (freq_cable < freq_max) )
/usr/local/src/pysmurf/python/pysmurf/client/util/pub.py in wrapper(S, pub_action, *args, **kwargs)
48 S.pub._action_ts = S.get_timestamp()
49
---> 50 rv = func(S, *args, **kwargs)
51
52 finally:
/usr/local/src/pysmurf/python/pysmurf/client/tune/smurf_tune.py in full_band_resp(self, band, n_scan, nsamp, make_plot, save_plot, show_plot, save_data, timestamp, save_raw_data, correct_att, swap, hw_trigger, write_log, return_plot_path, check_if_adc_is_saturated)
643 #If ADC is saturated, throws an exception.
644 if check_if_adc_is_saturated:
--> 645 adc_is_saturated = self.check_adc_saturation(band)
646 if adc_is_saturated:
647 raise ValueError('Playing the noise file saturates the '+
/usr/local/src/pysmurf/python/pysmurf/client/util/pub.py in wrapper(S, pub_action, *args, **kwargs)
48 S.pub._action_ts = S.get_timestamp()
49
---> 50 rv = func(S, *args, **kwargs)
51
52 finally:
/usr/local/src/pysmurf/python/pysmurf/client/util/smurf_util.py in check_adc_saturation(self, band)
1487 """
1488 adc = self.read_adc_data(band, data_length=2**12, make_plot=False,
-> 1489 save_data=False, show_plot=False, save_plot=False)
1490 adc_max = int(np.max((adc.real.max(), adc.imag.max())))
1491 adc_min = int(np.min((adc.real.min(), adc.imag.min())))
/usr/local/src/pysmurf/python/pysmurf/client/util/pub.py in wrapper(S, pub_action, *args, **kwargs)
48 S.pub._action_ts = S.get_timestamp()
49
---> 50 rv = func(S, *args, **kwargs)
51
52 finally:
/usr/local/src/pysmurf/python/pysmurf/client/util/smurf_util.py in read_adc_data(self, band, data_length, hw_trigger, make_plot, save_data, timestamp, show_plot, save_plot, plot_ylimits)
1573
1574 res = self.read_stream_data_daq(data_length, bay=bay,
-> 1575 hw_trigger=hw_trigger)
1576 dat = res[1] + 1.j * res[0]
1577
/usr/local/src/pysmurf/python/pysmurf/client/util/pub.py in wrapper(S, pub_action, *args, **kwargs)
48 S.pub._action_ts = S.get_timestamp()
49
---> 50 rv = func(S, *args, **kwargs)
51
52 finally:
/usr/local/src/pysmurf/python/pysmurf/client/util/smurf_util.py in read_stream_data_daq(self, data_length, bay, hw_trigger, write_log)
1462
1463 time.sleep(.1)
-> 1464 sg.wait()
1465
1466 vals = sg.get_values()
/usr/local/src/pysmurf/python/pysmurf/client/command/sync_group.py in wait(self, epics_poll)
67 time.sleep(.001)
68 if time.time() - t0 > self.timeout:
---> 69 raise Exception('Timeout waiting for PVs to update.')
Exception: Timeout waiting for PVs to update.
@jlashner trying to fix this. Running
counter=0
while True:
for band in range(8):
print(f'Band{band} counter={counter}')
S.estimate_phase_delay(band,show_plot=False)
counter+=1
on a production carrier with two production LB AMCs and so far this has run 20+ times on all 8 bands without crashing. When you run estimate_phase_delay
, do you give it any other non-default arguments? And when you're running it, what are the conditions? E.g. do you run it on multiple slots at the same time? I am running this loop on only one carrier, right after running setup
.
Describe the bug
The estimate_phase_delay function currently fails about a quarter of the time because it takes too long for PV's to update. There doesn't seem to be anything wrong with the backend, so we are usually able to re-run the function and get it to work the second or third attempt. However this makes it difficult to write functions that automate the setup procedure, since when running estimate phase delay for all 8 bands there is a very good chance that one of the bands will fail.
Here is the full traceback for the error:
To Reproduce
Try running estimate phase delay for 8 bands in a row.