[BUG] Quantum Machines Qprogram fails to execute qprogram and crashes

Expected behavior

Loop of Qprograms should run without crashing

Actual behavior

After running a (software) loop of Qprograms for two-tone spectoscopy it throws an error after 70 % and crashes. The data up until this point is saved.

Additional information

The same error has occurred before when running a single Qprogram (without any software loops) like "resonator spectroscopy".

Source code

LabScripts/Personal_folders/Oscar/QM_Qprogram/Qprogram_QM_spectroscopy_6_1.ipynb

2Tone LO:
RESONATOR = "E"

# Buses
drive_bus = "rs_14"
readout_bus = "readout_q0_rf_bus"

# Qubit drive
DRIVE_LO_START = 1e9
DRIVE_LO_STOP = 6.35e9
DRIVE_LO_STEP = 1e6

DRIVE_LO_POWER = 0 #-12

# Resonator 
RESONATOR_FREQ = 6.482232268e9
RESON_LO = resonator_frequencies_dict[RESONATOR] - 150e6
RESON_IF = RESONATOR_FREQ - RESON_LO

RESON_DUR = 2000
RESON_AMP = 0.10

AVERAGES = 100000

# Bias
xtalk_E.phi_x(0.40)
xtalk_E.phi_z(-0.4909)

platform.set_parameter(alias=readout_bus, parameter=ql.Parameter.LO_FREQUENCY, value=RESON_LO)
platform.set_parameter(alias=readout_bus, parameter=ql.Parameter.IF, value=RESON_IF)
platform.set_parameter(alias=drive_bus, parameter=ql.Parameter.POWER, value=DRIVE_LO_POWER)

frequency = np.arange(DRIVE_LO_START, DRIVE_LO_STOP + DRIVE_LO_STEP / 2, DRIVE_LO_STEP)
qprogram = single_point(AVERAGES, RESON_DUR, RESON_AMP, )

# qprogram = single_point(AVERAGES, RESON_DUR, RESON_AMP, )
# results = platform.execute_qprogram(qprogram, bus_mapping={"readout": readout_bus, "drive": drive_bus}).results
# dB(results["readout_q0_rf_bus"][0].array[0] + results["readout_q0_rf_bus"][0].array[1]*1j)

stream_array = ql.stream_results(
    shape=(len(frequency), 2),
    loops={"lo_frequency": frequency},
    path=create_directories("/home/jupytershared/data/") + "data.h5",
)

with stream_array:
    for ii, v in enumerate(tqdm(frequency)):
        platform.set_parameter(alias=drive_bus, parameter=ql.Parameter.LO_FREQUENCY, value=v)
        results = platform.execute_qprogram(qprogram, bus_mapping={"readout": readout_bus, "drive": drive_bus}).results
        stream_array[(ii,)] = results[readout_bus][0].array.T

platform.set_parameter(alias=drive_bus, parameter=ql.Parameter.POWER, value=-80) # set power low

set_all_to_zero()

Tracebacks

2024-04-09 12:58:54,000 - qm - INFO     - Compiling program
...
Unclosed connection: Channel('192.168.1.67', 10251, ..., path=None)
Unclosed connection: Channel('192.168.1.67', 10251, ..., path=None)
........etc
Unclosed connection: Channel('192.168.1.67', 10251, ..., path=None)
Unclosed connection: Channel('192.168.1.67', 10251, ..., path=None)
2024-04-09 13:36:37,345 - qm - ERROR    - Internal error
2024-04-09 13:36:37,346 - qm - ERROR    - Job 1705079177372 failed. Failed to execute program.

{
    "name": "FailedToAddJobToQueueException",
    "message": "Job 1705079177372 failed. Failed to execute program.",
    "stack": "---------------------------------------------------------------------------
FailedToAddJobToQueueException            Traceback (most recent call last)
Cell In[15], line 10
      8     for ii, v in enumerate(tqdm(frequency)):
      9         platform.set_parameter(alias=drive_bus, parameter=ql.Parameter.LO_FREQUENCY, value=v)
---> 10         results = platform.execute_qprogram(qprogram, bus_mapping={\"readout\": readout_bus, \"drive\": drive_bus}).results
     11         stream_array[(ii,)] = results[readout_bus][0].array.T
     13 platform.set_parameter(alias=drive_bus, parameter=ql.Parameter.POWER, value=-80) # set power low

File ~/miniconda3/envs/qililab/lib/python3.10/site-packages/qililab/platform/platform.py:628, in Platform.execute_qprogram(self, qprogram, bus_mapping, debug)
    624         raise NotImplementedError(
    625             \"Executing QProgram in more than one Quantum Machines Cluster is not supported.\"
    626         )
    627     cluster: QuantumMachinesCluster = instruments.pop()  # type: ignore[assignment]
--> 628     return self._execute_qprogram_with_quantum_machines(
    629         cluster=cluster, qprogram=qprogram, bus_mapping=bus_mapping, debug=debug
    630     )
    631 raise NotImplementedError(\"Executing QProgram in a mixture of instruments is not supported.\")

File ~/miniconda3/envs/qililab/lib/python3.10/site-packages/qililab/platform/platform.py:697, in Platform._execute_qprogram_with_quantum_machines(self, cluster, qprogram, bus_mapping, debug)
    694     self._qua_program_cache[qua_program_hash] = cluster.compile(program=qua_program)
    695 compiled_program_id = self._qua_program_cache[qua_program_hash]
--> 697 job = cluster.run_compiled_program(compiled_program_id=compiled_program_id)
    699 acquisitions = cluster.get_acquisitions(job=job)
    701 results = QProgramResults()

File ~/miniconda3/envs/qililab/lib/python3.10/site-packages/qililab/instruments/quantum_machines/quantum_machines_cluster.py:392, in QuantumMachinesCluster.run_compiled_program(self, compiled_program_id)
    390 def run_compiled_program(self, compiled_program_id: str) -> RunningQmJob:
    391     \"\"\"Runs a compiled QUA Program.\"\"\"
--> 392     pending_job = self._qm.queue.add_compiled(compiled_program_id)
    393     return pending_job.wait_for_execution()

File ~/miniconda3/envs/qililab/lib/python3.10/site-packages/qm/jobs/job_queue.py:107, in QmQueue.add_compiled(self, program_id, overrides)
     81 \"\"\"Adds a compiled QUA program to the end of the queue, optionally
     82 overriding the values of analog waveforms defined in the program.
     83 Programs in the queue will play as soon as possible.
   (...)
    103     
    104 \"\"\"
    105 execution_overrides = overrides or {}
--> 107 job_id = self._frontend.add_compiled_to_queue(
    108     machine_id=self.machine_id,
    109     program_id=program_id,
    110     execution_overrides=ExecutionOverridesSchema().load(execution_overrides),
    111 )
    112 return QmPendingJob(
    113     job_id=job_id,
    114     machine_id=self.machine_id,
   (...)
    117     store=self._store,
    118 )

File ~/miniconda3/envs/qililab/lib/python3.10/site-packages/qm/api/base_api.py:28, in connection_error_handle_decorator.<locals>.wrapped(*args, **kwargs)
     25 @functools.wraps(func)
     26 def wrapped(*args: P.args, **kwargs: P.kwargs) -> Ret:
     27     try:
---> 28         return func(*args, **kwargs)
     29     except grpclib.exceptions.GRPCError as e:
     30         if is_debug():

File ~/miniconda3/envs/qililab/lib/python3.10/site-packages/qm/api/frontend_api.py:241, in FrontendApi.add_compiled_to_queue(self, machine_id, program_id, execution_overrides)
    239 if not response.ok:
    240     logger.error(f\"Job {job_id} failed. Failed to execute program.\")
--> 241     raise FailedToAddJobToQueueException(f\"Job {job_id} failed. Failed to execute program.\")
    243 return job_id

FailedToAddJobToQueueException: Job 1705079177372 failed. Failed to execute program."
}

System Information

branch: BUG-changing-waveform-using-aprogram-for-qm-causes-crash

Name: qililab
Version: 0.24.0
Summary: Fundamental package for fast characterization and calibration of quantum chips.
Home-page: https://github.com/qilimanjaro-tech/qililab
Author: Qilimanjaro Quantum Tech
Author-email: info@qilimanjaro.tech
License: Apache License 2.0
Location: /home/opihl/miniconda3/envs/qililab/lib/python3.10/site-packages
Requires: h5py, lmfit, networkx, pandas, papermill, PyVISA-py, qblox-instruments, qcodes, qcodes-contrib-drivers, qibo, qiboconnection, qm-qua, qpysequence, qualang-tools, ruamel.yaml, rustworkx, submitit, tqdm, urllib3
Required-by: experiment-portfolio, qilitools

Platform info:             Linux-5.15.0-94-generic-x86_64-with-glibc2.35
Python version:            3.10.13
PyVISA version:            0.7.1
QCodes version:            0.42.0
QCodes Contrib version:    0.18.0
Qblox Instrument version:  0.10.1
Qpysequence version:       0.10.0
Quantum Machines version:  1.1.6
Qibo version:              0.1.12.dev0

Existing GitHub issues

[X] I have searched existing GitHub issues to make sure the issue does not already exist.

After trying to run another Qprogram after this I get the following error: 2024-04-09 15:50:32,317 - qm - ERROR - Failed: Machine not found qm-1712660655914 { "name": "QMRequestError", "message": "Failed: Machine not found qm-1712660655914", "stack": "--------------------------------------------------------------------------- QMRequestError Traceback (most recent call last) Cell In[25], line 2 1 platform.set_parameter(alias=readout_bus, parameter=ql.Parameter.LO_FREQUENCY, value=RESON_LO) ----> 2 platform.set_parameter(alias=readout_bus, parameter=ql.Parameter.IF, value=RESON_IF) 3 platform.set_parameter(alias=drive_bus, parameter=ql.Parameter.POWER, value=DRIVE_LO_POWER) 5 frequency = np.arange(DRIVE_LO_START, DRIVE_LO_STOP + DRIVE_LO_STEP / 2, DRIVE_LO_STEP)

File ~/miniconda3/envs/qililab/lib/python3.10/site-packages/qililab/platform/platform.py:528, in Platform.set_parameter(self, parameter, value, alias, channel_id) 526 return 527 element = self.get_element(alias=alias) --> 528 element.set_parameter(parameter=parameter, value=value, channel_id=channel_id)

File ~/miniconda3/envs/qililab/lib/python3.10/site-packages/qililab/platform/components/bus.py:178, in Bus.set_parameter(self, parameter, value, channel_id) 176 else: 177 try: --> 178 self.system_control.set_parameter( 179 parameter=parameter, value=value, channel_id=channel_id, port_id=self.port, bus_alias=self.alias 180 ) 181 except ParameterNotFound as error: 182 raise ParameterNotFound( 183 f\"No parameter with name {parameter.value} was found in the bus with alias {self.alias}\" 184 ) from error

File ~/miniconda3/envs/qililab/lib/python3.10/site-packages/qililab/system_control/system_control.py:130, in SystemControl.set_parameter(self, parameter, value, channel_id, port_id, bus_alias) 128 with contextlib.suppress(ParameterNotFound): 129 if isinstance(instrument, QuantumMachinesCluster) and bus_alias is not None: --> 130 instrument.set_parameter_of_bus(bus=bus_alias, parameter=parameter, value=value) 131 return 132 if isinstance(instrument, QbloxModule) and channel_id is None and port_id is not None:

File ~/miniconda3/envs/qililab/lib/python3.10/site-packages/qililab/instruments/quantum_machines/quantum_machines_cluster.py:361, in QuantumMachinesCluster.set_parameter_of_bus(self, bus, parameter, value) 359 if parameter == Parameter.IF: 360 intermediate_frequency = float(value) --> 361 self._qm.set_intermediate_frequency(element=bus, freq=intermediate_frequency) 362 element[\"intermediate_frequency\"] = intermediate_frequency 363 self._config[\"elements\"][bus][\"intermediate_frequency\"] = intermediate_frequency

File ~/miniconda3/envs/qililab/lib/python3.10/site-packages/qm/QuantumMachine.py:403, in QuantumMachine.set_intermediate_frequency(self, element, freq) 394 \"\"\"Sets the intermediate frequency of the element 395 396 Args: (...) 400 element 401 \"\"\" 402 element_inst = self._elements[element] --> 403 element_inst.set_intermediate_frequency(freq)

File src/dependency_injector/_cwiring.pyx:28, in dependency_injector._cwiring._get_sync_patched._patched()

File ~/miniconda3/envs/qililab/lib/python3.10/site-packages/qm/elements/element.py:53, in Element.set_intermediate_frequency(self, freq, capabilities) 51 freq = float(freq) 52 logger.debug(f\"Setting element '{self._name}' intermediate frequency to '{freq}'.\") ---> 53 self._frontend.set_intermediate_frequency(self._id, self._name, freq) 54 self._config.intermediate_frequency_double = float(freq) if capabilities.supports_double_frequency else 0.0 55 self._config.intermediate_frequency = int(freq)

File ~/miniconda3/envs/qililab/lib/python3.10/site-packages/qm/api/base_api.py:28, in connection_error_handle_decorator..wrapped(*args, kwargs) 25 @functools.wraps(func) 26 def wrapped(*args: P.args, *kwargs: P.kwargs) -> Ret: 27 try: ---> 28 return func(args, kwargs) 29 except grpclib.exceptions.GRPCError as e: 30 if is_debug():

File ~/miniconda3/envs/qililab/lib/python3.10/site-packages/qm/api/frontend_api.py:292, in FrontendApi.set_intermediate_frequency(self, machine_id, element, value) 290 set_frequency_request = HighQmApiRequestSetFrequency(qe=element, value=value) 291 request = HighQmApiRequest(quantum_machine_id=machine_id, set_frequency=set_frequency_request) --> 292 self._perform_qm_request(request)

File ~/miniconda3/envs/qililab/lib/python3.10/site-packages/qm/api/frontend_api.py:271, in FrontendApi._perform_qm_request(self, request) 269 error_message = \"\ \".join(it.message for it in response.errors) 270 logger.error(f\"Failed: {error_message}\") --> 271 raise QMRequestError(f\"Failed: {error_message}\")

QMRequestError: Failed: Machine not found qm-1712660655914" }

qilimanjaro-tech / qililab