pyiron tables cannot be loaded from unpacked project

[pmrv]

I've tried running the ACE notebook to reproduce the phase diagram, but encountered two problems:

1. the job id to load the table job is hardcoded, so it doesn't match the imported table (same is true in the EAM notebook)
2. when I load the table job by name I get this error from cloudpickle

2024-06-03 11:43:39,759 - pyiron_log - WARNING - Could not instantiate analysis_project, no such path /cmmc/u/smenon/potentials_2022_ACE/ace_results/AlLi_phase_dia_2/.

---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
Cell In[7], line 1
----> 1 table = pr.load('table')
      2 df = table.get_dataframe()

File [~/software/pyiron_base/pyiron_base/project/jobloader.py:104](https://localhost:8000/user/zora/lab/tree/zora/pyiron/projects/2024/05_POTENTIALS_PhaseDiagrams/~/software/pyiron_base/pyiron_base/project/jobloader.py#line=103), in JobLoader.__call__(self, job_specifier, convert_to_object)
    103 def __call__(self, job_specifier, convert_to_object=None) -> GenericJob:
--> 104     return super().__call__(job_specifier, convert_to_object=convert_to_object)

File [~/software/pyiron_base/pyiron_base/project/jobloader.py:75](https://localhost:8000/user/zora/lab/tree/zora/pyiron/projects/2024/05_POTENTIALS_PhaseDiagrams/~/software/pyiron_base/pyiron_base/project/jobloader.py#line=74), in _JobByAttribute.__call__(self, job_specifier, convert_to_object)
     71     state.logger.warning(
     72         f"Job '{job_specifier}' does not exist and cannot be loaded"
     73     )
     74     return None
---> 75 return self._project.load_from_jobpath(
     76     job_id=job_id,
     77     convert_to_object=(
     78         convert_to_object
     79         if convert_to_object is not None
     80         else self.convert_to_object
     81     ),
     82 )

File [~/software/pyiron_atomistics/pyiron_atomistics/project.py:327](https://localhost:8000/user/zora/lab/tree/zora/pyiron/projects/2024/05_POTENTIALS_PhaseDiagrams/~/software/pyiron_atomistics/pyiron_atomistics/project.py#line=326), in Project.load_from_jobpath(self, job_id, db_entry, convert_to_object)
    313 def load_from_jobpath(self, job_id=None, db_entry=None, convert_to_object=True):
    314     """
    315     Internal function to load an existing job either based on the job ID or based on the database entry dictionary.
    316 
   (...)
    325         GenericJob, JobCore: Either the full GenericJob object or just a reduced JobCore object
    326     """
--> 327     job = super(Project, self).load_from_jobpath(
    328         job_id=job_id, db_entry=db_entry, convert_to_object=convert_to_object
    329     )
    330     job.project_hdf5._project = self.__class__(path=job.project_hdf5.file_path)
    331     return job

File [~/software/pyiron_base/pyiron_base/project/generic.py:1001](https://localhost:8000/user/zora/lab/tree/zora/pyiron/projects/2024/05_POTENTIALS_PhaseDiagrams/~/software/pyiron_base/pyiron_base/project/generic.py#line=1000), in Project.load_from_jobpath(self, job_id, db_entry, convert_to_object)
    999 job = JobPath.from_job_id(db=self.db, job_id=job_id)
   1000 if convert_to_object:
-> 1001     job = job.to_object()
   1002     job.reset_job_id(job_id=job_id)
   1003     job.set_input_to_read_only()

File [~/software/pyiron_base/pyiron_base/jobs/job/core.py:586](https://localhost:8000/user/zora/lab/tree/zora/pyiron/projects/2024/05_POTENTIALS_PhaseDiagrams/~/software/pyiron_base/pyiron_base/jobs/job/core.py#line=585), in JobCore.to_object(self, object_type, **qwargs)
    580 if self.project_hdf5.is_empty:
    581     raise ValueError(
    582         'The HDF5 file of this job with the job_name: "'
    583         + self.job_name
    584         + '" is empty, so it can not be loaded.'
    585     )
--> 586 return self.project_hdf5.to_object(object_type, **qwargs)

File [~/software/pyiron_base/pyiron_base/storage/hdfio.py:1142](https://localhost:8000/user/zora/lab/tree/zora/pyiron/projects/2024/05_POTENTIALS_PhaseDiagrams/~/software/pyiron_base/pyiron_base/storage/hdfio.py#line=1141), in ProjectHDFio.to_object(self, class_name, **kwargs)
   1128 def to_object(self, class_name=None, **kwargs):
   1129     """
   1130     Load the full pyiron object from an HDF5 file
   1131 
   (...)
   1140         pyiron object of the given class_name
   1141     """
-> 1142     return _to_object(self, class_name, **kwargs)

File [~/software/pyiron_base/pyiron_base/storage/hdfio.py:138](https://localhost:8000/user/zora/lab/tree/zora/pyiron/projects/2024/05_POTENTIALS_PhaseDiagrams/~/software/pyiron_base/pyiron_base/storage/hdfio.py#line=137), in _to_object(hdf, class_name, **kwargs)
    135 init_args.update(kwargs)
    137 obj = class_object(**init_args)
--> 138 obj.from_hdf(hdf=hdf.open(".."), group_name=hdf.h5_path.split("/")[-1])
    139 if static_isinstance(obj=obj, obj_type="pyiron_base.jobs.job.generic.GenericJob"):
    140     module_name = module_path.split(".")[0]

File [~/software/pyiron_base/pyiron_base/jobs/datamining.py:738](https://localhost:8000/user/zora/lab/tree/zora/pyiron/projects/2024/05_POTENTIALS_PhaseDiagrams/~/software/pyiron_base/pyiron_base/jobs/datamining.py#line=737), in TableJob.from_hdf(self, hdf, group_name)
    730 def from_hdf(self, hdf=None, group_name=None):
    731     """
    732     Restore pyiron table job from HDF5
    733 
   (...)
    736         group_name:
    737     """
--> 738     super(TableJob, self).from_hdf(hdf=hdf, group_name=group_name)
    739     hdf_version = self.project_hdf5.get("HDF_VERSION", "0.1.0")
    740     if hdf_version == "0.3.0":

File [~/software/pyiron_base/pyiron_base/jobs/job/generic.py:1142](https://localhost:8000/user/zora/lab/tree/zora/pyiron/projects/2024/05_POTENTIALS_PhaseDiagrams/~/software/pyiron_base/pyiron_base/jobs/job/generic.py#line=1141), in GenericJob.from_hdf(self, hdf, group_name)
   1140     exe_dict["READ_ONLY"] = self._hdf5["executable/executable/READ_ONLY"]
   1141     job_dict["executable"] = {"executable": exe_dict}
-> 1142 self.from_dict(job_dict=job_dict)

File [~/software/pyiron_base/pyiron_base/jobs/datamining.py:715](https://localhost:8000/user/zora/lab/tree/zora/pyiron/projects/2024/05_POTENTIALS_PhaseDiagrams/~/software/pyiron_base/pyiron_base/jobs/datamining.py#line=714), in TableJob.from_dict(self, job_dict)
    713 self._enforce_update = bool_dict["enforce_update"]
    714 self._pyiron_table.convert_to_object = bool_dict["convert_to_object"]
--> 715 self._pyiron_table.add._from_hdf(job_dict["input"])

File [~/software/pyiron_base/pyiron_base/jobs/datamining.py:88](https://localhost:8000/user/zora/lab/tree/zora/pyiron/projects/2024/05_POTENTIALS_PhaseDiagrams/~/software/pyiron_base/pyiron_base/jobs/datamining.py#line=87), in FunctionContainer._from_hdf(self, hdf)
     87 def _from_hdf(self, hdf):
---> 88     self._user_function_dict = _from_pickle(hdf=hdf, key="user_function_dict")
     89     self._system_function_dict = _from_pickle(hdf=hdf, key="system_function_dict")

File [~/software/pyiron_base/pyiron_base/jobs/datamining.py:45](https://localhost:8000/user/zora/lab/tree/zora/pyiron/projects/2024/05_POTENTIALS_PhaseDiagrams/~/software/pyiron_base/pyiron_base/jobs/datamining.py#line=44), in _from_pickle(hdf, key)
     43 def _from_pickle(hdf, key):
     44     try:
---> 45         return cloudpickle.loads(codecs.decode(hdf[key].encode(), "base64"))
     46     except ModuleNotFoundError:
     47         import dill

TypeError: code expected at most 16 arguments, got 18

@jan-janssen This seems to be related to cloudpickle or at least the we write the pickled state. Trying to unpickle with dill didn't help in this case.

@srmnitc I will try to recreate teh etable locally as is done in the HDNNP notebook and report back.

[pmrv]

Apparently @srmnitc ran the notebooks with python 3.11 and I tried load them with 3.10, which may be the issue here.

[pmrv]

@srmnitc I will try to recreate teh etable locally as is done in the HDNNP notebook and report back.

This is how I ended up doing it and it worked without problems.