Open arielleleon opened 2 months ago
Here is an example of how to use the new BasicUploadJobConfig with JobSetttings to tell aind-data-transfer-service to generate a session.json on HPC (provided by Jon Y)
from aind_data_transfer_models.core import (
ModalityConfigs,
BasicUploadJobConfigs,
)
from aind_metadata_mapper.models import (
SessionSettings,
JobSettings as GatherMetadataJobSettings,
)
from aind_metadata_mapper.bergamo.models import JobSettings as BergamoSessionSettings
from aind_data_schema_models.modalities import Modality
from aind_data_schema_models.platforms import Platform
from datetime import datetime
acq_datetime = datetime.fromisoformat("2000-01-01T01:11:33")
bergamo_session_settings = BergamoSessionSettings(
input_source="/allen/aind/scratch/svc_aind_upload/test_data_sets/bci/061022",
experimenter_full_name=["John Apple"],
subject_id="655019",
imaging_laser_wavelength=920,
fov_imaging_depth=200,
fov_targeted_structure="Primary Motor Cortex",
notes="test upload",
)
session_settings = SessionSettings(job_settings=bergamo_session_settings)
# directory_to_write_to is required, but will be set later.
# We can set it to "stage" for now.
metadata_job_settings = GatherMetadataJobSettings(
directory_to_write_to="stage", session_settings=session_settings
)
ephys_config = ModalityConfigs(
modality=Modality.ECEPHYS,
source=(
"/allen/aind/scratch/svc_aind_upload/test_data_sets/ecephys/655019_2023-04-03_18-17-07"
),
)
project_name = "Ephys Platform"
subject_id = "655019"
platform = Platform.ECEPHYS
s3_bucket = "private"
upload_job_configs = BasicUploadJobConfigs(
project_name=project_name,
s3_bucket=s3_bucket,
platform=platform,
subject_id=subject_id,
acq_datetime=acq_datetime,
modalities=[ephys_config],
metadata_configs=metadata_job_settings,
)
upload_jobs = [upload_job_configs]
# submit_request = SubmitJobRequest(
# upload_jobs=upload_jobs
# )
# post_request_content = json.loads(submit_request.model_dump_json(round_trip=True, exclude_none=True))
# submit_job_response = requests.post(url="http://aind-data-transfer-service-dev/api/v1/submit_jobs", json=post_request_content)
The JobSettings to model to generate session.json files on upload can now be passed to aind-data-transfer-service. Watchdog needs to allow users to input models and should parse them with the BaseJobSettings model and pass them the new aind-data-transfer-service endpoint metadata_configs