search

SBNSoftware / icarus-production

The repository is intended to support ICARUS production activities
GNU General Public License v3.0
0 stars 0 forks source link

clean up samweb #50

Open mt82 opened 1 month ago

mt82 commented 1 month ago

Clean up samweb datasets. In particular:

mt82 commented 1 month ago
These are datasets with two filesystem paths associated dataset nfiles path1 n. files in path1 n. existing files in path1 path2 n. files in path2 n. existing files in path2
2024_MinBias_Run_8654_Data_TriggerWG_v09_72_00_05p04_bnbminbias_online_purity_histos 4153 /pnfs/sbn/data/sbn_fd/poms_production/data/Data_TriggerWG 2710 2710 /pnfs/sbn/data/sbn_fd/poms_test/data/Data_TriggerWG 1443 0
Icaruspro_2024_Run2_Reprocessing_Run2_DataReprocess_v09_89_01_numimajority_online_purity_histos 59871 /pnfs/sbn/data/sbn_fd/poms_production/data/Run2_DataReprocess 38989 0 /pnfs/sbn/data/sbn_fd/poms_test/data/Run2_DataReprocess 20588 20588
Icaruspro_2024_Run2_Reprocessing_Run2_DataReprocess_v09_89_01_offbeamnumiminbias_online_purity_histos 13027 /pnfs/sbn/data/sbn_fd/poms_production/data/Run2_DataReprocess 13019 0 /pnfs/sbn/data/sbn_fd/poms_test/data/Run2_DataReprocess 8 8
Icaruspro_2024_Run2_Reprocess_Run2_Data_v09_89_01_numiminbias_online_purity_histos 8252 /pnfs/sbn/data/sbn_fd/poms_production/data/Run2_Data 2246 1903 /pnfs/sbn/data/sbn_fd/poms_test/data/Run2_Data 6004 6004
Icaruspro_2024_Run2_Rep_Run2_Rep_v09_89_01_bnbmajority_online_purity_histos 32465 /pnfs/sbn/data/sbn_fd/poms_production/data/Run2_Rep 29936 18492 /pnfs/sbn/data/sbn_fd/poms_test/data/Run2_Rep 2522 0
Icaruspro_2024_Run2_Rep_Run2_Rep_v09_89_01_numiminbias_online_purity_histos 3023 /pnfs/sbn/data/sbn_fd/poms_production/data/Run2_Rep 3020 3020 /pnfs/sbn/data/sbn_fd/poms_test/data/Run2_Rep 2 0
Icaruspro_2024_Run2_Rep_Run2_Rep_v09_89_01_offbeamnumiminbias_online_purity_histos 7214 /pnfs/sbn/data/sbn_fd/poms_production/data/Run2_Rep 7210 7210 /pnfs/sbn/data/sbn_fd/poms_test/data/Run2_Rep 2 0
Icaruspro_2024_Run3_11816_bnbmajority_Data_Run3_11816_allPMT_v09_90_00_bnbmajority_online_purity_histos 197 /pnfs/sbn/data/sbn_fd/poms_production/data/Data_Run3_11816_allPMT 196 196 /pnfs/sbn/data/sbn_fd/poms_test/data/Data_Run3_11816_allPMT 1 1
Icaruspro_2024_Run3_Run11813_OpRecWG_bnbmajority_Data_OpticalReconstructionWG_Run3_11813_bnbmajority_v09_87_00_bnbmajority_online_purity_histos 314 /pnfs/sbn/data/sbn_fd/poms_production/data/Data_OpticalReconstructionWG_Run3_11813_bnbmajority 163 163 /pnfs/sbn/data/sbn_fd/poms_test/data/Data_OpticalReconstructionWG_Run3_11813_bnbmajority 151 0
Icaruspro_2024_Run3_Run11813_OpRecWG_bnbminbias_Data_OpticalReconstructionWG_Run3_11813_bnbminbias_v09_87_00_bnbminbias_online_purity_histos 32 /pnfs/sbn/data/sbn_fd/poms_production/data/Data_OpticalReconstructionWG_Run3_11813_bnbminbias 11 11 /pnfs/sbn/data/sbn_fd/poms_test/data/Data_OpticalReconstructionWG_Run3_11813_bnbminbias 21 0
Icaruspro_2024_Run3_Run11813_OpRecWG_offbeambnbminbias_Data_OpticalReconstructionWG_Run3_11813_offbeambnbminbias_v09_87_00_offbeambnbminbias_online_purity_histos 235 /pnfs/sbn/data/sbn_fd/poms_production/data/Data_OpticalReconstructionWG_Run3_11813_offbeambnbminbias 93 93 /pnfs/sbn/data/sbn_fd/poms_test/data/Data_OpticalReconstructionWG_Run3_11813_offbeambnbminbias 142 0
Test_FT_stage0_Data_test_FT_v09_89_01_01_bnbmajority_online_purity_histos 535 /pnfs/sbn/data/sbn_fd/poms_production/data/Data_test_FT 216 216 /pnfs/sbn/data/sbn_fd/poms_test/data/Data_test_FT 319 319
icaruspro_production_v09_89_01_01_2024A_ICARUS_MC_CV_Sys_2024A_Sys_VAR2_tpcind2opaque_caf 5661 /pnfs/sbn/data/sbn_fd/poms_production/2024A_ICARUS_MC_CV_Sys/2024A_Sys_VAR1_tpcsignalshape 5635 5635 /pnfs/sbn/data/sbn_fd/poms_production/2024A_ICARUS_MC_CV_Sys/2024A_Sys_VAR2_tpcind2opaque 24 24
icaruspro_production_v09_89_01_01_2024A_ICARUS_MC_CV_Sys_2024A_Sys_VAR2_tpcind2opaque_calibtuple 5664 /pnfs/sbn/data/sbn_fd/poms_production/2024A_ICARUS_MC_CV_Sys/2024A_Sys_VAR1_tpcsignalshape 5640 5640 /pnfs/sbn/data/sbn_fd/poms_production/2024A_ICARUS_MC_CV_Sys/2024A_Sys_VAR2_tpcind2opaque 24 24
icaruspro_production_v09_89_01_01_2024A_ICARUS_MC_CV_Sys_2024A_Sys_VAR2_tpcind2opaque_flatcaf 5659 /pnfs/sbn/data/sbn_fd/poms_production/2024A_ICARUS_MC_CV_Sys/2024A_Sys_VAR1_tpcsignalshape 5635 5635 /pnfs/sbn/data/sbn_fd/poms_production/2024A_ICARUS_MC_CV_Sys/2024A_Sys_VAR2_tpcind2opaque 24 24
icaruspro_production_v09_89_01_01_2024A_ICARUS_MC_CV_Sys_2024A_Sys_VAR2_tpcind2opaque_stage1 5644 /pnfs/sbn/data/sbn_fd/poms_production/2024A_ICARUS_MC_CV_Sys/2024A_Sys_VAR1_tpcsignalshape 5620 5620 /pnfs/sbn/data/sbn_fd/poms_production/2024A_ICARUS_MC_CV_Sys/2024A_Sys_VAR2_tpcind2opaque 24 24
icaruspro_test_v09_72_00_04_2023A_ICARUS_BNB_Intime_Cosmics_test_stage1 6 /pnfs/sbn/data/sbn_fd/poms_test/2023A_ICARUS_BNB_Intime_Cosmics/small_BNB_sample 4 4 /pnfs/sbn/data/sbn_fd/poms_test/2023A_ICARUS_BNB_Intime_Cosmics/test 2 2
mt82 commented 1 month ago

files of dataset: 2024_MinBias_Run_8654_Data_TriggerWG_v09_72_00_05p04_bnbminbias_online_purity_histos in path: /pnfs/sbn/data/sbn_fd/poms_test/data/Data_TriggerWG/ were removed

for f in $(samweb -e icarus list-files "DataSet.Tag = 2024_MinBias_Run_8654_Data_TriggerWG_v09_72_00_05p04_bnbminbias_online_purity_histos and full_path like 'dcache:/pnfs/sbn/data/sbn_fd/poms_test/data/Data_TriggerWG/%'"); do 
  samweb -e icarus retire-file $f; 
done
mt82 commented 1 month ago

After investigating, all files in the second path are leftover files that can be safely deleted

$ cat retire_dataset_files_in_path.sh
#!/bin/bash

retire_ds_files_in_path() {
 ds=${1}
 path=${2}
 exp=${3}

 for f in $(samweb -e ${exp} list-files "DataSet.Tag = ${ds} and full_path like 'dcache:${path}/%'"); do
   samweb -e ${exp} retire-file $f;
 done
}

retire_ds_files_in_path "Icaruspro_2024_Run2_Reprocessing_Run2_DataReprocess_v09_89_01_numimajority_online_purity_histos" "/pnfs/sbn/data/sbn_fd/poms_test/data/Run2_DataReprocess" icarus
retire_ds_files_in_path "Icaruspro_2024_Run2_Reprocess_Run2_Data_v09_89_01_numiminbias_online_purity_histos" "/pnfs/sbn/data/sbn_fd/poms_test/data/Run2_Data" icarus
retire_ds_files_in_path "Icaruspro_2024_Run2_Rep_Run2_Rep_v09_89_01_bnbmajority_online_purity_histos" "/pnfs/sbn/data/sbn_fd/poms_test/data/Run2_Rep" icarus
retire_ds_files_in_path "Icaruspro_2024_Run2_Rep_Run2_Rep_v09_89_01_numiminbias_online_purity_histos" "/pnfs/sbn/data/sbn_fd/poms_test/data/Run2_Rep" icarus
retire_ds_files_in_path "Icaruspro_2024_Run2_Rep_Run2_Rep_v09_89_01_offbeamnumiminbias_online_purity_histos" "/pnfs/sbn/data/sbn_fd/poms_test/data/Run2_Rep" icarus
retire_ds_files_in_path "icaruspro_production_v09_89_01_01_2024A_ICARUS_MC_CV_Sys_2024A_Sys_VAR2_tpcind2opaque_caf" "/pnfs/sbn/data/sbn_fd/poms_production/2024A_ICARUS_MC_CV_Sys/2024A_Sys_VAR2_tpcind2opaque" icarus
$ source retire_dataset_files_in_path.sh &> retire_dataset_files_in_path.$(date '+%Y-%m-%d').log