Open richelbilderbeek opened 2 years ago
1015 rm -rf data_issue_42_M1_p1_10
1016 rm -rf data_issue_42_M1_p1_10_ae
1017 ls | egrep "M3d.*p1.*10"
1018 ls | egrep "M1.*p1.*100"
1019 ls | egrep "M1.*p1*100"
1020 ls | egrep "M1.*p1.*100"
1021 rm -rf data_issue_42_M1_p1_100
1022 rm -rf data_issue_42_M1_p1_100_ae
1023 ls | egrep "M3d.*p1.*10"
1024 rm -rf data_issue_42_M3d_p1_10
1025 rm -rf data_issue_42_M3d_p1_10_ae
1026 ls | egrep "M3d.*p2.*1"
1027 rm -rf data_issue_42_M3d_p2_1
1028 rm -rf data_issue_42_M3d_p2_1_ae
1029 history
[richel@sens2021565-bianca ~]$ ./nsphs_ml_qt/scripts_bianca/20_start_issue_42_again.sh
Starting time: 2022-06-22T17:13:10+0200
Running on computer with HOSTNAME: sens2021565-bianca.uppmax.uu.se
Running at location /home/richel
autoenoder_model: M1
phenotype_prediction_model: p1
window_kb: 10
unique_id: issue_42_M1_p1_10
gcae_experiment_params_filename: /proj/sens2021565/nobackup/nsphs_ml_qt_results/data_issue_42_M1_p1_10/experiment_params.csv
jobid_21: 28703
jobid_22: 28704
jobid_24: 28705
jobid_25: 28706
jobid_26: 28707
jobid_29: 28708
autoenoder_model: M1
phenotype_prediction_model: p1
window_kb: 100
unique_id: issue_42_M1_p1_100
gcae_experiment_params_filename: /proj/sens2021565/nobackup/nsphs_ml_qt_results/data_issue_42_M1_p1_100/experiment_params.csv
jobid_21: 28709
jobid_22: 28710
jobid_24: 28711
jobid_25: 28712
jobid_26: 28713
jobid_29: 28714
autoenoder_model: M3d
phenotype_prediction_model: p1
window_kb: 10
unique_id: issue_42_M3d_p1_10
gcae_experiment_params_filename: /proj/sens2021565/nobackup/nsphs_ml_qt_results/data_issue_42_M3d_p1_10/experiment_params.csv
jobid_21: 28715
jobid_22: 28716
jobid_24: 28717
jobid_25: 28718
jobid_26: 28719
jobid_29: 28720
autoenoder_model: M3d
phenotype_prediction_model: p2
window_kb: 1
unique_id: issue_42_M3d_p2_1
gcae_experiment_params_filename: /proj/sens2021565/nobackup/nsphs_ml_qt_results/data_issue_42_M3d_p2_1/experiment_params.csv
jobid_21: 28721
jobid_22: 28722
jobid_24: 28723
jobid_25: 28724
jobid_26: 28725
jobid_29: 28726
End time: 2022-06-22T17:13:13+0200
Duration: 3 seconds
Still running:
[richel@sens2021565-bianca nsphs_ml_qt_results]$ cat 25_run_issue_42_M3d_p2_1.log
Parameters: /proj/sens2021565/nobackup/nsphs_ml_qt_results/data_issue_42_M3d_p2_1/experiment_params.csv
Number of parameters: 1
Correct number of arguments: 1
gcae_experiment_params_filename: /proj/sens2021565/nobackup/nsphs_ml_qt_results/data_issue_42_M3d_p2_1/experiment_params.csv
singularity_filename: nsphs_ml_qt/nsphs_ml_qt.sif
Starting time: 2022-06-22T17:26:08+0200
Running on computer with HOSTNAME: sens2021565-b16
Running at location /home/richel
'nsphs_ml_qt.sif' running with arguments 'Rscript nsphs_ml_qt/scripts_rackham/25_run.R /proj/sens2021565/nobackup/nsphs_ml_qt_results/data_issue_42_M3d_p2_1/experiment_params.csv'
gcae_experiment_params_filename: /proj/sens2021565/nobackup/nsphs_ml_qt_results/data_issue_42_M3d_p2_1/experiment_params.csv
Running the GCAE experiment
[richel@sens2021565-bianca nsphs_ml_qt_results]$ squeue
JOBID PARTITION NAME USER ST TIME NODES NODELIST(REASON)
28726 core 29_zip.s richel PD 0:00 1 (Dependency)
28720 core 29_zip.s richel PD 0:00 1 (Dependency)
28714 core 29_zip.s richel PD 0:00 1 (Dependency)
28708 core 29_zip.s richel PD 0:00 1 (Dependency)
28724 core 25_run.s richel R 18:23:50 1 sens2021565-b16
28718 core 25_run.s richel R 18:23:59 1 sens2021565-b16
28706 core 25_run.s richel R 18:28:10 1 sens2021565-b16
28712 core 25_run.s richel R 18:28:10 1 sens2021565-b16
[richel@sens2021565-bianca nsphs_ml_qt_results]$ cat 25_run_issue_42_M3d_p2_1.log
Parameters: /proj/sens2021565/nobackup/nsphs_ml_qt_results/data_issue_42_M3d_p2_1/experiment_params.csv
Number of parameters: 1
Correct number of arguments: 1
gcae_experiment_params_filename: /proj/sens2021565/nobackup/nsphs_ml_qt_results/data_issue_42_M3d_p2_1/experiment_params.csv
singularity_filename: nsphs_ml_qt/nsphs_ml_qt.sif
Starting time: 2022-06-22T17:26:08+0200
Running on computer with HOSTNAME: sens2021565-b16
Running at location /home/richel
'nsphs_ml_qt.sif' running with arguments 'Rscript nsphs_ml_qt/scripts_rackham/25_run.R /proj/sens2021565/nobackup/nsphs_ml_qt_results/data_issue_42_M3d_p2_1/experiment_params.csv'
gcae_experiment_params_filename: /proj/sens2021565/nobackup/nsphs_ml_qt_results/data_issue_42_M3d_p2_1/experiment_params.csv
Running the GCAE experiment
Save the GCAE experiment results
slurmstepd: error: *** JOB 28724 ON sens2021565-b16 CANCELLED AT 2022-06-26T21:26:21 DUE TO TIME LIMIT ***
gcae_experiment_params_filename: /proj/sens2021565/nobackup/nsphs_ml_qt_results/data_issue_42_M1_p1_10/experiment_params.csv
cat 25_run_issue_42_M1_p1_10.log
cd data_issue_42_M1_p1_10_ae
WORKED
gcae_experiment_params_filename: /proj/sens2021565/nobackup/nsphs_ml_qt_results/data_issue_42_M1_p1_100/experiment_params.csv
cat 25_run_issue_42_M1_p1_100.log
cd data_issue_42_M1_p1_100_ae
WORKED
gcae_experiment_params_filename: /proj/sens2021565/nobackup/nsphs_ml_qt_results/data_issue_42_M3d_p1_10/experiment_params.csv
cat 25_run_issue_42_M3d_p1_10.log
WORKED
gcae_experiment_params_filename: /proj/sens2021565/nobackup/nsphs_ml_qt_results/data_issue_42_M3d_p2_1/experiment_params.csv
cat 25_run_issue_42_M3d_p2_1.log
TIMOUT
There we go again:
[richel@sens2021565-bianca ~]$ ./nsphs_ml_qt/scripts_bianca/20_start_issue_42_again.sh
Starting time: 2022-06-27T09:37:18+0200
Running on computer with HOSTNAME: sens2021565-bianca.uppmax.uu.se
Running at location /home/richel
autoenoder_model: M3d
phenotype_prediction_model: p2
window_kb: 1
unique_id: issue_42_M3d_p2_1
gcae_experiment_params_filename: /proj/sens2021565/nobackup/nsphs_ml_qt_results/data_issue_42_M3d_p2_1/experiment_params.csv
jobid_21: 28735
jobid_22: 28736
jobid_24: 28737
jobid_25: 28738
jobid_26: 28739
jobid_29: 28740
End time: 2022-06-27T09:37:19+0200
Duration: 1 seconds
This run is know to need at least 100 hours, so Thursday 14:00 is the earliest. I will check on Friday morning.
The error message is clear:
[richel@sens2021565-bianca nsphs_ml_qt_results]$ cat 25_run_issue_42_M3d_p2_1.log
Parameters: /proj/sens2021565/nobackup/nsphs_ml_qt_results/data_issue_42_M3d_p2_1/experiment_params.csv
Number of parameters: 1
Correct number of arguments: 1
gcae_experiment_params_filename: /proj/sens2021565/nobackup/nsphs_ml_qt_results/data_issue_42_M3d_p2_1/experiment_params.csv
singularity_filename: nsphs_ml_qt/nsphs_ml_qt.sif
Starting time: 2022-06-27T09:37:36+0200
Running on computer with HOSTNAME: sens2021565-b16
Running at location /home/richel
'nsphs_ml_qt.sif' running with arguments 'Rscript nsphs_ml_qt/scripts_rackham/25_run.R /proj/sens2021565/nobackup/nsphs_ml_qt_results/data_issue_42_M3d_p2_1/experiment_params.csv'
gcae_experiment_params_filename: /proj/sens2021565/nobackup/nsphs_ml_qt_results/data_issue_42_M3d_p2_1/experiment_params.csv
Running the GCAE experiment
Error in gcaer::do_gcae_experiment(gcae_experiment_params = gcae_experiment_params) :
There is less projected then intended.
Tip 1: this is likely to be due to a continued run.
Tip 2: run 'gcaer::clean_gcaer_tempfolder()'
nrow(losses_from_project_table): 101
length(gcae_experiment_params$analyse_epochs): 100
head(losses_from_project_table):
| epoch| losses_from_project|
|-----:|-------------------:|
| 10| 0.8466854|
| 20| 0.7800358|
| 30| 0.9017211|
| 40| 0.7502268|
| 50| 0.7531211|
| 60| 0.6763079|
head(gcae_experiment_params$analyse_epochs):
10
20
30
40
50
60
Execution halted
End time: 2022-06-28T16:42:27+0200
Duration: 111891 seconds
[richel@sens2021565-bianca nsphs_ml_qt_results]$ rm -rf data_issue_42_M3d_p2_1
[richel@sens2021565-bianca nsphs_ml_qt_results]$ rm -rf data_issue_42_M3d_p2_1_ae
[richel@sens2021565-bianca ~]$ ./nsphs_ml_qt/scripts_bianca/20_start_issue_42_again.sh
Starting time: 2022-06-29T09:17:00+0200
Running on computer with HOSTNAME: sens2021565-bianca.uppmax.uu.se
Running at location /home/richel
autoenoder_model: M3d
phenotype_prediction_model: p2
window_kb: 1
unique_id: issue_42_M3d_p2_1
gcae_experiment_params_filename: /proj/sens2021565/nobackup/nsphs_ml_qt_results/data_issue_42_M3d_p2_1/experiment_params.csv
jobid_21: 28745
jobid_22: 28746
jobid_24: 28747
jobid_25: 28748
jobid_26: 28749
jobid_29: 28750
End time: 2022-06-29T09:17:01+0200
Duration: 1 seconds
100 hours from now is Sunday 13:00, so will check on Monday morning again :-)
~/GitHubs/nsphs_ml_qt_results/issue_42_20220622//data_issue_42_M3e_p1_1000_ae/genotype_concordances.csv does not exist
[richel@sens2021565-bianca nsphs_ml_qt_results]$ rm -rf data_issue_42_M3e_p1_1000_ae
[richel@sens2021565-bianca nsphs_ml_qt_results]$ rm -rf data_issue_42_M3e_p1_1000
[richel@sens2021565-bianca ~]$ ./nsphs_ml_qt/scripts_bianca/20_start_issue_42_again.sh
Starting time: 2022-06-29T09:44:50+0200
Running on computer with HOSTNAME: sens2021565-bianca.uppmax.uu.se
Running at location /home/richel
autoenoder_model: M3e
phenotype_prediction_model: p1
window_kb: 1000
unique_id: issue_42_M3e_p1_1000
gcae_experiment_params_filename: /proj/sens2021565/nobackup/nsphs_ml_qt_results/data_issue_42_M3e_p1_1000/experiment_params.csv
jobid_21: 28757
jobid_22: 28758
jobid_24: 28759
jobid_25: 28760
jobid_26: 28761
jobid_29: 28762
End time: 2022-06-29T09:44:51+0200
Duration: 1 seconds