LM training

[simon-clematide]

export LNG=de && python3 lm/preprocessing.py with configs/${LNG}_word_pp.json >configs/${LNG}_word_pp.json.output

export LNG=es && CUDA_VISIBLE_DEVICES=3 nohup python3 lm/trainer.py with configs/${LNG}_word_lm.json &> logs/${LNG}_word_lm.log &

WORD LMs

• sl: gpu 0 cutoff 20 started MONDAY 18h export LNG=sl && CUDA_VISIBLE_DEVICES=0 nohup python3 lm/trainer.py with configs/${LNG}_word_lm.json &> logs/${LNG}_word_lm.log &

• pt: gpu 1 cutoff 98

  • command export LNG=pt && CUDA_VISIBLE_DEVICES=1 nohup python3 lm/trainer.py with configs/${LNG}_word_lm.json &> logs/${LNG}_word_lm.log &
  • start 2019-05-06 18:35:46 end ` training duration ` validation ppl

• es: gpu 3 cutoff 200

  • command export LNG=es && CUDA_VISIBLE_DEVICES=3 nohup python3 lm/trainer.py with configs/${LNG}_word_lm.json &> logs/${LNG}_word_lm.log &
  • start 2019-05-06 18:27:23 end ` training duration ` validation ppl

• de: gpu 4 Flair vocabulary of types (UNK <= 220) has 100398 types. Coverage: 0.780

  • command export LNG=de && CUDA_VISIBLE_DEVICES=4 nohup python3 lm/trainer.py with configs/${LNG}_word_lm.json &> logs/${LNG}_word_lm.log &
  • start 2019-05-07 02:11:03 end ` training duration ` validation ppl

• sv: gpu 2 Flair vocabulary of types (UNK <= 49) has 90786 types. Coverage: 0.8

  • command export LNG=sv && CUDA_VISIBLE_DEVICES=2 nohup python3 lm/trainer.py with configs/${LNG}_word_lm.json &> logs/${LNG}_word_lm.log &
  • start ` end training duration ` validation ppl

• hu: gpu 2 Flair vocabulary of types (UNK <= 4) has 532247 types. Coverage: 0.797 UNK has a train set frequency of 3293080 (max: 6812055) and a relative frequency of 0.04075

  • command export LNG=hu && CUDA_VISIBLE_DEVICES=2 nohup python3 lm/trainer.py with configs/${LNG}_word_lm.json &> logs/${LNG}_word_lm.log &
  • start ` end training duration ` validation ppl

• is: gpu 2

ON s3it

• is: Flair vocabulary of types (UNK <= 4) has 59766 types. Coverage: 0.824 UNK has a train set frequency of 379524 (max: 315204) and a relative frequency of 0.07392 export LNG=is && CUDA_VISIBLE_DEVICES=0 nohup python3 lm/trainer.py with configs/${LNG}_word_lm.json &> logs/${LNG}_word_lm.log &

CHAR LMs

export LNG=XXXX && python3 lm/preprocessing.py with configs/${LNG}_char_pp.json >configs/${LNG}_char_pp.json.output

export LNG=XXXX && CUDA_VISIBLE_DEVICES=666 nohup python3 lm/trainer.py with configs/${LNG}_char_lm.json &> logs/${LNG}_char_lm.log &

• is: PID 8938 gpu 2 cutoff 14 export LNG=is && CUDA_VISIBLE_DEVICES=2 nohup python3 lm/trainer.py with configs/${LNG}_char_lm.json &> logs/${LNG}_char_lm.log &

• es: gpu ? cutoff 14 export LNG=es && CUDA_VISIBLE_DEVICES=3 nohup python3 lm/trainer.py with configs/${LNG}_char_lm.json &> logs/${LNG}_char_lm.log &

• sl: PID 16202 gpu 1 cutoff 51 export LNG=sl && CUDA_VISIBLE_DEVICES=1 nohup python3 lm/trainer.py with configs/${LNG}_char_lm.json &> logs/${LNG}_char_lm.log &

• pt: PID 13588 gpu 3 cutoff 346 export LNG=pt && CUDA_VISIBLE_DEVICES=3 nohup python3 lm/trainer.py with configs/${LNG}_char_lm.json &> logs/${LNG}_char_lm.log & -de: gpu4 export LNG=de && CUDA_VISIBLE_DEVICES=4 nohup python3 lm/trainer.py with configs/${LNG}_char_lm.json &> logs/${LNG}_char_lm.log &

-en: gpu2 export LNG=en && CUDA_VISIBLE_DEVICES=2 nohup python3 lm/trainer.py with configs/${LNG}_char_lm.json &> logs/${LNG}_char_lm.log &

deleting the corpus output directory of a language

rm -r wplmdata-preprocessed/sl/

[simon-clematide]

overview on models

(https://gitlab.ifi.uzh.ch/siclemat/neural-wp-language-models-for-hististorical-normaliization)

lang	mtype	status	best valid ppl	training start	training end	duration
de	word	done	63.68	2019-05-07 02:11:03	2019-05-14 14:42:46	7 days 12 hrs 31 mins 43 secs
de	char	done	2.71	2019-05-17 19:16:37	2019-05-24 17:00:37	6 days 21 hrs 44 mins
en	word
en	char
es	word	done	49.97	2019-05-06 18:27:23	2019-05-14 14:37:03	7 days 20 hrs 9 mins 40 secs
es	char	done	2.54	2019-05-17 01:57:45	2019-05-24 21:30:44	7 days 19 hrs 32 mins 59 secs
is	word	done	105.95	2019-05-10 14:00:50	2019-05-14 18:06:47	1 day 4 hrs 49 mins 43 secs
is	char	done	2.97	2019-05-14 15:46:19	2019-05-15 20:36:02	1 day 4 hrs 49 mins 43 secs
pt	word	done	61.69	2019-05-06 18:09:35	2019-05-14 14:42:20	7 days 20 hrs 32 mins 45 secs
pt	char	done	2.65	2019-05-15 17:37:54	2019-05-23 07:27:57	7 days 13 hrs 50 mins 3 secs
sl	word	done	83.17	2019-05-06 17:58:21	2019-05-14 14:42:33	7 days 20 hrs 44 mins 12 secs
sl	char	done	2.87	2019-05-14 17:32:12	2019-05-19 22:13:10	5 days 4 hrs 40 mins 58 secs
sv	word	done	12.44	2019-05-16 02:00:53	2019-05-24 21:10:12	8 days 19 hrs 9 mins 19 secs
sv	char	onrattle		Fri 24 May 23:55:03

(venv) siclemat@rattle:~/nnlm-2019/nn_lms/models.d/pt_char_lm$ grep -oP "valid ppl .{8}" < loss.txt|sort -rn

(http://www.grun1.com/utils/dateTimeDiff.cfm)

[simon-clematide]

Start slurm job on s3it

#!/bin/bash
#SBATCH --ntasks=1
#SBATCH --time=7-0:00:00
#SBATCH --gres gpu:Tesla-K80:1 --mem=10000
#SBATCH --cpus-per-task=2
#SBATCH --mail-type=ALL        # notifications for job done & fail
#SBATCH --mail-user=simon.clematide@uzh.ch # send-to address
#SBATCH -p vesta
#SBATCH --qos vesta
#SBATCH -A uzh
conda activate p36lm2019
python3 lm/trainer.py with configs/is_word_lm.json &> logs/is_word_lm.log