25 Sept 2013 (Flowcell ID: D27GPACXX, 1x50)

[smondet]

• Steve Small sjs1@nyu.edu, Rhea Datta rrd5@nyu.edu, 4 lanes
• Sevinc Ercan se71@nyu.edu, Maxwell J Kramer mk174@nyu.edu, Sarah Albritton sea283@nyu.edu, 2 lanes
• Andreas Hochwagen ah128@nyu.edu, Tovah Elise Markowitz tem298@nyu.edu, 1 lane
• [x] create Submission Sheet(s) and email
• [x] validate Submission Sheet(s)
• [x] if needed, remove previous run directories on rabbot:/hiseq to make space
• [x] if needed, remove previous run directories on bowery:/scratch to make space
• [x] transfer data gencore@bowery$ qsub -v DIR=$RUNID ~/bin/rsync.pbs
• [x] make tarball qsub -v DIR=$RUNID ~/bin/tar.pbs
• [x] backup tarball to sftp.bio qsub -v DIR=$RUNID ~/bin/sftp.bio-backup.pbs
• [x] validate Submission Sheet(s) again and remove edit permissions
• [x] import Submission Sheet(s) and register flowcell
• [x] demultiplex
• [x] fastx
• [x] review statistics
• [x] deliver
• [x] send emails
• [ ] explore tovah's undetermined

[smondet]

 sm4431@pod:    0 jobs in /hiseq
 $ cat ~/do_targz
GZIP=-2 nice time tar  --exclude=Data/Intensities/L00*/C*.1  -czf ~/130927_SN911_0182_BD27GPACXX.tar.gz 130927_SN911_0182_BD27GPACXX
 sm4431@pod:    0 jobs in /hiseq
==  $ sh ~/do_targz
6819.02user 367.93system 2:47:23elapsed 71%CPU (0avgtext+0avgdata 5472maxresident)k
263781840inputs+180039856outputs (4major+713minor)pagefaults 0swaps
 sm4431@pod:    0 jobs in /hiseq
==  $ echo $?
0

 sm4431@pod:    0 jobs in ~
==  $ time nice md5sum 130927_SN911_0182_BD27GPACXX.tar.gz
a187c2058c13d024d756b5d56d97393a  130927_SN911_0182_BD27GPACXX.tar.gz

real    12m46.533s
user    3m52.076s
sys     0m50.083s

on Bowery

 $ qsub -v DIR=130927_SN911_0182_BD27GPACXX ~/bin/rsync-tar.pbs 
2599907.crunch.local

PBS Job Id: 2599907.crunch.local
Job Name:   tar-hiseq-rsync
Exec host:  compute-12-6/5
Execution terminated
Exit_status=0
resources_used.cput=00:20:25
resources_used.mem=8852kb
resources_used.vmem=347656kb
resources_used.walltime=00:57:08

$ qsub -v DIR=130927_SN911_0182_BD27GPACXX ~/bin/sftp.bio-backup.pbs
2609812.crunch.local
PBS Job Id: 2609812.crunch.local
Job Name:   sftp.bio-backup
Exec host:  compute-12-12/8
Execution terminated
Exit_status=0
resources_used.cput=00:18:39
resources_used.mem=9484kb
resources_used.vmem=275040kb
resources_used.walltime=00:26:53

 $ qsub -v DIR=130927_SN911_0182_BD27GPACXX ~/bin/untar.pbs 
2601357.crunch.local

PBS Job Id: 2601357.crunch.local
Job Name:   tar
Exec host:  compute-12-3/5
Execution terminated
Exit_status=0
resources_used.cput=00:34:24
resources_used.mem=4376kb
resources_used.vmem=216096kb
resources_used.walltime=04:22:44

gencore@bowery-0-3:/data/cgsb/gencore-raw/hiseq-700911 $ time md5sum 130927_SN911_0182_BD27GPACXX.tar.gz > `date +%F.%T`.130927_SN911_0182_BD27GPACXX.tar.gz.md5

real    7m6.085s
user    2m57.564s
sys     1m20.802s
gencore@bowery-0-3:/data/cgsb/gencore-raw/hiseq-700911 $ cat 2013-10-01.11\:31\:32.130927_SN911_0182_BD27GPACXX.tar.gz.md5 
a187c2058c13d024d756b5d56d97393a  130927_SN911_0182_BD27GPACXX.tar.gz

[smondet]

PBS Job Id: 2604381.crunch.local
Job Name:   HS-B2F-34346-D27GPACXX_2013-10-01_09-56-27.303118
Exec host:  compute-12-12/10+compute-12-12/8+compute-12-12/7+compute-12-12/6+compute-12-12/5+compute-12-12/4+compute-12-12/3+compute-12-12/2
Execution terminated
Exit_status=0
resources_used.cput=04:14:45
resources_used.mem=313152kb
resources_used.vmem=4245336kb
resources_used.walltime=01:22:57

 $ hitscore production fxqs start -user sm4431 -from-b2f 34346

Evaluation 34377 started!

PBS Job Id: 2604417.crunch.local
Job Name:   hs_fxqs_34377
Exec host:  compute-12-14/3
Execution terminated
Exit_status=0
resources_used.cput=01:10:04
resources_used.mem=33080kb
resources_used.vmem=436456kb
resources_used.walltime=01:12:09

[smondet]

gencore@bowery-0-3:/scratch/gencore/pbs2 $ qsub script_pbs2.pbs 
2629893.crunch.local
gencore@bowery-0-3:/scratch/gencore/pbs2 $ cat script_pbs2.pbs 
#!/bin/bash

#PBS -m abe
#PBS -M sm4431@nyu.edu
#PBS -l walltime=24:00:00
#PBS -V
#PBS -o /scratch/gencore/pbs2/pbs2.stdout
#PBS -e /scratch/gencore/pbs2/pbs2.stderr
#PBS -N pbs2
#PBS -q cgsb-s
export NAME=pbs2
export OUT_DIR=/scratch/gencore/pbs2/

echo "Script $NAME Starts on `date -R`"
FASTQ=$OUT_DIR/undetermined.fastq
gunzip -c /data/cgsb/gencore/out/Hochwagen/2013-09-25_D27GPACXX/Lane8/Sample_UndeterminedLane8/UndeterminedLane8_Undetermined_L008_R1_001.fastq.gz > $FASTQ

echo "Script $NAME Ends on `date -R`"

for i in `cat ../tovah_undetermined_minus_Ns ` ; do printf "$i.count: $i.barcode\n\tgrep 1:N:0:$i \$(FASTQ) | wc -l > $i.count\n\n" >> barcodes/Makefile ; done

and added at the top of the makefile:

FASTQ= /scratch/gencore/pbs2/undetermined.fastq

all: $(patsubst %.barcode,%.count,$(wildcard *.barcode))

gencore@bowery-0-3:/scratch/gencore/pbs3/barcodes $ cat ../script_pbs3.pbs 
#!/bin/bash

#PBS -m abe
#PBS -M sm4431@nyu.edu
#PBS -V
#PBS -o /scratch/gencore/pbs3/pbs3.stdout
#PBS -e /scratch/gencore/pbs3/pbs3.stderr
#PBS -N pbs3
#PBS -l nodes=1:ppn=12,walltime=48:00:00
#PBS -q cgsb-s
export NAME=pbs3
export OUT_DIR=/scratch/gencore/pbs3/

echo "Script $NAME Starts on `date -R`"

cd $OUT_DIR/barcodes
make -j 12 -k

echo "Script $NAME Ends on `date -R`"

gencore@bowery-0-3:/scratch/gencore/pbs3/barcodes $ qsub ../script_pbs3.pbs 
2629926.crunch.local

PBS Job Id: 2629926.crunch.local
Job Name:   pbs3
Exec host:  compute-13-10/11+compute-13-10/10+compute-13-10/9+compute-13-10/8+compute-13-10/7+compute-13-10/6+compute-13-10/5+compute-13-10/4+compute-13-10/3+compute-13-10/2+compute-13-10/1+compute-13-10/0
Execution terminated
Exit_status=0
resources_used.cput=06:38:04
resources_used.mem=63244kb
resources_used.vmem=2442376kb
resources_used.walltime=00:36:38

[smondet]

for i in *.count ; do printf "%09d\t$i\n" `cat $i` ; done | sort > results.tsv

 $ tail -n 22 results.tsv | sed 's/.count//' | sed 's/^0\+//'
48614   CACGGG
49831   GAAGGG
50044   GGTAAA
51462   AGGTAA
52810   GGGCAA
53480   GTGACC
55420   AAGAAA
55688   CAAGGT
57242   CAAGTG
58187   AAGTCC
59113   AAATCC
59939   TAAGTG
63462   ATGCAA
73320   GAAAAA
74193   AGGACC
74272   GTTCAA
85149   AATGCC
95235   AAAGGG
104927  GAGGAA
134789  CAAGGG
147113  TAAGGG
166346  AGGAAA

[smondet]

Nothing left to do on the computational side.