bamToFastq -i IonXpress_015_IMP3_Ab42Round2.bam -fq IonXpress_015_IMP3_Ab42Round2.fq bamToFastq -i IonXpress_011_IMP2_A4VRound4.bam -fq IonXpress_011_IMP2_A4VRound4.fq bamToFastq -i IonXpress_009_IMP1_4-5-6-7PeptideLinrary.bam -fq IonXpress_009_IMP1_4-5-6-7PeptideLinrary.fq /data/results/tools/align/bowtie2-2.2.3/bowtie2-build backbone_wo_insert.fa backbone_wo_insert /data/results/tools/align/bowtie2-2.2.3/bowtie2 -x bb_wo_ins IonXpress_015_IMP3_Ab42Round2.fq -S IonXpress_015_IMP3_Ab42Round2.sam 13825029 reads; of these: 13825029 (100.00%) were unpaired; of these: 3318079 (24.00%) aligned 0 times 10506950 (76.00%) aligned exactly 1 time 0 (0.00%) aligned >1 times 76.00% overall alignment rate /data/results/tools/align/bowtie2-2.2.3/bowtie2 -x bb_wo_ins IonXpress_015_IMP3_Ab42Round2.fq | /data/results/tools/samtools/samtools-0.1.19/samtools view -uhS -F4 - > IonXpress_015_IMP3_Ab42Round2.aligned.bam /data/results/tools/align/bowtie2-2.2.3/bowtie2 -x bb_wo_ins IonXpress_011_IMP2_A4VRound4.fq | /data/results/tools/samtools/samtools-0.1.19/samtools view -uhS -F4 - > IonXpress_011_IMP2_A4VRound4.aligned.bam 15233125 reads; of these: 15233125 (100.00%) were unpaired; of these: 7881760 (51.74%) aligned 0 times 7351365 (48.26%) aligned exactly 1 time 0 (0.00%) aligned >1 times 48.26% overall alignment rate /data/results/tools/align/bowtie2-2.2.3/bowtie2 -x bb_wo_ins IonXpress_009_IMP1_4-5-6-7PeptideLinrary.fq | /data/results/tools/samtools/samtools-0.1.19/samtools view -uhS -F4 - > IonXpress_009_IMP1_4-5-6-7PeptideLinrary.aligned.bam 16102381 reads; of these: 16102381 (100.00%) were unpaired; of these: 7311491 (45.41%) aligned 0 times 8790890 (54.59%) aligned exactly 1 time 0 (0.00%) aligned >1 times 54.59% overall alignment rate reczko@estia:/data/images/proton/run149$ /data/results/tools/samtools/samtools-1.0/samtools sort -@ 5 IonXpress_009_IMP1_4-5-6-7PeptideLinrary.aligned.bam IonXpress_009_IMP1_4-5-6-7PeptideLinrary.aligned.bam [bam_sort_core] merging from 10 files... reczko@estia:/data/images/proton/run149$ ls -lt | head total 66006956 -rw-rw-r-- 1 reczko reczko 1262909531 Mar 4 14:21 IonXpress_009_IMP1_4-5-6-7PeptideLinrary.aligned.bam.bam reczko@estia:/data/images/proton/run149$ /data/results/tools/samtools/samtools-1.0/samtools index IonXpress_009_IMP1_4-5-6-7PeptideLinrary.aligned.bam.bam #reczko@max:/data/results/tools/align/jvarkit/samtools-htsjdk-96aa8be$ ant htsjdk-jar export JAVA_HOME=/home/reczko/a/tools/libs/java/jdk1.8.0_66 apt-get install xsltproc reczko@max:/data/results/tools/align/jvarkit/jvarkit-master$ make biostar59647 /mnt/fix/b/solid.data_results.recovered.data/tools/align/jvarkit/jvarkit-master/dist/biostar59647 head -999 *sam > foo cat foo | java -Xmx500m -cp "/data/results/tools/align/jvarkit/jvarkit/htsjdk-1.128/dist/commons-jexl-2.1.1.jar:/data/results/tools/align/jvarkit/jvarkit/htsjdk-1.128/dist/commons-logging-1.1.1.jar:/data/results/tools/align/jvarkit/jvarkit/htsjdk-1.128/dist/htsjdk-1.128.jar:/data/results/tools/align/jvarkit/jvarkit/htsjdk-1.128/dist/snappy-java-1.0.3-rc3.jar:/data/results/tools/align/jvarkit/jvarkit/dist-1.128/biostar59647.jar" com.github.lindenb.jvarkit.tools.biostar.Biostar59647 -r backbone_wo_insert.fa foo ref:9-20 | xmllint --format - |awk -f /data/images/proton/run149/extract_insert1.awk cat IonXpress_015_IMP3_Ab42Round2.sam | java -Xmx500m -cp "/data/results/tools/align/jvarkit/jvarkit/htsjdk-1.128/dist/commons-jexl-2.1.1.jar:/data/results/tools/align/jvarkit/jvarkit/htsjdk-1.128/dist/commons-logging-1.1.1.jar:/data/results/tools/align/jvarkit/jvarkit/htsjdk-1.128/dist/htsjdk-1.128.jar:/data/results/tools/align/jvarkit/jvarkit/htsjdk-1.128/dist/snappy-java-1.0.3-rc3.jar:/data/results/tools/align/jvarkit/jvarkit/dist-1.128/biostar59647.jar" com.github.lindenb.jvarkit.tools.biostar.Biostar59647 -r backbone_wo_insert.fa IonXpress_015_IMP3_Ab42Round2.sam ref:9-20 | xmllint --format - |awk -f /data/images/proton/run149/extract_insert1.awk > IonXpress_015_IMP3_Ab42Round2.ins.fa http://www.ncbi.nlm.nih.gov/Web/Newsltr/Spring04/blastlab.html BLASTClust accepts a number of parameters that can be used to control the stringency of clustering including thresholds for score density, percent identity, and alignment length. The BLASTClust program has a number of applications, the simplest of which is to create a non-redundant set of sequences from a source database. As an example, one might have a library of a few thousand short nucleotide sequence reads and wish to replace these with a non-redundant set. To produce the non-redundant set, one might use: blastclust -i infile -o outfile -p F -L .9 -b T -S 95 The sequences in "infile" will be clustered and the results will be written to "outfile". The input sequences are identified as nucleotide (-p F); "-p T", or protein, is the default. To register a pairwise match two sequences will need to be 95% identical (-S 95) over an area covering 90% of the length (-L .9) of each sequence (-b T) . Using "-b F" instead of "-b T" would enforce the alignment length threshold on only one member of a sequence pair. The parameter "S", used here to specify the percent identity, can also be used to specify, instead, a "score density." The latter is equivalent to the BLAST score divided by the alignment length. If "S" is given as a number between 0 and 3, it is interpreted as a score density threshold; otherwise it is interpreted as a percent identity threshold. To create a stringent non-redundant protein sequence set, use the following command line: blastclust -i infile -o outfile -p T -L 1 -b T -S 100 In this case, only sequences which are identical will be clustered together. The “blastclust.txt” file in the standalone BLAST package details the full range of BLASTClust parameters. reczko@solid:/data/images/proton/run149/extract_insert$ /data/results/tools/align/blast/blast-2.2.26/bin/blastclust --help blastclust 2.2.26 arguments: -i FASTA input file (program will format the database and remove files in the end) [File In] default = stdin -a Number of CPU's to use [Integer] default = 1 -o Output file for list of clusters [File Out] default = stdout -L Length coverage threshold [Real] default = 0.9 -S Score coverage threshold (bit score / length if < 3.0, percentage of identities otherwise) [Real] default = 1.75 -b Require coverage on both neighbours? [T/F] default = TRUE -s File to save all neighbours [File Out] Optional -r File to restore neighbors for reclustering [File In] Optional -d Input as a database [File In] Optional -v Print progress messages (verbose mode) [File Out] default = stdout -C Complete unfinished clustering [T/F] default = FALSE -l Restrict reclustering to id list [File In] Optional -p Is input proteins? [T/F] default = TRUE -e Enable id parsing in database formatting? [T/F] default = TRUE -c Configuration file with advanced options [File In] Optional -W Word size to use (0 for default: proteins 3, nucleotides 32) [Integer] default = 0 PATH=/data/results/tools/align/blast/blast-2.2.26/bin:$PATH; export PATH cat ../baz | /data/results/tools/align/blast/blast-2.2.26/bin/blastclust -o baz.clust -p F -L .9 -b T -S 95 -W 3 @@ cd /data/results/tools/align/multiple/cd-hit-v4.6.1-2012-08-27 make openmp=yes cd-hit - est -i est_human -o est_human95 -c 0.95 -n 8 /data/results/tools/align/multiple/cd-hit-v4.6.1-2012-08-27/cd-hit-est ====== CD-HIT version 4.6 (built on Mar 4 2015) ====== Usage: ./cd-hit-est [Options] Options -i input filename in fasta format, required -o output filename, required -c sequence identity threshold, default 0.9 this is the default cd-hit's "global sequence identity" calculated as: number of identical amino acids in alignment divided by the full length of the shorter sequence -G use global sequence identity, default 1 if set to 0, then use local sequence identity, calculated as : number of identical amino acids in alignment divided by the length of the alignment NOTE!!! don't use -G 0 unless you use alignment coverage controls see options -aL, -AL, -aS, -AS -b band_width of alignment, default 20 -M memory limit (in MB) for the program, default 800; 0 for unlimitted; -T number of threads, default 1; with 0, all CPUs will be used -n word_length, default 10, see user's guide for choosing it -l length of throw_away_sequences, default 10 -d length of description in .clstr file, default 20 if set to 0, it takes the fasta defline and stops at first space -s length difference cutoff, default 0.0 if set to 0.9, the shorter sequences need to be at least 90% length of the representative of the cluster -S length difference cutoff in amino acid, default 999999 if set to 60, the length difference between the shorter sequences and the representative of the cluster can not be bigger than 60 -aL alignment coverage for the longer sequence, default 0.0 if set to 0.9, the alignment must covers 90% of the sequence -AL alignment coverage control for the longer sequence, default 99999999 if set to 60, and the length of the sequence is 400, then the alignment must be >= 340 (400-60) residues -aS alignment coverage for the shorter sequence, default 0.0 if set to 0.9, the alignment must covers 90% of the sequence -AS alignment coverage control for the shorter sequence, default 99999999 if set to 60, and the length of the sequence is 400, then the alignment must be >= 340 (400-60) residues -A minimal alignment coverage control for the both sequences, default 0 alignment must cover >= this value for both sequences -uL maximum unmatched percentage for the longer sequence, default 1.0 if set to 0.1, the unmatched region (excluding leading and tailing gaps) must not be more than 10% of the sequence -uS maximum unmatched percentage for the shorter sequence, default 1.0 if set to 0.1, the unmatched region (excluding leading and tailing gaps) must not be more than 10% of the sequence -U maximum unmatched length, default 99999999 if set to 10, the unmatched region (excluding leading and tailing gaps) must not be more than 10 bases -B 1 or 0, default 0, by default, sequences are stored in RAM if set to 1, sequence are stored on hard drive it is recommended to use -B 1 for huge databases -p 1 or 0, default 0 if set to 1, print alignment overlap in .clstr file -g 1 or 0, default 0 by cd-hit's default algorithm, a sequence is clustered to the first cluster that meet the threshold (fast cluster). If set to 1, the program will cluster it into the most similar cluster that meet the threshold (accurate but slow mode) but either 1 or 0 won't change the representatives of final clusters -r 1 or 0, default 1, by default do both +/+ & +/- alignments if set to 0, only +/+ strand alignment -mask masking letters (e.g. -mask NX, to mask out both 'N' and 'X') -match matching score, default 2 (1 for T-U and N-N) -mismatch mismatching score, default -2 -gap gap opening score, default -6 -gap-ext gap extension score, default -1 -bak write backup cluster file (1 or 0, default 0) -h print this help Questions, bugs, contact Limin Fu at l2fu@ucsd.edu, or Weizhong Li at liwz@sdsc.edu For updated versions and information, please visit: http://cd-hit.org cd-hit web server is also available from http://cd-hit.org If you find cd-hit useful, please kindly cite: "Clustering of highly homologous sequences to reduce thesize of large protein database", Weizhong Li, Lukasz Jaroszewski & Adam Godzik. Bioinformatics, (2001) 17:282-283 "Cd-hit: a fast program for clustering and comparing large sets of protein or nucleotide sequences", Weizhong Li & Adam Godzik. Bioinformatics, (2006) 22:1658-1659 /data/results/tools/align/multiple/cd-hit-v4.6.1-2012-08-27/cd-hit-est -i baz -o baz.clust -c 0.95 -n 5 -S 0 -r 0 samtools view -h IonXpress_009_IMP1_4-5-6-7PeptideLinrary.aligned.bam > IonXpress_009_IMP1_4-5-6-7PeptideLinrary.aligned.sam reczko@n2:/data/images/proton/run149/t$ split -99999 ../IonXpress_011_IMP2_A4VRound4.aligned.sam reczko@n1:/data/images/proton/run149/n1$ split -99999 ../IonXpress_009_IMP1_4-5-6-7PeptideLinrary.aligned.sam reczko@n0:/data/images/proton/run149/n0$ split -99999 ../IonXpress_015_IMP3_Ab42Round2.sam reczko@n0:/data/images/proton/run149/n0$ for i in *ins > do > cat $i >> ins.fa > done mv n1/ins.fa 009_IMP1_4-5-6-7PeptideLinrary.ins.fa mv n0/ins.fa 015_IMP3_Ab42Round2.ins.fa mv t/ins.fa 011_IMP2_A4VRound4.ins.fa /data/results/tools/align/multiple/cd-hit-v4.6.1-2012-08-27/cd-hit-est -i 011_IMP2_A4VRound4.ins.fa -o 011_IMP2_A4VRound4.ins.fa.clust -c 0.95 -n 5 -S 0 -r 0 reczko@estia:/data/images/proton/run149$ /data/results/tools/align/multiple/cd-hit-v4.6.1-2012-08-27/cd-hit-est -i 011_IMP2_A4VRound4.ins.fa -o 011_IMP2_A4VRound4.ins.fa.clust -c 0.95 -n 5 -S 0 -r 0 -M 40000 &> 011_IMP2_A4VRound4.ins.log /data/results/tools/align/multiple/cd-hit-v4.6.1-2012-08-27/cd-hit-est -i 015_IMP3_Ab42Round2.ins.fa -o 015_IMP3_Ab42Round2.ins.fa.clust -c 0.95 -n 5 -S 0 -r 0 -M 40000 &> 015_IMP3_Ab42Round2.ins.log /data/results/tools/align/multiple/cd-hit-v4.6.1-2012-08-27/cd-hit-est -i 011_IMP2_A4VRound4.ins.fa -o 011_IMP2_A4VRound4.ins.fa.clust -c 0.95 -n 5 -S 0 -r 0 -M 40000 &> 011_IMP2_A4VRound4.ins.log /data/results/tools/align/multiple/cd-hit-v4.6.1-2012-08-27/cd-hit-est -i 009_IMP1_4-5-6-7PeptideLinrary.ins.fa -o 009_IMP1_4-5-6-7PeptideLinrary.ins.fa.clust -c 0.95 -n 5 -S 0 -r 0 -M 40000 &> 009_IMP1_4-5-6-7PeptideLinrary.ins.log echo "011_IMP2_A4VRound4.ins.fa.clust" | awk -f /data/results/tools/align/multiple/cd-hit-clust-sizes1.awk > foo echo "009_IMP1_4-5-6-7PeptideLinrary.ins.fa.clust" | awk -f /data/results/tools/align/multiple/cd-hit-clust-sizes1.awk > foo echo "015_IMP3_Ab42Round2.ins.fa.clust"| awk -f /data/results/tools/align/multiple/cd-hit-clust-sizes1.awk > foo awk -f merge-inserts1.awk insert_samples.txt > insert_samples.csv #@ reczko@estia:/data/images/proton/run149$ samtools flagstat IonXpress_009_IMP1_4-5-6-7PeptideLinrary.aligned.bam 8790890 in total 0 QC failure 0 duplicates 8790890 mapped (100.00%) 0 paired in sequencing 0 read1 0 read2 0 properly paired (-nan%) 0 with itself and mate mapped 0 singletons (-nan%) 0 with mate mapped to a different chr 0 with mate mapped to a different chr (mapQ>=5) wc insert_samples.csv 4407515 35260118 249555943 insert_samples.csv => (- 8790890 4407515) 4383375 without insert reczko@estia:/data/images/proton/run149$ awk -f /data/images/proton/run268/simulate-random-split1.awk 4384985 4385293 4385117 reczko@estia:/data/images/proton/run149$ awk -f /data/images/proton/run268/simulate-random-split-with-contamination1.awk 2193453 2193726 4389640 3 start-codons 6 S positions 12 N positions =3 * 2^6 * 4^12 =3*24*16777216 =2147483648