From 557d41a16502b6790cd0e91ab878a7b6423bb876 Mon Sep 17 00:00:00 2001
From: DavidAustinNix <david.nix@hci.utah.edu>
Date: Thu, 27 Sep 2018 11:15:41 -0600
Subject: [PATCH] Modifications to the TNRunner classes, updated VCFCallFreq to
 work off the file system, included log4j.properties in the jar to silence
 error messages.

---
 .classpath                                    |   1 +
 .../USeqDocumentation/cmdLnMenus.html         | 397 +++++++++++++-----
 .../mpileup/concordance/BamConcordance.java   |  48 ++-
 Source/edu/utah/seq/run/FastqDataset.java     |  16 +-
 Source/edu/utah/seq/run/TNRunner.java         |   2 +-
 Source/edu/utah/seq/run/TNSample.java         |  20 +-
 .../edu/utah/seq/vcf/AnnotatedVcfParser.java  |   3 +-
 Source/edu/utah/seq/vcf/VCFCallFrequency.java | 268 +++++++-----
 .../utah/seq/vcf/splice/VCFSpliceScanner.java |  12 +-
 Source/util/gen/Gzipper.java                  |  25 +-
 Source/util/gen/IO.java                       |  20 +
 build.properties                              |   2 +-
 build.xml                                     |   1 +
 log4j.properties                              |   9 +
 14 files changed, 596 insertions(+), 228 deletions(-)
 create mode 100644 log4j.properties
diff --git a/.classpath b/.classpath
index 091b6e30..b1cf0cc2 100644
--- a/.classpath
+++ b/.classpath
@@ -70,5 +70,6 @@
 	<classpathentry kind="lib" path="LibraryJars/json-20140107.jar"/>
 	<classpathentry kind="lib" path="LibraryJars/commons-codec-1.10.jar"/>
 	<classpathentry kind="lib" path="LibraryJars/sqljdbc41.jar"/>
+	<classpathentry combineaccessrules="false" kind="src" path="/Query"/>
 	<classpathentry kind="output" path="Classes"/>
 </classpath>
diff --git a/Documentation/USeqDocumentation/cmdLnMenus.html b/Documentation/USeqDocumentation/cmdLnMenus.html
index 742ddf9f..94dbd394 100644
--- a/Documentation/USeqDocumentation/cmdLnMenus.html
+++ b/Documentation/USeqDocumentation/cmdLnMenus.html
@@ -1,5 +1,5 @@
 
-			<html><head><title>USeq Command Line Menus 9.1.5
+			<html><head><title>USeq Command Line Menus 9.2.0
 			</title><style type=text/css>#rt{text-align:right; color: #000000; font-weight: bold}
 			#grBk {background-color: #CC9966;}TD {font-family: Verdana, Arial, Helvetica, sans-serif;
 			font-size:12;}H1 {color: #996633; font:arial; font-size:16;}H2 {color: #996633;
@@ -33,6 +33,8 @@ <H1>Command Line Menus</H1> <p>
 				
 					<a href="#ArupPipelineWrapper">ArupPipelineWrapper</a><br>
 				
+					<a href="#AvatarAssembler">AvatarAssembler</a><br>
+				
 					<a href="#BamConcordance">BamConcordance</a><br>
 				
 					<a href="#BamContextInspector">BamContextInspector</a><br>
@@ -51,6 +53,8 @@ <H1>Command Line Menus</H1> <p>
 				
 					<a href="#BedStats">BedStats</a><br>
 				
+					<a href="#BedTabix">BedTabix</a><br>
+				
 					<a href="#Bed2UCSCRefFlat">Bed2UCSCRefFlat</a><br>
 				
 					<a href="#BedRegionSplitter">BedRegionSplitter</a><br>
@@ -121,6 +125,8 @@ <H1>Command Line Menus</H1> <p>
 				
 					<a href="#FastqInterlacer">FastqInterlacer</a><br>
 				
+					<a href="#FastqRenamer">FastqRenamer</a><br>
+				
 					<a href="#FetchGenomicSequences">FetchGenomicSequences</a><br>
 				
 					<a href="#FindNeighboringGenes">FindNeighboringGenes</a><br>
@@ -213,6 +219,8 @@ <H1>Command Line Menus</H1> <p>
 				
 					<a href="#MpileupParser">MpileupParser</a><br>
 				
+					<a href="#MpileupRandomizer">MpileupRandomizer</a><br>
+				
 					<a href="#MultipleReplicaScanSeqs">MultipleReplicaScanSeqs</a><br>
 				
 					<a href="#MultiSampleVCFFilter">MultiSampleVCFFilter</a><br>
@@ -321,6 +329,8 @@ <H1>Command Line Menus</H1> <p>
 				
 					<a href="#Telescriptor">Telescriptor</a><br>
 				
+					<a href="#TNRunner">TNRunner</a><br>
+				
 					<a href="#UCSCBig2USeq">UCSCBig2USeq</a><br>
 				
 					<a href="#USeq2UCSCBig">USeq2UCSCBig</a><br>
@@ -331,20 +341,22 @@ <H1>Command Line Menus</H1> <p>
 				
 					<a href="#VCFBackgroundChecker">VCFBackgroundChecker</a><br>
 				
-					<a href="#VCFBackgroundCheckerGraphite">VCFBackgroundCheckerGraphite</a><br>
-				
 					<a href="#VCFBamAnnotator">VCFBamAnnotator</a><br>
 				
 					<a href="#VCF2Bed">VCF2Bed</a><br>
 				
 					<a href="#VCFAnnotator">VCFAnnotator</a><br>
 				
+					<a href="#VCFCallFrequency">VCFCallFrequency</a><br>
+				
 					<a href="#VCFComparator">VCFComparator</a><br>
 				
 					<a href="#VCFConsensus">VCFConsensus</a><br>
 				
 					<a href="#VCFMerger">VCFMerger</a><br>
 				
+					<a href="#VCFMpileupAnnotator">VCFMpileupAnnotator</a><br>
+				
 					<a href="#VCFVariantMaker">VCFVariantMaker</a><br>
 				
 					<a href="#VCFNoCallFilter">VCFNoCallFilter</a><br>
@@ -359,7 +371,7 @@ <H1>Command Line Menus</H1> <p>
 				
 					<a href="#VCFTabix">VCFTabix</a><br>
 				
-					<a href="#VCF2Tvs">VCF2Tvs</a><br>
+					<a href="#VCF2Tsv">VCF2Tsv</a><br>
 				
 					<a href="#Wig2Bar">Wig2Bar</a><br>
 				
@@ -636,7 +648,7 @@ <H1>Command Line Menus</H1> <p>
 </pre><br><p>
 				
 					<a name="AnnotatedVcfParser"><pre>**************************************************************************************
-**                              Annotated Vcf Parser  Feb 2018                      **
+**                             Annotated Vcf Parser  Sept 2018                      **
 **************************************************************************************
 Splits VCF files that have been annotated with SnpEff w/ dbNSFP and clinvar, plus the
 VCFBackgroundChecker and VCFSpliceScanner USeq apps into passing and failing records.
@@ -662,9 +674,10 @@ <H1>Command Line Menus</H1> <p>
 -e Comma delimited list of CLINSIG terms to select against.
 -i Comma delimited list of VCF ID keys to select for. If the VCF ID contains one or
        more, the record is passed regardless of other filters. The match is not exact.
- -o Only require, if set or present, SnpEff ANN or CLINSIG or Splice to be true to pass.
+-o Only require, if set or present, SnpEff ANN or CLINSIG or Splice to be true to pass.
        Defaults to require that all set pass.
 -r Verbose per record output.
+-y Path to a config txt file for setting the above.
 
 Example: java -jar pathToUSeq/Apps/AnnotatedVcfParser -v VCFFiles/ -s Parsed/
         -d 75 -m 0.05 -x 0.75 -j -p 0.02 -b 0.1 -g D5S,D3S,G5S,G3S -n 3.5 -a
@@ -707,11 +720,26 @@ <H1>Command Line Menus</H1> <p>
     ~/HCIAtlatl/data/Human/B37/human_g1k_v37_decoy.fasta -u CNV36B_unfiltered.bam -f 
     CNV36B_final.bam -v CNV36B_snvIndel.vcf
 
+**************************************************************************************
+</pre><br><p>
+				
+					<a name="AvatarAssembler"><pre>**************************************************************************************
+**                                TNRunner  September 2018                          **
+**************************************************************************************
+Tool for assembling fastq avatar datasets based on the results of three sql queries.
+Beta!
+
+Options:
+-i info.
+-d diagnosis.
+-g gender.
+-p path to Exp dir, e.g. /Repository/PersonData/2018/
+-r results file.
 **************************************************************************************
 </pre><br><p>
 				
 					<a name="BamConcordance"><pre>**************************************************************************************
-**                               Bam Concordance: Feb  2018                         **
+**                               Bam Concordance: Sept  2018                        **
 **************************************************************************************
 BC calculates sample level concordance based on uncommon homozygous SNVs found in bam
 files. Samples from the same person will show high similarity (>0.9). Run BC on
@@ -741,12 +769,13 @@ <H1>Command Line Menus</H1> <p>
 -m Minimum allele frequency to count a homozygous match, defaults to 0.9
 -q Minimum base quality, defaults to 20.
 -u Minimum mapping quality, defaults to 20.
+-j Write gzipped summary stats in json format to this file.
 -t Number of threads to use.  If not set, determines this based on the number of
       threads and memory available to the JVM so set the -Xmx value to the max.
 
-Example: java -Xmx120G -jar pathTo/USeq/Apps/BamConcordance -r ~/exomeTargets.bed
+Example: java -Xmx100G -jar pathTo/USeq/Apps/BamConcordance -r ~/exomeTargets.bed
       -s ~/Samtools1.3.1/bin/samtools -b ~/Patient7Bams -d 10 -a 0.9 -m 0.8 -f
-      ~/B37/human_g1k_v37.fasta -c ~/B37/b38ComSnps.bed.gz
+      ~/B37/human_g1k_v37.fasta -c ~/B37/b38ComSnps.bed.gz -j bc.json.gz 
 
 **************************************************************************************
 </pre><br><p>
@@ -794,29 +823,33 @@ <H1>Command Line Menus</H1> <p>
 </pre><br><p>
 				
 					<a name="BamBlaster"><pre>**************************************************************************************
-**                               Bam Blaster : June 2014                            **
+**                              Bam Blaster : May 2018                              **
 **************************************************************************************
 Injects SNVs and INDELs from a vcf file into bam alignments. These and their mates are
 extracted as fastq for realignment. For SNVs, only alignment bases that match the
 reference and have a CIGAR of M are modified. Not all alignments can be modified.
-Secondary/supplemental are skipped. Only one variant per alignment. Variants within a
+Secondary/supplemental/not proper are skipped. One var per alignment. Variants within
 read length distance of prior are ignored and saved to file for iterative processing.
 Be sure to normalize and decompose your vcf file (e.g.https://github.com/atks/vt).
-Use the BamMixer to add proportions of your realignments (e.g. 10%) with the
-unmodified.bams (e.g. 90%). Use the VCFVariantMaker to generate random vcf variants.
+INDELs first base must be reference. Use the BamMixer to add proportions of your
+realignments (e.g. 10%) with the unmodified.bams (e.g. 90%). Use the VCFVariantMaker
+to generate random vcf variants or pull a VCF from Clinvar/ Cosmic.
 
 Required:
 -b Path to a coordinate sorted bam file with index.
 -v Path to a trimmed, normalized, decomposed vcf variant file, zip/gz OK.
 -r Full path to a directory to save the results.
+-s Max size INDEL, defaults to 50
+-d Min alignment depth, defaults to 25
+
 Example: java -Xmx10G -jar pathTo/USeq/Apps/BamBlaster -b ~/BMData/na12878.bam
-    -r ~/BMData/BB1 -v ~/BMData/clinvar.pathogenic.SnvIndel.vcf.gz 
+    -r ~/BMData/BB0 -v ~/BMData/clinvar.pathogenic.SnvIndel.vcf.gz 
 
 **************************************************************************************
 </pre><br><p>
 				
 					<a name="BamMixer"><pre>**************************************************************************************
-**                               Bam Mixer : June 2014                              **
+**                               Bam Mixer : April 2018                             **
 **************************************************************************************
 Combines bam alignment files in different fractions to simulate multiple variant
 frequencies. Run BamBlaster first.
@@ -917,6 +950,25 @@ <H1>Command Line Menus</H1> <p>
 
 Example: java -Xmx1500M -jar pathToUSeq/Apps/BedStats -b /Res/ex1.bed.gz -s 9 -t 10
 
+**************************************************************************************
+</pre><br><p>
+				
+					<a name="BedTabix"><pre>**************************************************************************************
+**                                BedTabix: Jan 2013                                **
+**************************************************************************************
+Converts bed files to a SAMTools compressed bed tabix format. Recursive.
+
+Required Options:
+-v Full path file or directory containing xxx.bed(.gz/.zip OK) file(s). Recursive!
+-t Full path tabix directory containing the compiled bgzip and tabix executables. See
+      http://sourceforge.net/projects/samtools/files/tabix/
+-f Force overwriting of existing indexed bed files, defaults to skipping.
+-d Do not delete non gzipped bed files after successful indexing, defaults to deleting.
+-e Only print error messages.
+
+Example: java -jar pathToUSeq/Apps/BedTabix -v /VarScan2/BEDFiles/
+     -t /Samtools/Tabix/tabix-0.2.6/ 
+
 **************************************************************************************
 </pre><br><p>
 				
@@ -1057,7 +1109,7 @@ <H1>Command Line Menus</H1> <p>
 </pre><br><p>
 				
 					<a name="BisStat"><pre>**************************************************************************************
-**                                 BisStat: May 2014                                **
+**                                 BisStat: June 2018                               **
 **************************************************************************************
 Takes PointData from converted and non-converted C bisulfite sequencing data parsed
 using the NovoalignBisulfiteParser and generates several xxCxx context statistics and
@@ -1829,19 +1881,25 @@ <H1>Command Line Menus</H1> <p>
 </pre><br><p>
 				
 					<a name="FastqBarcodeTagger"><pre>**************************************************************************************
-**                           Fastq Barcode Tagger: Jan 2017                         **
+**                          Fastq Barcode Tagger: August 2018                       **
 **************************************************************************************
-Takes three fastq files (paired end reads and a third containing barcodes), appends
-the barcode and quality to the fastq header, and writes out the modified records. 
+Takes 2 or 3 fastq files (paired end reads and possibly a third containing unique 
+molecular barcodes/ indexes), appends the barcode and quality to the fastq header, and
+writes out the modified records. For IDT inline 2 fastq UMI data sets, the barcode is
+parsed from the beginning of each fastq. Be sure to clip 5Ns from the 3' end when
+adapter trimming.
 
 Options:
 -f First fastq file, .gz/.zip OK.
 -s Second fastq file, .gz/.zip OK.
--b Barcode fastq file, .gz/.zip OK.
--i (Optional) Write interlaced fastq to stdout for direct piping to other apps
--r (Optional) Directory to save the modified fastqs, defaults to the parent of -f
--l (Optional) Max length of barcode, defaults to all. Use to trim 3' end.
--a (Optional) Append the line number to the read name to uniquify.
+-b Barcode fastq file, .gz/.zip OK, or set -e
+-e Parse barcodes from the first 3bp of each read and combine the two 3mers into a
+      6mer barcode. 5bp are trimmed from the ends of each read to remove the UMI and
+      2bp constant seq as well as an potential read through. IDT's current strategy.
+-i Write interlaced fastq to stdout for direct piping to other apps
+-r Directory to save the modified fastqs, defaults to the parent of -f
+-l Max length of barcode, defaults to all. Use to trim 3' end.
+-a Append the line number to the read name to uniquify.
 
 Example: java -Xmx1G -jar pathToUSeq/Apps/FastqBarcodeTagger -f lob_1.fastq.gz
      -s lob_2.fastq.gz -b lob_barcode.fastq.gz -i | bwa mem -p /ref/hg19.fa 
@@ -1861,6 +1919,22 @@ <H1>Command Line Menus</H1> <p>
 Example: java -Xmx1G -jar pathToUSeq/Apps/FastqInterlacer -f lob_1.fastq.gz
      -s lob_2.fastq.gz | cutadapt | bwa | samblaster .... 
 
+**************************************************************************************
+</pre><br><p>
+				
+					<a name="FastqRenamer"><pre>**************************************************************************************
+**                            Fastq Renamer: April 2018                             **
+**************************************************************************************
+Takes paired fastq files and replaces the header with the record count. 
+
+Options:
+-f First fastq file, .gz/.zip OK.
+-s Second fastq file, .gz/.zip OK.
+-d Path to a directory for saving the modified fastq files.
+
+Example: java -Xmx1G -jar pathToUSeq/Apps/FastqRenamer -f lob_1.fastq.gz
+     -s lob_2.fastq.gz -d UniquifiedFastq/ 
+
 **************************************************************************************
 </pre><br><p>
 				
@@ -2130,26 +2204,28 @@ <H1>Command Line Menus</H1> <p>
 </pre><br><p>
 				
 					<a name="GatkRunner"><pre>**************************************************************************************
-**                               Gatk Runner: Feb 2018                              **
+**                               Gatk Runner: March 2018                            **
 **************************************************************************************
 Takes a bed file of target regions, splits it by the number of threads, writes out
-each, executes the GATK Gatktype caller, and merges the results. 
+each, executes the GATK Gatktype caller, and merges the results. Set the -Xmx to the
+maximum available on the machine to enable correct cpu thread usage.
 
 Options:
 -r A regions bed file (chr, start, stop,...) to intersect, see
        http://genome.ucsc.edu/FAQ/FAQformat#format1 , gz/zip OK.
 -s Path to a directory for saving the results.
--t Number concurrent threads to run, will need > 4G RAM each.
+-t Number concurrent thread override. Sets itself based on the memory and cpus 
+     available to the JVM.
 -c GATK command to execute, see the example below, modify to match your enviroment.
      Most resources require full paths. Don't set -o or -L
 -l Use lowercased l for Lofreq compatability.
 -b Add a -bamout argument and merge bam chunks.
 
-Example: java -Xmx24G -jar pathToUSeq/Apps/GatkRunner -b -r /SS/targets.bed -t 8 -s
+Example: java -Xmx24G -jar pathToUSeq/Apps/GatkRunner -b -r /SS/targets.bed -s
      /SS/HC/ -c 'java -Xmx4G -jar /SS/GenomeAnalysisTK.jar -T MuTect2 
     -R /SS/human_g1k_v37.fasta --dbsnp /SS/dbsnp_138.b37.vcf 
-    --cosmic /SS/v76_GRCh37_CosmicCodingMuts.vcf.gz -I:tumor /SS/sarc.bam -I:normal 
-    /SS/norm.bam'
+    --cosmic /SS/v76_GRCh37_CosmicCodingMuts.vcf.gz -I:tumor /SS/sar.bam -I:normal 
+    /SS/normal.bam'
 
 **************************************************************************************
 </pre><br><p>
@@ -2403,7 +2479,7 @@ <H1>Command Line Menus</H1> <p>
 </pre><br><p>
 				
 					<a name="LofreqVCFParser"><pre>**************************************************************************************
-**                             Lofreq VCF Parser: March 2017                        **
+**                             Lofreq VCF Parser: May 2018                          **
 **************************************************************************************
 Parses Lofreq vcf files with options for filtering for minimum QUAL, modifying the
 FILTER field, removing non SNVs, and appending FORMAT info for downstream merging.
@@ -2416,13 +2492,14 @@ <H1>Command Line Menus</H1> <p>
 -m Minimum QUAL score, defaults to 0
 -d Minimum DP read depth, defaults to 0
 -t Minimum AF allele freq, defaults to 0
+-r Minimum Alt count, defaults to 0
 -i Remove non SNV records
 -f Replace the FILTER field with '.'
 -a Append FORMAT NORMAL TUMOR to #CHROM line and add empty columns to records
 -n Mark variants failing thresholds FAIL instead of not printing
 
 Example: java -jar pathToUSeq/Apps/LofreqVCFParser -v VCFFiles/ -m 32 -i -f -a
-      -s FilteredLofreqVcfs/ 
+      -s FilteredLofreqVcfs/ -r 3 
 
 **************************************************************************************
 </pre><br><p>
@@ -2732,7 +2809,7 @@ <H1>Command Line Menus</H1> <p>
 </pre><br><p>
 				
 					<a name="MergeUCSCGeneTable"><pre>**************************************************************************************
-**                           Merge UCSC Gene Table: Feb  2013                       **
+**                           Merge UCSC Gene Table: Aug  2018                       **
 **************************************************************************************
 Merges transcript models that share the same gene name (in column 0). Maximizes exons,
 minimizes introns. Assumes interbase coordinates.
@@ -2887,6 +2964,25 @@ <H1>Command Line Menus</H1> <p>
       H_sapiens_Feb_2009 -e 0.1 -w 25
 
 **************************************************************************************
+</pre><br><p>
+				
+					<a name="MpileupRandomizer"><pre>**************************************************************************************
+**                             Mpileup Randomizer: May 2018                         **
+**************************************************************************************
+Upon finding a gap in the coverage, the sample order is randomized and maintained. Use
+this app to 'de-identify' a multi sample mpileup file while maintaining INDEL blocks.
+
+Required Options:
+-m Path to a Samtools mpileup file (gz/zip OK).
+
+Default Options:
+-r Minimum read depth to pass a sample, default 10
+-s Minimum number of samples that must pass to save line, default 3
+-g Minimum gap, defaults to 125
+
+Example: java -Xmx4G -jar pathToUSeq/Apps/MpileupRandomizer -m normExo.mpileup.gz
+    -r 20 -s 4 
+**************************************************************************************
 </pre><br><p>
 				
 					<a name="MultipleReplicaScanSeqs"><pre>**************************************************************************************
@@ -2982,11 +3078,11 @@ <H1>Command Line Menus</H1> <p>
 </pre><br><p>
 				
 					<a name="MutectVCFParser"><pre>**************************************************************************************
-**                               Mutect VCF Parser: March 2018                      **
+**                               Mutect VCF Parser: May 2018                        **
 **************************************************************************************
 Parses Mutect2 VCF files, filtering for read depth, allele frequency diff ratio, etc.
 Inserts AF and DP into for the tumor sample into the INFO field. Changes the sample
-order to Normal and Tumor and updates the #CHROM line.
+order to Normal and Tumor and updates the #CHROM line. Replaces the QUAL with TLOD.
 
 Options:
 -v Full path file or directory containing xxx.vcf(.gz/.zip OK) file(s).
@@ -2994,14 +3090,16 @@ <H1>Command Line Menus</H1> <p>
 -t Minimum tumor allele frequency (AF), defaults to 0.
 -n Maximum normal AF, defaults to 1.
 -u Minimum tumor alignment depth, defaults to 0.
+-a Minimum tumor alt count, defaults to 0.
 -o Minimum normal alignment depth, defaults to 0.
 -d Minimum T-N AF difference, defaults to 0.
 -r Minimum T/N AF ratio, defaults to 0.
+-t Minimum TLOD score, defaults to 0.
 -p Remove non PASS filter field records.
 -s Print spreadsheet variant summary.
 
 Example: java -jar pathToUSeq/Apps/MutectVCFParser -v /VCFFiles/ -t 0.05 -n 0.5 -u 100
-        -o 20 -d 0.05 -r 2
+        -o 20 -d 0.05 -r 2 -a 3 
 
 **************************************************************************************
 </pre><br><p>
@@ -3437,7 +3535,7 @@ <H1>Command Line Menus</H1> <p>
 </pre><br><p>
 				
 					<a name="QueryIndexer"><pre>**************************************************************************************
-**                                Query Indexer: Feb 2017                           **
+**                                Query Indexer: May 2018                           **
 **************************************************************************************
 Builds index files for Query by recursing through a data directory looking for bgzip
 compressed and tabix indexed genomic data files (e.g. vcf, bed, maf, and custom).
@@ -3457,7 +3555,8 @@ <H1>Command Line Menus</H1> <p>
 -d A data directory containing bgzipped and tabix indexed data files. Known file types
      include xxx.vcf.gz, xxx.bed.gz, xxx.bedGraph.gz, and xxx.maf.txt.gz. Others will
      be parsed using info from the xxx.gz.tbi index. Be sure to normalize and
-     decompose_blocksub all VCF records, see http://genome.sph.umich.edu/wiki/Vt
+     decompose_blocksub all VCF records, see http://genome.sph.umich.edu/wiki/Vt.
+     Files may be hard linked but not soft.
 -t Full path directory containing the compiled bgzip and tabix executables. See
      https://github.com/samtools/htslib
 -i A directory in which to save the index files
@@ -3750,16 +3849,20 @@ <H1>Command Line Menus</H1> <p>
 </pre><br><p>
 				
 					<a name="Sam2Fastq"><pre>**************************************************************************************
-**                                 Sam 2 Fastq: July 2016                           **
+**                                Sam 2 Fastq: May 2018                              **
 **************************************************************************************
-Given a queryname sorted bam/sam alignment file, writes out fastq data for paired and 
-unpaired alignemnts. Any non primary, secondary, supplemental or vendor failing QC
+Given a query name sorted alignment file, S2F writes out fastq data for paired and 
+unpaired alignemnts. Any non-primary, secondary, or supplemental 
 alignments are written to a failed sam file.  This app doesn't have the memory leak
-found in Picard, writes gzipped fastq, and error checks the reads.
+found in Picard, writes gzipped fastq, and error checks the reads. Provide an
+unfiltered fastq-bam to use in retrieving missing mates. S2F will remove pre and
+post naming info from the BamBlaster restoring the original fragment names.
 
 Options:
--s Path to a directory for saving parsed data..
+-s Path to a directory for saving parsed data.
 -a Path to a query name sorted bam/sam alignment file. 
+-u (Optional) Path to a query name sorted unfiltered bam/sam alignment file for use
+      in fetching missing mates of the first bam. Convert fastq to bam, then qn sort.
 
 Example: java -Xmx2G -jar pathTo/USeq/Apps/Sam2Fastq -a myQNSorted.bam -s S2F/
 
@@ -3785,7 +3888,7 @@ <H1>Command Line Menus</H1> <p>
 </pre><br><p>
 				
 					<a name="Sam2USeq"><pre>**************************************************************************************
-**                                  Sam 2 USeq : Jan 2018                           **
+**                                  Sam 2 USeq : Aug 2018                           **
 **************************************************************************************
 Generates per base read depth stair-step graph files for genome browser visualization.
 By default, values are scaled per million mapped reads with no score thresholding. Can
@@ -3825,6 +3928,7 @@ <H1>Command Line Menus</H1> <p>
 -j Write summary stats in json format to this file, requires -b and -c.
 -y Include CIGAR Ns in read coverage, defaults to just M values.
 -z Flip the strand of the 2nd of pair alignments.
+-w Path to a config txt file for setting the above.
 
 Example: java -Xmx1500M -jar pathTo/USeq/Apps/Sam2USeq -f /Data/SamFiles/ -r
      -v H_sapiens_Feb_2009 -b ccdsExons.bed.gz 
@@ -3833,7 +3937,7 @@ <H1>Command Line Menus</H1> <p>
 </pre><br><p>
 				
 					<a name="SamAlignmentExtractor"><pre>**************************************************************************************
-**                         Sam Alignment Extractor: Jan 2018                        **
+**                        Sam Alignment Extractor: April 2018                       **
 **************************************************************************************
 Splits an alignment file into those that pass or fail thresholds and intersects
 regions of interest. Calculates a variety of QC statistics.
@@ -3855,6 +3959,7 @@ <H1>Command Line Menus</H1> <p>
 -x Save secondary, supplemental, and non primary alignments, that pass the thresholds
        defaults to fail.
 -w Skip writing failing alignments. Speeds up processing but kills the stats.
+-u Write unmapped alignments to the pass file, defaults to writing to fail.
 
 Example: java -Xmx4G -jar pathTo/USeq/Apps/SamAlignmentExtractor -q 20 -a 0.75 -d -b
       /Data/raw.bwaMem.bam -r /Data/targetsPad25bp.bed.gz -s/Data/SAE/ 
@@ -4506,6 +4611,57 @@ <H1>Command Line Menus</H1> <p>
 Example: java -Xmx4G -jar pathTo/USeq/Apps/Telescriptor -u hg19EnsTrans.ucsc -t Bam/T
        -c Bam/C -s GV_MOR 
 
+**************************************************************************************
+</pre><br><p>
+				
+					<a name="TNRunner"><pre>**************************************************************************************
+**                                TNRunner  September 2018                          **
+**************************************************************************************
+TNRunner is designed to execute several dockerized snakmake workflows on human tumor
+normal datasets via a slurm cluster.  Based on the availability of fastq, Hg38
+alignments are run, somatic and germline variants are called, and concordance measured
+between sample bams. To execute TNRunner, create the following directory structure and
+link or copy in the corresponding paired end Illumina gzipped fastq files.
+MyPatientSampleDatasets
+   MyPatient1
+      Fastq
+         NormalExome
+         TumorExome
+         TumorTranscriptome
+   MyPatient2
+      Fastq
+         TumorExome
+         TumorTranscriptome
+   MyPatient3....
+The Fastq directory and sub directories must match this naming. Only include those
+for which you have fastq.  Change the MyXXX to something relevant. TNRunner is
+stateless so as more Fastq becomes available or issues are addressed, relaunch the
+app. This won't effect running or queued slurm jobs. Relaunch periodically to assess
+the current processing status and queue additional tasks. Download the latest
+workflows from https://github.com/HuntsmanCancerInstitute/Workflows/tree/master/Hg38 
+and the matching resource bundle from
+https://hci-bio-app.hci.utah.edu/gnomex/gnomexFlex.jsp?analysisNumber=A5578
+
+Options:
+-p Directory containing one or more patient data directories to process.
+-e Workflow docs for launching exome alignments.
+-t Workflow docs for launching transcriptome alignments.
+-c Workflow docs for launching somatic variant calling.
+-a Workflow docs for launching variant annotation.
+-b Workflow docs for launching bam concordance.
+-j Workflow docs for launching joint genotyping.
+-g Germline AnnotatedVcfParser options, defaults to '-d 10 -m 0.2 -x 1 -p 0.01 -g 
+      D5S,D3S -n 5 -a HIGH -c Pathogenic,Likely_pathogenic -o -e Benign,Likely_benign'
+-s Somatic AnnotatedVcfParser options, defaults to '-d 20 -f'
+-f Force a restart of all uncompleted jobs.
+-q Quite output.
+
+Example: java -jar pathToUSeq/Apps/TNRunner -p AvatarPatients -e 
+     ~/Hg38/ExomeAlignQC/WorkflowDocs/ -c ~/Hg38/SomExoCaller/WorkflowDocs/ -a 
+     ~/Hg38/Annotator/WorkflowDocs/ -b ~/Hg38/BamConcordance/WorkflowDocs/ -j
+     ~/Hg38/JointGenotyping/WorkflowDocs/ -t ~/Hg38/TranscriptomeAlignQC/WorkflowDocs/ 
+     -s '-d 30 -f' 
+
 **************************************************************************************
 </pre><br><p>
 				
@@ -4595,7 +4751,7 @@ <H1>Command Line Menus</H1> <p>
 </pre><br><p>
 				
 					<a name="VCFBackgroundChecker"><pre>**************************************************************************************
-**                         VCF Background Checker : Dec 2017                        **
+**                         VCF Background Checker : May 2018                        **
 **************************************************************************************
 VBC calculates non-reference allele frequencies (AF) from a background multi-sample 
 mpileup file over each vcf record. It then calculates a z-score for the vcf AF and 
@@ -4610,13 +4766,14 @@ <H1>Command Line Menus</H1> <p>
 -m Path to a bgzip compressed and tabix indexed multi-sample mpileup file. e.g.:
       1) Mpileup: 'echo "#SampleOrder: "$(ls *bam) > bkg.mpileup; samtools mpileup
              -B -q 13 -d 1000000 -f $fastaIndex -l $bedFile *.bam >> bkg.mpileup'
-      2) Bgzip: 'tabix-0.2.6/bgzip bkg.mpileup'
-         Tabix: 'tabix-0.2.6/tabix -s 1 -b 2 -e 2 bkg.mpileup.gz'
+      2) (Optional) MpileupRandomizer: java -jar -Xmx10G ~/USeqApps/MpileupRandomizer
+             -r 20 -s 3 -m bkg.mpileup
+      3) Bgzip: 'tabix-0.2.6/bgzip bkg.mpileup_DP20MS3.txt'
+         Tabix: 'tabix-0.2.6/tabix -s 1 -b 2 -e 2 bkg.mpileup_DP20MS3.txt.gz'
 -s Path to directory in which to save the modified vcf file(s)
 
 Optional:
--b BP padding, defaults to 0
--z Minimum vcf z-score, defaults to 0, no filtering
+-z Minimum vcf z-score, defaults to 0, no filtering. Unscored vars are kept.
 -q Minimum mpileup sample bp quality, defaults to 20
 -c Minimum mpileup sample read coverge, defaults to 20
 -f Maximum mpileup sample AF, defaults to 0.3
@@ -4627,48 +4784,7 @@ <H1>Command Line Menus</H1> <p>
 -t Number of threads to use, defaults to all
 
 Example: java -Xmx4G -jar pathTo/USeq/Apps/VCFBackgroundChecker -v SomaticVcfs/ -z 3
--m bkg.mpileup.gz -s BkgFiltVcfs/ -b 1 -q 13 -u 
-
-**************************************************************************************
-</pre><br><p>
-				
-					<a name="VCFBackgroundCheckerGraphite"><pre>**************************************************************************************
-**                         VCF Background Checker : April 2016                      **
-**************************************************************************************
-VBC calculates non-ref allele frequencies (AF) from a normal background multi-sample 
-mpileup file over each snv or using normal bams and graphite for indel and multi alts.
-It then calculates a z-score for the vcf AF and appends it to the INFO field. Z-scores
-< ~4 are indicative of non reference bps in the background samples. Note, VBC requires
-an AF tag in the INFO field of each record. 
-
-Required:
--v Path to a xxx.vcf(.gz/.zip OK) file or directory containing such.
--m Path to a bgzip compressed and tabix indexed multi-sample mpileup file. e.g.:
-      1) Mpileup: 'echo "#SampleOrder: "$(ls *bam) > bkg.mpileup; samtools mpileup
-             -B -q 13 -d 1000000 -f $fastaIndex -l $bedFile *.bam >> bkg.mpileup'
-      2) Bgzip: 'tabix-0.2.6/bgzip bkg.mpileup'
-         Tabix: 'tabix-0.2.6/tabix -s 1 -b 2 -e 2 bkg.mpileup.gz'
--g Path to graphite executable, see https://github.com/dillonl/graphite.
--n Directory containing normal background indexed bams for graphite AF estimation.
--i Indexed fasta file used to align the normal bams.
--s Path to a directory to save the modified vcf file(s)
-
-Optional:
--z Minimum vcf z-score, defaults to 0, no filtering
--q Minimum mpileup sample bp quality, defaults to 20
--c Minimum sample read coverge, defaults to 20
--f Maximum sample AF, defaults to 0.3
--a Minimum # samples for z-score calculation, defaults to 3
--e Exclude vcf records that could not be z-scored
--r Replace QUAL value with z-score. Un scored vars will be assigned 0
--d Print verbose debugging output
--t Number of threads to use, defaults to all
--o Additional bash shell commands before executing graphite,e.g. 'module load gcc/4.9'
--b BP intersection radius for pushing SNVs to graphite for AF estimation, default 125
-
-Example: java -Xmx4G -jar pathTo/USeq/Apps/VCFBackgroundChecker -v SomaticVcfs/ -z 3
--m bkg.mpileup.gz -s BkgFiltVcfs/ -b 50 -q 13 -e -r -g ~/Graphite/bin/graphite -n
-~/BkgrndNormalBams/ -o 'module load gcc/4.9.2' -i ~/B37/b37.fa 
+-m bkg.mpileup.gz -s BkgFiltVcfs/ -q 13 -u 
 
 **************************************************************************************
 </pre><br><p>
@@ -4764,6 +4880,49 @@ <H1>Command Line Menus</H1> <p>
 Example: java -Xmx20G -jar pathTo/USeq/Apps/VCFAnnotator -v 9908R.vcf 
       -o 9908_ann.vcf.gz 
 
+**************************************************************************************
+</pre><br><p>
+				
+					<a name="VCFCallFrequency"><pre>**************************************************************************************
+**                            VCF Call Frequency: Sept 2018                         **
+**************************************************************************************
+Calculates a vcf call frequency for each variant when pointed at a genomic Query
+service (https://github.com/HuntsmanCancerInstitute/Query) or the Data and Index
+directories the service is accessing. CallFreq's are calculated 
+by first counting the number of exact vcf matches present and dividing it by the
+number of intersecting bed files. Use this to flag variants with high call rates that
+are potential false positives. Some are not, e.g. BRAF V600E occurs in 58% of cancers
+with a BRAF mutation. So treate the call freq as an annotation requiring context level
+interpretation. Use the file filter to limit which files are included in the call freq
+calculations. Only file paths containing the file filter will be included. Be sure to
+place both the sample vcf and associated callable region bed files in the same Query
+index folder path as defined by -f. 
+
+Required Options:
+-v Full path to a file or directory containing xxx.vcf(.gz/.zip OK) file(s)
+-s Directory to save the annotated vcf files
+-f Query service file filter, e.g. /B37/Somatic/Avatar/
+-c Config txt file containing two tab delimited columns with host, queryUrl, realm, 
+     userName, password, and (optionally) fileFilter and or maxCallFreq. 'chmod 600'
+     the file! e.g.: 
+     host hci-clingen1.hci.utah.edu
+     queryUrl http://hci-clingen1.hci.utah.edu:8080/Query/
+     realm QueryAPI
+     userName FColins
+     password g0QueryAP1
+-i (Alternative to -c), provide a path to the Index directory generated by the USeq 
+     QueryIndexer app.
+-d (Alternative to -c), provide a path to the Data directory used in creating the Index.
+
+Options:
+-m Maximum call freq, defaults to 1, before appending 'CallFreq' to the FILTER field.
+-x Remove failing max call freq records, not recommended.
+-b Minimum bed call count before applying a max call freq filter, defaults to 8.
+-e Print verbose debugging output.
+
+Example: java -jar pathToUSeq/Apps/VCFCallFrequency -v Vcf/ -s CFVcfs -f 
+    /B37/Somatic/Avatar/ -m 0.05 -c vcfCFConfig.txt 
+
 **************************************************************************************
 </pre><br><p>
 				
@@ -4843,6 +5002,34 @@ <H1>Command Line Menus</H1> <p>
 
 Example: java -Xmx4G -jar pathTo/USeq/Apps/VCFMerger -v /CancerSamples/
 
+**************************************************************************************
+</pre><br><p>
+				
+					<a name="VCFMpileupAnnotator"><pre>**************************************************************************************
+**                          VCF Mpileup Annotator : April 2018                      **
+**************************************************************************************
+VMA estimates the AF and DP of a vcf record from a single sample mpileup file.  It 
+replaces the AF or DP INFO values in the vcf records if present. For INDELs, the
+region effected is scanned and the maximum AF and DP assigned. Provide the max memory
+available to the app to maximize cpu usage.
+
+Required:
+-v Path to a xxx.vcf(.gz/.zip OK) file.
+-m Path to a bgzip compressed and tabix indexed single sample mpileup file. e.g.:
+      1) Mpileup: 'samtools mpileup -B -q 1 -d 1000000 -f $fastaIndex -l $bedFile
+      $indexedBamFile > bam.mpileup'
+      2) Bgzip: 'tabix-0.2.6/bgzip bam.mpileup'
+         Tabix: 'tabix-0.2.6/tabix -s 1 -b 2 -e 2 bam.mpileup.gz'
+-o Path to a xxx.vcf.gz file to save the modified vcf records.
+
+Optional:
+-q Minimum mpileup sample bp quality, defaults to 0
+-d Print verbose debugging output.
+-t Number of threads to use, defaults to all/ 5GB.
+
+Example: java -Xmx64G -jar pathTo/USeq/Apps/VCFMpileupAnnotator -v spikes.vcf -q 13
+-m bam.mpileup.gz -o spikes.mod.vcf.gz
+
 **************************************************************************************
 </pre><br><p>
 				
@@ -4979,13 +5166,13 @@ <H1>Command Line Menus</H1> <p>
 </pre><br><p>
 				
 					<a name="VCFSpliceScanner"><pre>**************************************************************************************
-**                             VCF Splice Scanner : Dec 2017                        **
+**                            VCF Splice Scanner : Sept 2018                        **
 **************************************************************************************
 Scores variants for changes in splicing using the MaxEntScan algorithms. See Yeo and
 Burge 2004, http://www.ncbi.nlm.nih.gov/pubmed/15285897 for details. Known splice
 acceptors and donors are scored for loss of a junction.  Exonic, intronic, and splice
 bases are scanned for novel junctions in a window around each variant. See the vcf
-INFO header for a description of the output. Use this information to identify likely
+INFO header for a description of the output. Use this information to identify 
 snv and indel variants that may effect splicing.
 
 Required Options:
@@ -5037,10 +5224,10 @@ <H1>Command Line Menus</H1> <p>
 **************************************************************************************
 </pre><br><p>
 				
-					<a name="VCF2Tvs"><pre>**************************************************************************************
-**                                 VCF 2 Tvs: March 2018                            **
+					<a name="VCF2Tsv"><pre>**************************************************************************************
+**                                 VCF 2 Tsv: March 2018                            **
 **************************************************************************************
-Converts vcf files' SNVs and INDELs to tvs Illumina CE/CA format. 
+Converts vcf files' SNVs and INDELs to tsv Illumina CE/CA format. 
 
 Required Options:
 -v Full path file or directory containing xxx.vcf(.gz/.zip OK) file(s).
@@ -5050,7 +5237,7 @@ <H1>Command Line Menus</H1> <p>
 -b Max frac BKAFs >= AF, defaults to 1, no threshold.
 -i ID column string forcing export. Case insensitive.
 
-Example: java -jar pathToUSeq/Apps/VCF2Tvs -v VCFss/ -q 3 -z -b 0.2 -i Foundation 
+Example: java -jar pathToUSeq/Apps/VCF2Tsv -v VCFss/ -q 3 -z -b 0.2 -i Foundation 
 
 **************************************************************************************
 </pre><br><p>
@@ -5187,18 +5374,24 @@ <H1>Command Line Menus</H1> <p>
 </pre><br><p>
 				
 					<a name="StrelkaVCFParser"><pre>**************************************************************************************
-**                             Strelka VCF Parser: Feb 2018                         **
+**                            Strelka VCF Parser: June 2018                         **
 **************************************************************************************
 Parses Strelka VCF INDEL and SNV files, replacing the QUAl score with the QSI or QSS
-score. Also filters for read depth, T/N alt allelic ratio and diff,
+score. Also filters for read depth, T/N alt allelic ratio and diff, ref/alt with '.',
 and tumor and normal alt allelic ratios. Lastly, it inserts the tumor DP and AF info.
+For somatic exome datasets sequenced at >100X unique observation read depth, try the 
+tier filtering to select for lists with approx 1%, 3-5%, and 9-15% FDR. Follow the
+example.
 
 Required Params:
 -v Full path file or directory containing xxx.vcf(.gz/.zip OK) file(s).
 -m Minimum QSI or QSS score, defaults to 0.
+-e Apply a tuned 100X exome QSI/S stringency tier: 1 (9-15%FDR), 2 (4-6%FDR),
+         3 (1-2%FDR), defaults to 0, no tiered filtiering (37-63%FDR).
 -t Minimum tumor allele frequency (AF), defaults to 0.
 -n Maximum normal AF, defaults to 1.
 -u Minimum tumor alignment depth, defaults to 0.
+-a Minimum tumor alt count, defaults to 0.
 -o Minimum normal alignment depth, defaults to 0.
 -d Minimum T-N AF difference, defaults to 0.
 -r Minimum T/N AF ratio, defaults to 0.
@@ -5206,8 +5399,8 @@ <H1>Command Line Menus</H1> <p>
 -s Print spreadsheet variant summary.
 -f Directory in which to save the parsed files, defaults to the parent dir of the vcfs.
 
-Example: java -jar pathToUSeq/Apps/StrelkaVCFParser -v /VCFFiles/ -m 32 -t 0.05
-        -n 0.5 -u 100 -o 20 -d 0.05 -r 2 -f ParsedVcfs 
+Example: java -jar pathToUSeq/Apps/StrelkaVCFParser -v /VCFFiles/ -t 0.03 -n 0.6 
+-u 30 -o 10 -a 3 -d 0.03 -r 2 -e 1 
 
 **************************************************************************************
 </pre><br><p>
diff --git a/Source/edu/utah/seq/parsers/mpileup/concordance/BamConcordance.java b/Source/edu/utah/seq/parsers/mpileup/concordance/BamConcordance.java
index d7569c1e..cc424267 100644
--- a/Source/edu/utah/seq/parsers/mpileup/concordance/BamConcordance.java
+++ b/Source/edu/utah/seq/parsers/mpileup/concordance/BamConcordance.java
@@ -29,6 +29,7 @@ public class BamConcordance {
 	private double minAFForHis = 0.05;
 	private int minBaseQuality = 20;
 	private int minMappingQuality = 20;
+	private File jsonOutputFile = null;
 
 	//internal fields
 	private File tempDirectory;
@@ -39,6 +40,8 @@ public class BamConcordance {
 	private Similarity[] similarities;
 	private String[] sampleNames = null;
 	private String bamNames = null;
+	private String[] similarityForJson = null;
+	private String[] genderForJson = null;
 	
 	//internal fields
 	ConcordanceChunk[] runners;
@@ -61,6 +64,8 @@ public BamConcordance(String[] args){
 
 			printHist();
 			
+			saveJson();
+			
 			//finish and calc run time
 			double diffTime = ((double)(System.currentTimeMillis() -startTime))/60000;
 			System.out.println("\nDone! "+Math.round(diffTime)+" Min\n");
@@ -154,20 +159,26 @@ public void printThresholds(){
 	}
 
 	public void printStats(){
+		similarityForJson = new String[similarities.length];
 		//calculate max match and sort
 		for (int i=0; i< similarities.length; i++) similarities[i].calculateMaxMatch();
 		Arrays.sort(similarities);
 		System.out.println("\nStats:");
-		for (int i=0; i< similarities.length; i++) System.out.println(similarities[i].toString(sampleNames));
+		for (int i=0; i< similarities.length; i++) {
+			similarityForJson[i] = similarities[i].toString(sampleNames);
+			System.out.println(similarityForJson[i]);
+		}
 	}
 
 	public void printGenderRatios(){
+		genderForJson = new String[afHist.length];
 		IO.pl("Het/Hom histogram AF count ratios for AllChrs, ChrX, log2(All/X)");
 		for (int i=0; i< afHist.length; i++){
 			double allChr = ratioCenterVsLast(afHist[i]);
 			double chrX = ratioCenterVsLast(chrXAfHist[i]);
 			double lgrto = Num.log2(allChr/chrX);
-			IO.pl(sampleNames[i] +"\t"+Num.formatNumber(allChr, 3)+"\t"+Num.formatNumber(chrX, 3)+"\t"+Num.formatNumber(lgrto, 3));
+			genderForJson[i] = sampleNames[i] +"\t"+Num.formatNumber(allChr, 3)+"\t"+Num.formatNumber(chrX, 3)+"\t"+Num.formatNumber(lgrto, 3);
+			IO.pl(genderForJson[i]);
 		}
 	}
 	
@@ -208,6 +219,28 @@ public void printHist(){
 			System.out.println();
 		}
 	}
+	
+	@SuppressWarnings("unchecked")
+	private void saveJson() {
+		if (jsonOutputFile == null) return;
+		
+		try {
+			String[] names = new String[bamFiles.length];
+			for (int i=0; i< names.length; i++) names[i] = Misc.removeExtension(bamFiles[i].getName());
+			//output simple json, DO NOT change the key names without updated downstream apps that read this file!
+			Gzipper gz = new Gzipper(jsonOutputFile);
+			gz.println("{");
+			gz.printJson("bamFileNames", names, true);
+			gz.printJson("similarities", similarityForJson, true);
+			gz.printJson("genderChecks", genderForJson, false);
+			gz.println("}");
+			gz.close();
+			
+		} catch (Exception e){
+			e.printStackTrace();
+			Misc.printErrAndExit("\nProblem writing json file! "+jsonOutputFile);
+		}
+	}
 
 
 	public static void main(String[] args) {
@@ -241,6 +274,7 @@ public void processArgs(String[] args) throws IOException{
 					case 'm': minAFForMatch = Double.parseDouble(args[++i]); break;
 					case 'q': minBaseQuality = Integer.parseInt(args[++i]); break;
 					case 'u': minMappingQuality = Integer.parseInt(args[++i]); break;
+					case 'j': jsonOutputFile = new File(args[++i]); break;
 					case 't': numberThreads = Integer.parseInt(args[++i]); break;
 					default: Misc.printErrAndExit("\nProblem, unknown option! " + mat.group());
 					}
@@ -259,9 +293,10 @@ public void processArgs(String[] args) throws IOException{
 		if (fasta == null || fasta.canRead() == false) Misc.printErrAndExit("\nError: cannot find your indexed fasta reference file "+fasta);
 		//check bams
 		if (bamFiles == null || bamFiles.length == 0) Misc.printErrAndExit("\nError: cannot find your bam files to parse? ");
+		
 		//threads to use
 		double gigaBytesAvailable = ((double)Runtime.getRuntime().maxMemory())/ 1073741824.0;
-		int numPossCores = (int)Math.round(gigaBytesAvailable/5.0);
+		int numPossCores = (int)Math.round(gigaBytesAvailable/5);
 		if (numPossCores < 1) numPossCores = 1;
 		int numPossThreads = Runtime.getRuntime().availableProcessors();
 		if (numberThreads == 0){
@@ -291,7 +326,7 @@ public void processArgs(String[] args) throws IOException{
 	public static void printDocs(){
 		System.out.println("\n" +
 				"**************************************************************************************\n" +
-				"**                               Bam Concordance: Feb  2018                         **\n" +
+				"**                               Bam Concordance: Sept  2018                        **\n" +
 				"**************************************************************************************\n" +
 				"BC calculates sample level concordance based on uncommon homozygous SNVs found in bam\n"+
 				"files. Samples from the same person will show high similarity (>0.9). Run BC on\n"+
@@ -321,12 +356,13 @@ public static void printDocs(){
 				"-m Minimum allele frequency to count a homozygous match, defaults to 0.9\n"+
 				"-q Minimum base quality, defaults to 20.\n"+
 				"-u Minimum mapping quality, defaults to 20.\n"+
+				"-j Write gzipped summary stats in json format to this file.\n"+
 				"-t Number of threads to use.  If not set, determines this based on the number of\n"+
 				"      threads and memory available to the JVM so set the -Xmx value to the max.\n\n"+
 
-				"Example: java -Xmx120G -jar pathTo/USeq/Apps/BamConcordance -r ~/exomeTargets.bed\n"+
+				"Example: java -Xmx100G -jar pathTo/USeq/Apps/BamConcordance -r ~/exomeTargets.bed\n"+
 				"      -s ~/Samtools1.3.1/bin/samtools -b ~/Patient7Bams -d 10 -a 0.9 -m 0.8 -f\n"+
-				"      ~/B37/human_g1k_v37.fasta -c ~/B37/b38ComSnps.bed.gz\n\n" +
+				"      ~/B37/human_g1k_v37.fasta -c ~/B37/b38ComSnps.bed.gz -j bc.json.gz \n\n" +
 
 				"**************************************************************************************\n");
 
diff --git a/Source/edu/utah/seq/run/FastqDataset.java b/Source/edu/utah/seq/run/FastqDataset.java
index c675c75b..447a5d3c 100644
--- a/Source/edu/utah/seq/run/FastqDataset.java
+++ b/Source/edu/utah/seq/run/FastqDataset.java
@@ -2,6 +2,9 @@
 
 import java.io.File;
 import java.io.IOException;
+import java.util.ArrayList;
+
+import util.gen.IO;
 
 /**Info related to a directory of fastq files.*/
 public class FastqDataset{
@@ -10,17 +13,26 @@ public class FastqDataset{
 	private File[] fastqs = null;
 	private String name = null;
 
-	public FastqDataset (File fastqDir, String name) throws IOException{
+	public FastqDataset (File fastqDir, String name, ArrayList<String> info) throws IOException{
 		if (fastqDir != null) {
 			this.name = name;
 			File dir = new File(fastqDir, name);
 			if (dir.exists()) {
 				fastqDirExists = true;
 				//ok it exits check that there are two files
-				fastqs = TNSample.checkNumberFiles(dir, ".gz", 2);
+				fastqs = checkNumberFiles(dir, ".gz", 2);
+				if (fastqs == null) info.add("\t\tSkipping "+name+". Failed to find two fastq files within "+fastqDir);
 			}
 		}
 	}
+	
+	public File[] checkNumberFiles(File dir, String extension, int requiredNumberFiles) throws IOException {
+		File[] f = IO.extractFiles(dir, extension);
+		if (f.length != requiredNumberFiles) {
+			return null;
+		}
+		return f;
+	}
 
 	public boolean isFastqDirExists() {
 		return fastqDirExists;
diff --git a/Source/edu/utah/seq/run/TNRunner.java b/Source/edu/utah/seq/run/TNRunner.java
index dfdec680..3eed1355 100644
--- a/Source/edu/utah/seq/run/TNRunner.java
+++ b/Source/edu/utah/seq/run/TNRunner.java
@@ -405,7 +405,7 @@ public static void printDocs(){
 				"\nOptions:\n"+
 				"-p Directory containing one or more patient data directories to process.\n" +
 				"-e Workflow docs for launching exome alignments.\n"+
-				"-t Workflow docs for launching exome alignments.\n"+
+				"-t Workflow docs for launching transcriptome alignments.\n"+
 				"-c Workflow docs for launching somatic variant calling.\n"+
 				"-a Workflow docs for launching variant annotation.\n"+
 				"-b Workflow docs for launching bam concordance.\n"+
diff --git a/Source/edu/utah/seq/run/TNSample.java b/Source/edu/utah/seq/run/TNSample.java
index bcff20fa..38ef551c 100644
--- a/Source/edu/utah/seq/run/TNSample.java
+++ b/Source/edu/utah/seq/run/TNSample.java
@@ -249,13 +249,13 @@ private void removeBamConcordanceLinks(File jobDir) throws IOException{
 			new File(jobDir, "tumorExome.bai").delete();
 		}
 		if (normalExomeBamBedGvcf != null) {
-			new File(jobDir, "tumorTranscriptome.bam").delete();
-			new File(jobDir, "tumorTranscriptome.bai").delete();
-		}
-		if (tumorTranscriptomeBam != null) {
 			new File(jobDir, "normalExome.bam").delete();
 			new File(jobDir, "normalExome.bai").delete();
 		}
+		if (tumorTranscriptomeBam != null) {
+			new File(jobDir, "tumorTranscriptome.bam").delete();
+			new File(jobDir, "tumorTranscriptome.bai").delete();
+		}
 	}
 
 	private void annotateSomaticVcf() throws IOException {
@@ -754,21 +754,15 @@ public static void removeProgressFiles(File alignDir) {
 	private boolean checkFastq() throws IOException {
 		info.add("Checking Fastq availability...");
 		File fastqDir = makeCheckFile(rootDir, "Fastq");
-		tumorExomeFastq = new FastqDataset(fastqDir, "TumorExome");
-		normalExomeFastq = new FastqDataset(fastqDir, "NormalExome");
-		tumorTransFastq = new FastqDataset(fastqDir, "TumorTranscriptome");
+		tumorExomeFastq = new FastqDataset(fastqDir, "TumorExome", info);
+		normalExomeFastq = new FastqDataset(fastqDir, "NormalExome", info);
+		tumorTransFastq = new FastqDataset(fastqDir, "TumorTranscriptome", info);
 		if (tumorExomeFastq.isFastqDirExists()) info.add("\tTumorExome\t"+ (tumorExomeFastq.getFastqs() != null));
 		if (normalExomeFastq.isFastqDirExists()) info.add("\tNormalExome\t"+ (normalExomeFastq.getFastqs() != null));
 		if (tumorTransFastq.isFastqDirExists()) info.add("\tTumorTranscriptome\t"+ (tumorTransFastq.getFastqs() != null));
 		if (tumorExomeFastq.getFastqs() != null || normalExomeFastq.getFastqs() != null || tumorTransFastq.getFastqs() != null) return true;
 		return false;
 	}
-	
-	public static File[] checkNumberFiles(File dir, String extension, int requiredNumberFiles) throws IOException {
-		File[] f = IO.extractFiles(dir, extension);
-		if (f.length != requiredNumberFiles) throw new IOException("Failed to find "+requiredNumberFiles+ " "+extension+ " files in "+dir);
-		return f;
-	}
 
 	public static File makeCheckFile(File parentDir, String fileName) throws IOException {
 		File f = new File(parentDir, fileName);
diff --git a/Source/edu/utah/seq/vcf/AnnotatedVcfParser.java b/Source/edu/utah/seq/vcf/AnnotatedVcfParser.java
index fc85d811..45dd3cdc 100644
--- a/Source/edu/utah/seq/vcf/AnnotatedVcfParser.java
+++ b/Source/edu/utah/seq/vcf/AnnotatedVcfParser.java
@@ -650,7 +650,6 @@ public Histogram makeReadDepthHistogram(){
 	private void modifySettingsForFoundation() {
 		if (minimumDP == 0) minimumDP = 50;
 		if (minimumAF == 0) minimumAF = 0.01;
-		if (maximumAF == 1) maximumAF = 0.5;
 		if (maxFracBKAFs == 0) maxFracBKAFs = 0.1;
 		if (maximumPopAF == 0) maximumPopAF = 0.01;
 		if (passingAnnImpact == null) {
@@ -882,7 +881,7 @@ public void processArgs(String[] args){
 	public static void printDocs(){
 		IO.pl("\n" +
 				"**************************************************************************************\n" +
-				"**                              Annotated Vcf Parser  Aug 2018                      **\n" +
+				"**                             Annotated Vcf Parser  Sept 2018                      **\n" +
 				"**************************************************************************************\n" +
 				"Splits VCF files that have been annotated with SnpEff w/ dbNSFP and clinvar, plus the\n"+
 				"VCFBackgroundChecker and VCFSpliceScanner USeq apps into passing and failing records.\n"+
diff --git a/Source/edu/utah/seq/vcf/VCFCallFrequency.java b/Source/edu/utah/seq/vcf/VCFCallFrequency.java
index f9b986d9..a43357a1 100644
--- a/Source/edu/utah/seq/vcf/VCFCallFrequency.java
+++ b/Source/edu/utah/seq/vcf/VCFCallFrequency.java
@@ -18,11 +18,13 @@
 import org.apache.commons.httpclient.UsernamePasswordCredentials;
 import org.apache.commons.httpclient.auth.AuthScope;
 import org.apache.commons.httpclient.methods.GetMethod;
-import com.eclipsesource.json.JsonArray;
-import com.eclipsesource.json.JsonObject;
-import com.eclipsesource.json.JsonValue;  
+import org.json.JSONArray;
+import org.json.JSONObject;
+import edu.utah.hci.query.LocalQuery;
+import edu.utah.hci.query.UserQuery;  
 
-/**Calculates call frequencies for each vcf record from a db of vcf files and callable region files.*/
+/**Calculates call frequencies for each vcf record from a db of vcf files and callable region files. 
+ * Most mutated single site in p53 is seen at 0.039 across all cancers in TCGA.  0.57 for BRAF V600E across all cancers.*/
 public class VCFCallFrequency {
 
 	private File[] vcfFiles;
@@ -42,7 +44,13 @@ public class VCFCallFrequency {
 	private double[] idVcfCount;
 	private double maxCallFreq = 1.0;
 	private int minBedCount = 8;
-	private boolean appendFilter = false;
+	private boolean appendFilter = true;
+	//local searching
+	private File dataDir = null;
+	private File indexDir = null;
+	private LocalQuery localQuery = null;
+	private UserQuery userQueryVcf = null;
+	private UserQuery userQueryBed = null;
 
 	private boolean debug = false;
 	private HttpClient client = new HttpClient();
@@ -52,13 +60,14 @@ public class VCFCallFrequency {
 	private int numFailingCallFreq = 0;
 	private int numPassingCallFreq = 0;
 	private int numWithLowBedCount = 0;
-	
+
 
 	public VCFCallFrequency (String[] args) {
 		long startTime = System.currentTimeMillis();
 		processArgs(args);
 
-		buildSearchUrls(); 
+		if (queryURL != null) buildSearchUrls(); 
+		else localQuery = new LocalQuery(dataDir, indexDir, false);
 
 		for (File vcf: vcfFiles) {
 			IO.p(vcf.getName());
@@ -174,25 +183,40 @@ else if (addFilter && line.startsWith("##FILTER")){
 	}
 
 	private void processRecordBlock(String[][] vcfRecords, Gzipper out) throws IOException {
-		//create search strings, any vcfs to query?
-		String[] searchStrings = makeSearchUrls(vcfRecords);
-
-		if (searchStrings != null) {
-
-			//execute vcf query
-			String jsonResultsVcf = query(searchStrings[0]);
-			//IO.p("VCF RETURN\n"+jsonResultsVcf);			
-			JsonObject joVcf = JsonObject.readFrom(jsonResultsVcf);
-			JsonValue jvVcf = joVcf.get("queryResults");
-			loadVcfResults(jvVcf);
-
-			//execute bed query
-			String jsonBedResults = query(searchStrings[1]);
-			//IO.p("BED RETURN\n"+jsonBedResults);
-			JsonObject joBed = JsonObject.readFrom(jsonBedResults);
-			JsonValue jvBed = joBed.get("queryResults");
-			loadBedResults(jvBed);
+		
+		//run queries
+		JSONObject jsonResultsVcf = null;
+		JSONObject jsonResultsBed = null;
+		
+		//remote service?
+		if (queryURL != null){
+			//create search strings, any vcfs to query?
+			String[] searchStrings = makeSearchUrls(vcfRecords);
+			if (searchStrings != null) {
+				//execute vcf query
+				jsonResultsVcf = new JSONObject(query(searchStrings[0]));
+				jsonResultsBed =  new JSONObject(query(searchStrings[1]));
+			}
+		}
+		//local file system query?
+		else {
+			if (loadVcfsBedsToQuery(vcfRecords)) {
+				localQuery.query(userQueryVcf);
+				localQuery.query(userQueryBed);
+				jsonResultsVcf  = userQueryVcf.getResults();
+				jsonResultsBed = userQueryBed.getResults();
+			}
+		}
 
+		//parse and annotate
+		if (jsonResultsVcf != null && jsonResultsBed != null) {
+			
+			if (debug) IO.p("VCF RETURN\n"+jsonResultsVcf.toString(1));			
+			loadVcfResults(jsonResultsVcf);
+			
+			if (debug) IO.p("BED RETURN\n"+jsonResultsBed.toString(1));
+			loadBedResults(jsonResultsBed);
+			
 			//for each record
 			for (int i=0; i< vcfRecords.length; i++){
 				//pull counts and calc ratio
@@ -206,11 +230,11 @@ private void processRecordBlock(String[][] vcfRecords, Gzipper out) throws IOExc
 					if (ratio > 1.0) ratio = 1.0;
 				}
 				String cf = "CF="+Num.formatNumber(ratio, 3)+","+(int)vcfCount+","+(int)bedCount;
-				
+
 				//modify INFO field of each record
 				vcfRecords[i][7] = cf+";"+vcfRecords[i][7];
 				boolean printMe = true;
-				
+
 				//check bed count, must pass this before any filtering is applied.
 				if (bedCount >= minBedCount) {
 					hist.count(ratio);
@@ -234,14 +258,8 @@ private void processRecordBlock(String[][] vcfRecords, Gzipper out) throws IOExc
 					out.println();
 				}
 			}
-			
-			
-			
-			
-
-			
 		}
-		else throw new IOException("Error! Failed to generate search urls?! Aborting.");
+		//else throw new IOException("Error! Failed to generate search urls?! Aborting.");
 		System.out.print(".");
 
 	}
@@ -250,21 +268,20 @@ private String modifyFilter(String filter) {
 		if (filter.equals(".") || filter.toUpperCase().equals("PASS")) return "CallFreq";
 		return "CallFreq;"+filter;
 	}
-
+	
 	/**Loads the number of hits returned to each vcf query. Some won't be returned and thus will be zero.*/
-	private void loadVcfResults(JsonValue jvVcf) {
+	private void loadVcfResults(JSONObject results) {
 		//clear prior
 		idVcfCount = new double[numberRecordsPerQuery];
-		
-		JsonArray ja = jvVcf.asArray();
-		int numWithHits = ja.size();
+
+		JSONArray ja = results.getJSONArray("queryResults");
+		int numWithHits = ja.length();
 		totalQueriesWithHits+= numWithHits;
-		
+
 		for (int i=0; i< numWithHits; i++){
-			JsonObject joq = ja.get(i).asObject();
-			double numHits = joq.get("numberHits").asInt();
-			JsonValue input = joq.get("input");
-			String vcfRecord = input.asString();
+			JSONObject jo = ja.getJSONObject(i);
+			double numHits = jo.getDouble("numberHits");
+			String vcfRecord = jo.getString("input");
 			String[] splitVcf = Misc.TAB.split(vcfRecord);
 			int index = Integer.parseInt(splitVcf[2]);
 			//set in array
@@ -272,22 +289,19 @@ private void loadVcfResults(JsonValue jvVcf) {
 		}
 	}
 
+
 	/**Loads the number of hits returned to each bed query. Some may be returned and thus will be zero.*/
-	private void loadBedResults(JsonValue jvBed) {
+	private void loadBedResults(JSONObject results) {
 		//clear prior
 		idBedCount = new double[numberRecordsPerQuery];
-
-		JsonArray ja = jvBed.asArray();
-		int numWithHits = ja.size();
-
+		JSONArray ja = results.getJSONArray("queryResults");
+		int numWithHits = ja.length();
 		for (int i=0; i< numWithHits; i++){
-			JsonObject joq = ja.get(i).asObject();
-			double numHits = joq.get("numberHits").asInt();
-			JsonValue input = joq.get("input");
-			String bedRecord = input.asString();
+			JSONObject jo = ja.getJSONObject(i);
+			double numHits = jo.getDouble("numberHits");
+			String bedRecord = jo.getString("input");
 			String[] splitbed = Misc.TAB.split(bedRecord);
 			int index = Integer.parseInt(splitbed[3]);
-
 			//set in array
 			idBedCount[index] = numHits;
 		}
@@ -352,10 +366,50 @@ private StringBuilder[] pullVcfsBedsToQuery(String[][] vcfRecords) {
 			sbBed.append(zeroPos+maxSize(vcf[4])); sbBed.append("%09");
 			//name, score, strand
 			sbBed.append(i); sbBed.append("%090%09.;");
-
 		}
 		return new StringBuilder[]{sbVcf, sbBed};
 	}
+	
+	private boolean loadVcfsBedsToQuery(String[][] vcfRecords) {
+		boolean loaded = false;
+		userQueryVcf.clearResultsRegionsRecords();
+		userQueryBed.clearResultsRegionsRecords();
+		
+		for (int i=0; i< vcfRecords.length; i++){
+			String[] vcf = vcfRecords[i];
+			if (vcf == null) break;
+			StringBuilder sbVcf = new StringBuilder();
+			StringBuilder sbBed = new StringBuilder();
+			//build vcf query
+			//pull chr, pos, ref, alt
+			sbVcf.append(vcf[0]); sbVcf.append("\t");
+			//pos
+			sbVcf.append(vcf[1]); sbVcf.append("\t");	
+			//replace id with index number of vcfRecords[index][]
+			sbVcf.append(i); sbVcf.append("\t");
+			//ref
+			sbVcf.append(vcf[3]); sbVcf.append("\t");
+			//alt
+			sbVcf.append(vcf[4]);
+			//remainder
+			sbVcf.append("\t.\t.\t.");
+			userQueryVcf.addVcfRecord(sbVcf.toString());
+
+			//build bed region query
+			//chrom
+			sbBed.append(vcf[0]); sbBed.append("\t");
+			//start
+			int zeroPos = Integer.parseInt(vcf[1])-1;
+			sbBed.append(zeroPos); sbBed.append("\t");
+			//max stop
+			sbBed.append(zeroPos+maxSize(vcf[4])); sbBed.append("\t");
+			//name, score, strand
+			sbBed.append(i); sbBed.append("\t0\t.");
+			userQueryBed.addBedRegion(sbBed.toString());
+			loaded = true;
+		}
+		return loaded;
+	}
 
 	public static int maxSize(String alts){
 		String[] a = Misc.COMMA.split(alts);
@@ -388,12 +442,14 @@ public void processArgs(String[] args){
 					switch (test){
 					case 'v': forExtraction = new File(args[++i]); break;
 					case 'f': fileFilter = args[++i]; break;
-					case 'a': appendFilter = true; break;
+					case 'x': appendFilter = false; break;
 					case 'c': configFile = new File(args[++i]); break;
+					case 'i': indexDir = new File(args[++i]); break;
+					case 'd': dataDir = new File(args[++i]); break;
 					case 'm': maxCallFreq = Double.parseDouble(args[++i]); break;
 					case 'b': minBedCount = Integer.parseInt(args[++i]); break;
 					case 's': saveDirectory = new File(args[++i]); break;
-					case 'd': debug = true; break;
+					case 'e': debug = true; break;
 					case 'h': printDocs(); System.exit(0);
 					default: Misc.printErrAndExit("\nProblem, unknown option! " + mat.group());
 					}
@@ -403,17 +459,26 @@ public void processArgs(String[] args){
 				}
 			}
 		}
-		
-		//config file
-		if (configFile == null || configFile.canRead() == false) Misc.printErrAndExit("\nSorry, cannot find or read your config file: "+configFile+"\n");
-		HashMap<String, String> config = IO.loadFileIntoHashMapLowerCaseKey(configFile);
-		if (config.containsKey("queryurl")) queryURL = config.get("queryurl");
-		if (config.containsKey("host")) host = config.get("host");
-		if (config.containsKey("realm")) realm = config.get("realm");
-		if (config.containsKey("username")) userName = config.get("username");
-		if (config.containsKey("password")) password = config.get("password");
-		
-		
+
+		//config file? or local
+		if (configFile != null){
+			if (configFile.canRead() == false) Misc.printErrAndExit("\nSorry, cannot read your config file: "+configFile+"\n");
+			HashMap<String, String> config = IO.loadFileIntoHashMapLowerCaseKey(configFile);
+			if (config.containsKey("queryurl")) queryURL = config.get("queryurl");
+			if (config.containsKey("host")) host = config.get("host");
+			if (config.containsKey("realm")) realm = config.get("realm");
+			if (config.containsKey("username")) userName = config.get("username");
+			if (config.containsKey("password")) password = config.get("password");
+			if (config.containsKey("maxcallfreq")) maxCallFreq = Double.parseDouble(config.get("maxcallfreq"));
+			if (config.containsKey("filefilter")) fileFilter = config.get("filefilter");
+		}
+		else {
+			if (dataDir == null || indexDir == null || dataDir.isDirectory()== false || indexDir.isDirectory() == false) {
+				Misc.printErrAndExit("\nProvide either a configuration file for remotely accessing a genomic query service or "
+						+ "two directory paths to the Data and Index directories uses by the USeq QueryIndexer app.\n");;
+			}
+		}
+
 		IO.pl("\n"+IO.fetchUSeqVersion()+" Arguments:");
 		IO.pl("\t-v Vcfs "+forExtraction);
 		IO.pl("\t-s SaveDir "+saveDirectory);
@@ -422,10 +487,24 @@ public void processArgs(String[] args){
 		IO.pl("\t-b MinBedCount "+minBedCount);
 		IO.pl("\t-a Append FILTER "+appendFilter);
 		IO.pl("\t-d Verbose "+debug);
-		IO.pl("\tQueryUrl "+queryURL);
-		IO.pl("\tHost "+host);
-		IO.pl("\tRealm "+realm);
-		IO.pl("\tUserName "+userName);
+		if (configFile != null){
+			IO.pl("\tQueryUrl "+queryURL);
+			IO.pl("\tHost "+host);
+			IO.pl("\tRealm "+realm);
+			IO.pl("\tUserName "+userName);
+			//check config params
+			if (queryURL == null) Misc.printErrAndExit("\nError: failed to find a queryUrl in the config file, e.g. queryUrl http://hci-clingen1.hci.utah.edu:8080/Query/");
+			if (queryURL.endsWith("/") == false) queryURL = queryURL+"/";
+			if (host == null) Misc.printErrAndExit("\nError: failed to find a host in the config file, e.g. host hci-clingen1.hci.utah.edu");
+			if (realm == null) Misc.printErrAndExit("\nError: failed to find a realm in the config file, e.g. realm QueryAPI");
+			if (userName == null) Misc.printErrAndExit("\nError: failed to find a userName in the config file, e.g. userName FCollins");
+			if (password == null) Misc.printErrAndExit("\nError: failed to find a password in the config file, e.g. password g0QueryAP1");
+
+		}
+		else {
+			IO.pl("\tDataDir "+dataDir);
+			IO.pl("\tIndexDir "+indexDir);
+		}
 		IO.pl();
 
 
@@ -438,59 +517,58 @@ public void processArgs(String[] args){
 		vcfFiles = IO.collapseFileArray(tot);
 		if (vcfFiles == null || vcfFiles.length ==0 || vcfFiles[0].canRead() == false) Misc.printExit("\nError: cannot find your xxx.vcf(.zip/.gz OK) file(s)!\n");
 
-
-
 		//check params
-		if (queryURL == null) Misc.printErrAndExit("\nError: failed to find a queryUrl in the config file, e.g. queryUrl http://hci-clingen1.hci.utah.edu:8080/Query/");
-		if (queryURL.endsWith("/") == false) queryURL = queryURL+"/";
-		if (host == null) Misc.printErrAndExit("\nError: failed to find a host in the config file, e.g. host hci-clingen1.hci.utah.edu");
-		if (realm == null) Misc.printErrAndExit("\nError: failed to find a realm in the config file, e.g. realm QueryAPI");
-		if (userName == null) Misc.printErrAndExit("\nError: failed to find a userName in the config file, e.g. userName FCollins");
-		if (password == null) Misc.printErrAndExit("\nError: failed to find a password in the config file, e.g. password g0QueryAP1");
 		if (fileFilter == null) Misc.printErrAndExit("\nError: provide a fileFilter, e.g. /B37/Somatic/Avatar/ ");
-		
-
-
 		if (saveDirectory == null) Misc.printErrAndExit("\nError: provide a directory to save the annotated vcf files.");
 		else saveDirectory.mkdirs();
 		if (saveDirectory.exists() == false) Misc.printErrAndExit("\nError: could not find your save directory? "+saveDirectory);
+		
+		userQueryVcf = new UserQuery().addRegexAll(".vcf.gz").addRegexAll(fileFilter).matchVcf();
+		userQueryBed = new UserQuery().addRegexAll(".bed.gz").addRegexAll(fileFilter);
 	}	
 
 	public static void printDocs(){
 		IO.pl("\n" +
 				"**************************************************************************************\n" +
-				"**                            VCF Call Frequency: August 2018                       **\n" +
+				"**                            VCF Call Frequency: Sept 2018                         **\n" +
 				"**************************************************************************************\n" +
 				"Calculates a vcf call frequency for each variant when pointed at a genomic Query\n"+
-				"service (https://github.com/HuntsmanCancerInstitute/Query). CallFreq's are calculated \n"+
+				"service (https://github.com/HuntsmanCancerInstitute/Query) or the Data and Index\n"+
+				"directories the service is accessing. CallFreq's are calculated \n"+
 				"by first counting the number of exact vcf matches present and dividing it by the\n"+
 				"number of intersecting bed files. Use this to flag variants with high call rates that\n"+
-				"are potential false positives. Use the file filter to limit which files to use in the\n"+
+				"are potential false positives. Some are not, e.g. BRAF V600E occurs in 58% of cancers\n"+
+				"with a BRAF mutation. So treate the call freq as an annotation requiring context level\n"+
+				"interpretation. Use the file filter to limit which files are included in the call freq\n"+
 				"calculations. Only file paths containing the file filter will be included. Be sure to\n"+
 				"place both the sample vcf and associated callable region bed files in the same Query\n"+
-				"index folder.\n"+
+				"index folder path as defined by -f. \n"+
 
 				"\nRequired Options:\n"+
-				"-v Full path file or directory containing xxx.vcf(.gz/.zip OK) file(s)\n" +
+				"-v Full path to a file or directory containing xxx.vcf(.gz/.zip OK) file(s)\n" +
 				"-s Directory to save the annotated vcf files\n"+
 				"-f Query service file filter, e.g. /B37/Somatic/Avatar/\n"+
 				"-c Config txt file containing two tab delimited columns with host, queryUrl, realm, \n"+
-				"     userName, and password. 'chmod 600' the file! e.g.: \n"+
+				"     userName, password, and (optionally) fileFilter and or maxCallFreq. 'chmod 600'\n"+
+				"     the file! e.g.: \n"+
 				"     host hci-clingen1.hci.utah.edu\n"+
 				"     queryUrl http://hci-clingen1.hci.utah.edu:8080/Query/\n"+
 				"     realm QueryAPI\n"+
 				"     userName FColins\n"+
 				"     password g0QueryAP1\n"+
-				
+				"-i (Alternative to -c), provide a path to the Index directory generated by the USeq \n"+
+				"     QueryIndexer app.\n"+
+				"-d (Alternative to -c), provide a path to the Data directory used in creating the Index.\n"+
+
 				"\nOptions:\n"+
-				"-m Maximum call frequency, defaults to 1, no filtering.\n"+
+				"-m Maximum call freq, defaults to 1, before appending 'CallFreq' to the FILTER field.\n"+
+				"-x Remove failing max call freq records, not recommended.\n"+
 				"-b Minimum bed call count before applying a max call freq filter, defaults to 8.\n"+
-				"-a Keep failing records and append CallFreq to the FILTER field.\n"+
-				"-d Print verbose debugging output.\n"+
+				"-e Print verbose debugging output.\n"+
 
 
-				"\nExample: java -jar pathToUSeq/Apps/VCFCallFrequency -v Vcfs/ -s CFVcfs -f \n"+
-				"    /B37/Somatic/Avatar/ -m 0.2 -a -c vcfCFConfig.txt \n\n" +
+				"\nExample: java -jar pathToUSeq/Apps/VCFCallFrequency -v Vcf/ -s CFVcfs -f \n"+
+				"    /B37/Somatic/Avatar/ -m 0.05 -c vcfCFConfig.txt \n\n" +
 				"**************************************************************************************\n");
 
 	}
diff --git a/Source/edu/utah/seq/vcf/splice/VCFSpliceScanner.java b/Source/edu/utah/seq/vcf/splice/VCFSpliceScanner.java
index e6ec0ca1..698577c3 100644
--- a/Source/edu/utah/seq/vcf/splice/VCFSpliceScanner.java
+++ b/Source/edu/utah/seq/vcf/splice/VCFSpliceScanner.java
@@ -329,7 +329,8 @@ private void loadChromosomeData(String chrom) {
 	}
 
 	public void loadAndModifyHeader(BufferedReader in) throws IOException{
-		File header = new File(tempDirectory, "headerPlusUnAnnotated.vcf.gz");
+		File header = new File(tempDirectory, "headerPlusUnAnnotated.vcf.gz").getCanonicalFile();
+		header.createNewFile();
 		gzippedToConcat.add(header);
 		
 		vcfOut = new Gzipper(header);
@@ -373,6 +374,7 @@ else if (line.startsWith("#CHROM")){
 				else vcfOut.println(line);
 			}
 		}
+		vcfOut.flush();
 		if (foundChrom == false) throw new IOException("\tError: Failed to find the #CHROM header line? Aborting.\n");
 		if (addedInfo == false) throw new IOException("\tError: Failed to find any ##INFO header lines? Aborting.\n");
 	}
@@ -446,7 +448,7 @@ public void processArgs(String[] args) throws FileNotFoundException{
 		
 		//threads to use
 		double totalGbAvailable = (double)(Runtime.getRuntime().maxMemory()/1000000000.0);
-		int numPossCores = (int)Math.round(totalGbAvailable/5.0);
+		int numPossCores = (int)Math.round(totalGbAvailable/10.0);
 		if (numPossCores < 1) numPossCores = 1;
 		int numPossThreads = Runtime.getRuntime().availableProcessors();
 		
@@ -456,7 +458,7 @@ public void processArgs(String[] args) throws FileNotFoundException{
 		
 		System.out.println("Core usage:\n\tTotal GB available to Java:\t"+ Num.formatNumber(totalGbAvailable, 1));
 		System.out.println("\tTotal available cores:\t"+numPossThreads);
-		System.out.println("\tNumber cores to use @ 5GB/core:\t"+numberThreads+"\n");
+		System.out.println("\tNumber cores to use @ 10GB/core:\t"+numberThreads+"\n");
 		
 		
 		//flip booleans?
@@ -593,13 +595,13 @@ private static void scoreKnownSplices(HashMap<String,UCSCGeneLine[]> chromGenes,
 	public static void printDocs(){
 		System.out.println("\n" +
 				"**************************************************************************************\n" +
-				"**                             VCF Splice Scanner : Dec 2017                        **\n" +
+				"**                            VCF Splice Scanner : Sept 2018                        **\n" +
 				"**************************************************************************************\n" +
 				"Scores variants for changes in splicing using the MaxEntScan algorithms. See Yeo and\n"+
 				"Burge 2004, http://www.ncbi.nlm.nih.gov/pubmed/15285897 for details. Known splice\n"+
 				"acceptors and donors are scored for loss of a junction.  Exonic, intronic, and splice\n"+
 				"bases are scanned for novel junctions in a window around each variant. See the vcf\n"+
-				"INFO header for a description of the output. Use this information to identify likely\n"+
+				"INFO header for a description of the output. Use this information to identify \n"+
 				"snv and indel variants that may effect splicing.\n\n" +
 
 				"Required Options:\n"+
diff --git a/Source/util/gen/Gzipper.java b/Source/util/gen/Gzipper.java
index 8374e10e..c970ad3b 100644
--- a/Source/util/gen/Gzipper.java
+++ b/Source/util/gen/Gzipper.java
@@ -17,7 +17,6 @@ public Gzipper (File gzipFile) throws FileNotFoundException, IOException{
 		if (gzipFile.getName().endsWith(".gz") == false) this.gzipFile = new File (gzipFile+".gz");
 		else this.gzipFile = gzipFile;
 		out = new GZIPOutputStream(new FileOutputStream(this.gzipFile));
-		
 	}
 	
 	public Gzipper(OutputStream gzipFile) throws IOException {
@@ -29,6 +28,12 @@ public void close() throws IOException{
 		out.close();
 	}
 	
+	/**Flush the stream
+	 * @throws IOException */
+	public void flush() throws IOException{
+		out.flush();
+	}
+	
 	/**Attempts a close, no exceptions thrown or warnings.*/
 	public void closeNoException(){
 		try {
@@ -110,6 +115,24 @@ public void printJson(String key, double[] values, boolean addComma) throws IOEx
 		if (addComma) out.write(rtnComma);
 		else out.write(rtn);
 	}
+	/**Calls toString on each object in the array.*/
+	public void printJson(String key, Object[] values, boolean addComma) throws IOException {
+		StringBuilder sb = new StringBuilder();
+		sb.append("\"");
+		sb.append(key);
+		sb.append("\"");
+		sb.append(": [");
+		sb.append(values[0].toString());
+		for (int i=1; i< values.length; i++){
+			sb.append(", ");
+			sb.append(values[i].toString());
+		}
+		sb.append("]");
+		out.write(sb.toString().getBytes());
+		if (addComma) out.write(rtnComma);
+		else out.write(rtn);
+	}
+
 	public void printJson(String key, double value, boolean addComma) throws IOException {
 		StringBuilder sb = new StringBuilder();
 		sb.append("\"");
diff --git a/Source/util/gen/IO.java b/Source/util/gen/IO.java
index 308fea59..365da554 100755
--- a/Source/util/gen/IO.java
+++ b/Source/util/gen/IO.java
@@ -1904,6 +1904,26 @@ public static HashSet<String> loadFileIntoHashSet(File file){
 		}
 		return names;
 	}
+	
+	/**Loads a file's lines into a hash set, counts duplicates.*/
+	public static HashMap<String, Integer> loadFileIntoHashMapCounter(File file){
+		HashMap<String,Integer> names = new HashMap<String,Integer>(10000);
+		try{
+			BufferedReader in = fetchBufferedReader(file);
+			String line;
+			while ((line = in.readLine())!=null){
+				line = line.trim();
+				if (line.length() ==0) continue;
+				Integer i = names.get(line);
+				if (i == null) names.put(line, new Integer(1));
+				else names.put(line, new Integer(i.intValue() + 1));
+			}
+		}catch(Exception e){
+			System.out.println("Prob loadFileInttoHash()");
+			e.printStackTrace();
+		}
+		return names;
+	}
 
 
 
diff --git a/build.properties b/build.properties
index 67f17ba7..96283967 100644
--- a/build.properties
+++ b/build.properties
@@ -1,4 +1,4 @@
-version=9.1.9
+version=9.2.0
 src.dir=Source
 classes.dir=Classes
 orig.doc.dir=Documentation/USeqDocumentation
diff --git a/build.xml b/build.xml
index eba0e8d5..e0e9a4f7 100644
--- a/build.xml
+++ b/build.xml
@@ -14,6 +14,7 @@
 		<!-- Copy base class files to the build directory -->
 		<echo>Unpacking base classes</echo>
 		<copy file="Misc/JavaxComOrgInfoNetClasses.zip" todir="${release.build.dir}"/>
+		<copy file="log4j.properties" todir="${release.build.dir}"/>
 		<unzip src="${release.build.dir}/JavaxComOrgInfoNetClasses.zip" dest="${release.build.dir}"/>
 		<move todir="${release.build.dir}">
 			<fileset dir="${release.build.dir}/JavaxComOrgInfoNetClasses"/>
diff --git a/log4j.properties b/log4j.properties
new file mode 100644
index 00000000..5e798180
--- /dev/null
+++ b/log4j.properties
@@ -0,0 +1,9 @@
+# Root logger option
+log4j.rootLogger=INFO, stdout
+
+# Direct log messages to stdout
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.Target=System.out
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
+ 
\ No newline at end of file
