15.0.62

divonlan · Jun 29, 2024 · 7df14ef · 7df14ef
1 parent 10b9229
commit 7df14ef
Show file tree

Hide file tree

Showing 83 changed files with 5,044 additions and 4,657 deletions.
diff --git a/.gitattributes b/.gitattributes
@@ -1,49 +1,50 @@
-# Set the default behavior, in case people don't have core.autocrlf set.
-* text=auto
-
-*.sh eol=lf
-*.txt eol=lf
-*.md eol=lf
-*.eb eol=lf
-*.json eol=lf
-*.sed eol=lf
-*.c eol=lf
-*.cpp eol=lf
-*.h eol=lf
-*.asm eol=lf
-*.S eol=lf
-Makefile eol=lf
-
-# Declare our textual formats as binary so we can check in test files with Unix or Windows style end-of-lines
-*.vcf binary
-*.bcf binary
-*.sam binary
-*.bam binary
-*.cram binary
-*.gff3 binary
-*.gff binary
-*.gvf binary
-*.gtf binary
-genome_*.txt binary
-*.fa binary
-*.fasta binary
-*.fa binary
-*.faa binary
-*.ffn binary
-*.fnn binary
-*.fna binary
-*.frn binary
-*.fas binary
-*.fq binary
-*.fastq binary
-*.phy binary
-*.bed binary
-*.kraken binary
-*.chain binary
-*.locs binary
-*.genozip binary
-
-# installer files
-*.tar binary
-*.exe binary
-*.o binary
+# Set the default behavior, in case people don't have core.autocrlf set.
+* text=lf
+
+*.sh eol=lf
+*.txt eol=lf
+*.md eol=lf
+*.eb eol=lf
+*.json eol=lf
+*.sed eol=lf
+*.c eol=lf
+*.cpp eol=lf
+*.h eol=lf
+*.asm eol=lf
+*.S eol=lf
+*.yaml eol=lf
+Makefile eol=lf
+
+# Declare our textual formats as binary so we can check in test files with Unix or Windows style end-of-lines
+*.vcf binary
+*.bcf binary
+*.sam binary
+*.bam binary
+*.cram binary
+*.gff3 binary
+*.gff binary
+*.gvf binary
+*.gtf binary
+genome_*.txt binary
+*.fa binary
+*.fasta binary
+*.fa binary
+*.faa binary
+*.ffn binary
+*.fnn binary
+*.fna binary
+*.frn binary
+*.fas binary
+*.fq binary
+*.fastq binary
+*.phy binary
+*.bed binary
+*.kraken binary
+*.chain binary
+*.locs binary
+*.genozip binary
+
+# installer files
+*.tar binary
+*.exe binary
+*.o binary
diff --git a/LICENSE.txt b/LICENSE.txt
@@ -159,5 +159,5 @@ ABOVE STATED REMEDY FAILS OF ITS ESSENTIAL PURPOSE.
 
 END OF TERMS AND CONDITIONS
 
-Genozip license version: 15.0.61
+Genozip license version: 15.0.62
 
diff --git a/RELEASE_NOTES.txt b/RELEASE_NOTES.txt
@@ -3,6 +3,12 @@ Note on versioning:
 - Minor version changes with bug fixes and minor feature updates
 - Some minor versions are skipped due to failed deployment pipelines
 
+15.0.62
+- I/O optimizations for faster compression
+- Bug fixes
+- New diagnostic options: --show-gz-uncomp, --generate-gzil
+- Removed bash autocomplete for genozip as it didn't work very well. If this was installed, it can be removed by manually editing ~/.bash_completion
+
 15.0.61 22/6/2024
 - --optimize can now take an optional argument for fine-grained control of which fields get optimized: --optimize=QUAL,rx:f (optimize if possible, but only these fields) or --optimize=^QUAL,rx:f (optimize all fields possible, except for these fields)
 - VCF: better compression of files generated by freebayes ; better compression of Type=Float annotations

diff --git a/installers/LICENSE.html b/installers/LICENSE.html
@@ -34,4 +34,4 @@
 10. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides Genozip on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Genozip and assume any risks associated with Your exercise of permissions under this License.<br><br>
 11. LIMITATION OF LIABILITY. TO THE FULLEST EXTENT PERMITTED BY APPLICABLE LAW, IN NO EVENT AND UNDER NO LEGAL THEORY, WHETHER IN TORT (INCLUDING NEGLIGENCE), CONTRACT, STRICT LIABILITY OR OTHER LEGAL OR EQUITABLE THEORY, SHALL LICENSOR OR DEVELOPER BE LIABLE FOR DAMAGES, INCLUDING ANY DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES OF ANY CHARACTER ARISING AS A RESULT OF THIS LICENSE OR OUT OF THE USE OR INABILITY TO USE GENOZIP (INCLUDING BUT NOT LIMITED TO DAMAGES FOR LOSS OF GOODWILL, WORK STOPPAGE, COMPUTER FAILURE OR MALFUNCTION, FILE CORRUPTION, DATA LOSS, OR ANY AND ALL OTHER COMMERCIAL DAMAGES OR LOSSES), EVEN IF LICENSOR OR DEVELOPER HAVE BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.  IN NO EVENT WILL LICENSOR'S OR DEVELOPER'S TOTAL LIABILITY TO LICENSEE FOR ALL DAMAGES (OTHER THAN AS MAY BE REQUIRED BY APPLICABLE LAW IN CASES INVOLVING PERSONAL INJURY) EXCEED THE AMOUNT OF $500 USD. THE FOREGOING LIMITATIONS WILL APPLY EVEN IF THE ABOVE STATED REMEDY FAILS OF ITS ESSENTIAL PURPOSE.<br><br>
 END OF TERMS AND CONDITIONS<br><br>
-Genozip license version: 15.0.61<br><br>
+Genozip license version: 15.0.62<br><br>
diff --git a/installers/genozip-installer.exe b/installers/genozip-installer.exe
diff --git a/installers/genozip-linux-x86_64.tar b/installers/genozip-linux-x86_64.tar
diff --git a/installers/genozip-osx-arm.tar b/installers/genozip-osx-arm.tar
diff --git a/installers/genozip-osx-x86.tar b/installers/genozip-osx-x86.tar
diff --git a/src/Makefile b/src/Makefile
@@ -156,7 +156,7 @@ MY_SRCS = genozip.c genols.c context.c container.c strings.c stats.c arch.c tip.
 		  sam_blasr.c sam_dragen.c sam_minimap2.c sam_10xGenomics.c sam_biobambam.c sam_pos.c sam_deep.c 		\
 		  sam_star.c sam_abra2.c sam_optimize.c																	\
 		  fastq.c fastq_desc.c fastq_seq.c fastq_qual.c fastq_deep.c fastq_saux.c deep.c						\
-		  fasta.c gff.c bed.c me23.c locs.c generic.c lookback.c compressor.c									\
+		  fasta.c gff.c bed.c me23.c locs.c generic.c lookback.c compressor.c 							\
 		  buffer.c buf_struct.c buf_list.c random_access.c sections.c base64.c bgzf.c coverage.c txtheader.c  	\
 		  codec.c codec_bz2.c codec_lzma.c codec_acgt.c codec_domq.c codec_bsc.c codec_pacb.c					\
 		  codec_pbwt.c codec_none.c codec_htscodecs.c codec_longr.c codec_normq.c codec_homp.c codec_t0.c		\
@@ -193,7 +193,7 @@ INCLUDES += dict_id_gen.h aes.h dispatcher.h profiler.h dict_id.h aliases.h txtf
             buffer.h buf_struct.h buf_list.h file.h context.h context_struct.h container.h seg.h text_license.h version.h compressor.h 		\
             crypt.h genozip.h piz.h vblock.h zfile.h random_access.h regions.h reconstruct.h tar.h qname.h qname_flavors.h codec.h  		\
 		 	lookback.h tokenizer.h codec_longr_alg.c gencomp.h dict_io.h recon_plan_io.h tip.h deep.h filename.h stats.h multiplexer.h 		\
-		 	reference.h ref_private.h refhash.h ref_iupacs.h aligner.h mutex.h bgzf.h coverage.h threads.h local_type.h 					\
+		 	reference.h ref_private.h refhash.h ref_iupacs.h aligner.h mutex.h bgzf.h coverage.h threads.h local_type.h				\
 			arch.h license.h file_types.h data_types.h base64.h txtheader.h writer.h zriter.h bases_filter.h genols.h contigs.h chrom.h 	\
 			vcf.h vcf_private.h sam.h sam_private.h me23.h fasta.h fasta_private.h gff.h bed.h locs.h generic.h 							\
 			fastq.h fastq_private.h user_message.h mac_compat.h b250.h zip_dyn_int.h qname_filter.h 										\
@@ -335,9 +335,9 @@ DEBUG_OBJS := $(addprefix $(OBJDIR)/,$(C_SRCS:.c=.debug-o)) $(addprefix $(OBJDIR
 OPT_OBJS   := $(addprefix $(OBJDIR)/,$(C_SRCS:.c=.opt-o))   $(addprefix $(OBJDIR)/,$(CXX_SRCS:.cpp=.opt-o))   $(IGZIP_OBJS) $(BSC_OPT_OBJS)  # optimized but with debug info, for debugging issues that only manifest with compiler optimization
 DEPS       := $(addprefix $(OBJDIR)/,$(C_SRCS:.c=.d))       $(addprefix $(OBJDIR)/,$(CXX_SRCS:.cpp=.d))       $(IGZIP_DEPS)
 
-EXECUTABLES       = genozip$(EXE)       genounzip$(EXE)       genocat$(EXE)       genols$(EXE)        autocomplete.sh
-DEBUG_EXECUTABLES = genozip-debug$(EXE) genounzip-debug$(EXE) genocat-debug$(EXE) genols-debug$(EXE)  autocomplete-debug.sh
-OPT_EXECUTABLES   = genozip-opt$(EXE)   genounzip-opt$(EXE)   genocat-opt$(EXE)   genols-opt$(EXE)    autocomplete-debug.sh
+EXECUTABLES       = genozip$(EXE)       genounzip$(EXE)       genocat$(EXE)       genols$(EXE)        # autocomplete.sh
+DEBUG_EXECUTABLES = genozip-debug$(EXE) genounzip-debug$(EXE) genocat-debug$(EXE) genols-debug$(EXE)  # autocomplete-debug.sh
+OPT_EXECUTABLES   = genozip-opt$(EXE)   genounzip-opt$(EXE)   genocat-opt$(EXE)   genols-opt$(EXE)    # autocomplete-debug.sh
 
 all   : CFLAGS += $(OPTFLAGS) 
 all   : CXXFLAGS += $(OPTFLAGS) 
@@ -611,7 +611,7 @@ CONDA_RECIPE_DIR = $(CONDA_FEEDSTOCK)/recipe
 # publish to conda-forge 
 conda/.conda-timestamp: conda/meta.yaml conda/README.md conda/build.sh conda/bld.bat 
 	@echo "Publishing to conda-forge"
-	@$(SH_VERIFY_ALL_COMMITTED)
+	@$(SH_VERIFY_ALL_STAGED)
 	@echo " "
 	@echo "Copying $^ to conda feedstock"
 	@(cd $(CONDA_FEEDSTOCK); git reset --hard; git pull)

diff --git a/src/aes.c b/src/aes.c
@@ -255,12 +255,10 @@ void aes_initialize (VBlockP vb, bytes key)
 rom aes_display_data (bytes data, unsigned data_len)
 {
     char *str = MALLOC (data_len * 2 + 1);
-    return str_to_hex (STRa(data), str, false);
+    return str_to_hex_ (STRa(data), str, false);
 }
 
 StrText aes_display_key (bytes key) 
 { 
-    StrText s;
-    str_to_hex (key, AES_KEYLEN, s.s, false);
-    return s;
+    return str_to_hex (key, AES_KEYLEN);
 }
diff --git a/src/aliases.c b/src/aliases.c
@@ -19,12 +19,10 @@ void show_aliases (void)
         iprint0 ("No aliases in this file\n");
 
     else {
-        static rom names[] = ALIAS_TYPE_NAMES;
-
         iprintf ("Contents of SEC_DICT_ID_ALIASES section (num_aliases=%u):\n", z_file->aliases.len32);
 
         for_buf (DictIdAlias, alias, z_file->aliases)
-            iprintf ("type=%-4s\talias=%s/%-8s\tdst=%s/%-8s\n", names[alias->alias_type],
+            iprintf ("type=%-4s\talias=%s/%-8s\tdst=%s/%-8s\n", (rom[])ALIAS_TYPE_NAMES[alias->alias_type],
                     dtype_name_z (alias->alias), dis_dict_id (alias->alias).s, 
                     dtype_name_z (alias->dst),   dis_dict_id (alias->dst).s);
     }

diff --git a/src/bam_seg.c b/src/bam_seg.c
@@ -218,7 +218,7 @@ void bam_seg_BIN (VBlockSAMP vb, ZipDataLineSAMP dl, uint16_t bin /* used only i
 
 static inline void bam_seg_ref_id (VBlockSAMP vb, ZipDataLineSAMP dl, Did did_i, int32_t ref_id, int32_t compare_to_ref_i)
 {
-    ASSERT (ref_id == -1 || (sam_hdr_contigs && ref_id >= 0 && ref_id < (int32_t)sam_hdr_contigs->contigs.len), 
+    ASSERT (ref_id == -1 || (sam_hdr_contigs && IN_RANGE (ref_id, 0, sam_hdr_contigs->contigs.len32-1)), 
             "%s: encountered %s.ref_id=%d but header has only %u contigs%s", 
             LN_NAME, CTX(did_i)->tag_name, ref_id, sam_hdr_contigs ? sam_hdr_contigs->contigs.len32 : 0,
             MP(LONGRANGER) ? ". This is a known longranger bug (samtools won't accept this file either)." : "");
@@ -412,8 +412,8 @@ rom bam_seg_txt_line (VBlockP vb_, rom alignment /* BAM terminology for one line
 
     // a non-sensical block_size might indicate an false-positive identification of a BAM alignment in bam_unconsumed
     ASSERT (block_size + 4 >= sizeof (BAMAlignmentFixed) && block_size + 4 <= remaining_txt_len, 
-            "%s: (block_size+4)=%u is out of range - too small, or goes beyond end of txt data: remaining_txt_len=%u",
-            LN_NAME, block_size+4, remaining_txt_len);
+            "%s: (block_size+4)=%u is out of range - too small, or goes beyond end of txt data: txt_data.len=%u remaining_txt_len=%u",
+            LN_NAME, block_size+4, vb->txt_data.len32, remaining_txt_len);
 
     rom after = alignment + block_size + sizeof (uint32_t);
Original file line number	Diff line number	Diff line change
Expand Up		@@ -159,5 +159,5 @@ ABOVE STATED REMEDY FAILS OF ITS ESSENTIAL PURPOSE.

		END OF TERMS AND CONDITIONS

		Genozip license version: 15.0.61
		Genozip license version: 15.0.62