Merge pull request #191 from daveneiman/fix_jpeg_2000_handling

fix identification of JP2000 files when not well-formed
harvard-lts · Apr 9, 2019 · aa678b2 · aa678b2
2 parents 5b7e573 + c69946b
commit aa678b2
Show file tree

Hide file tree

Showing 8 changed files with 202 additions and 61 deletions.
diff --git a/src/test/java/edu/harvard/hul/ois/fits/junit/MixTest.java b/src/test/java/edu/harvard/hul/ois/fits/junit/MixTest.java
@@ -19,7 +19,10 @@
 package edu.harvard.hul.ois.fits.junit;
 
 import java.io.File;
+import java.util.Scanner;
 
+import org.jdom.output.Format;
+import org.jdom.output.XMLOutputter;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
@@ -153,5 +156,17 @@ public void testJp2_2() throws Exception {
 		fitsOut.addStandardCombinedFormat(); // output all data to file
 		fitsOut.saveToDisk("test-generated-output/" + inputFilename + "_Output.xml");
 	}
+
+	@Test
+	public void testNotWellFormedJp2() throws Exception {
+
+		String inputFilename = "2339337_not_well_formed.jp2";
+    	File input = new File("testfiles/" + inputFilename);
+
+    	FitsOutput fitsOut = fits.examine(input);
+
+		fitsOut.addStandardCombinedFormat(); // output all data to file
+		fitsOut.saveToDisk("test-generated-output/" + inputFilename + "_Output.xml");
+	}
 
 }
diff --git a/src/test/java/edu/harvard/hul/ois/fits/junit/MixXmlUnitTest.java b/src/test/java/edu/harvard/hul/ois/fits/junit/MixXmlUnitTest.java
@@ -269,5 +269,29 @@ public void testJp2_2() throws Exception {
 
 		testActualAgainstExpected(actualXmlStr, expectedXmlStr, inputFilename);
 	}
+
+	@Test
+	public void testNotWellFormedJp2() throws Exception {
+
+		String inputFilename = "2339337_not_well_formed.jp2";
+    	File input = new File("testfiles/" + inputFilename);
+
+    	FitsOutput fitsOut = fits.examine(input);
+
+		fitsOut.addStandardCombinedFormat(); // output all data to file
+		fitsOut.saveToDisk("test-generated-output/" + inputFilename + ACTUAL_OUTPUT_FILE_SUFFIX);
+
+		XMLOutputter serializer = new XMLOutputter(Format.getPrettyFormat());
+		String actualXmlStr = serializer.outputString(fitsOut.getFitsXml());
+
+		// Read in the expected XML file
+		Scanner scan = new Scanner(new File(
+				"testfiles/output/" + inputFilename + EXPECTED_OUTPUT_FILE_SUFFIX));
+		String expectedXmlStr = scan.
+				useDelimiter("\\Z").next();
+		scan.close();
+
+		testActualAgainstExpected(actualXmlStr, expectedXmlStr, inputFilename);
+	}
 
 }
diff --git a/testfiles/2339337_not_well_formed.jp2 b/testfiles/2339337_not_well_formed.jp2
diff --git a/testfiles/output/006607203_00018.jp2_XmlUnitExpectedOutput.xml b/testfiles/output/006607203_00018.jp2_XmlUnitExpectedOutput.xml
@@ -1,12 +1,13 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <fits xmlns="http://hul.harvard.edu/ois/xml/ns/fits/fits_output" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://hul.harvard.edu/ois/xml/ns/fits/fits_output http://hul.harvard.edu/ois/xml/xsd/fits/fits_output.xsd" version="1.2.x" timestamp="2/13/18 2:58 PM">
   <identification>
-    <identity format="JPEG 2000 JP2" mimetype="image/jp2" toolname="FITS" toolversion="1.2.x">
-      <tool toolname="Droid" toolversion="6.3" />
-      <tool toolname="Jhove" toolversion="1.16" />
-      <tool toolname="Exiftool" toolversion="10.00" />
-      <tool toolname="Tika" toolversion="1.10" />
-      <externalIdentifier toolname="Droid" toolversion="6.3" type="puid">x-fmt/392</externalIdentifier>
+    <identity format="JPEG 2000 JP2" mimetype="image/jp2" toolname="FITS" toolversion="1.4.1">
+      <tool toolname="Droid" toolversion="6.4" />
+      <tool toolname="Jhove" toolversion="1.20.1" />
+      <tool toolname="file utility" toolversion="5.31" />
+      <tool toolname="Exiftool" toolversion="11.14" />
+      <tool toolname="Tika" toolversion="1.19.1" />
+      <externalIdentifier toolname="Droid" toolversion="6.4" type="puid">x-fmt/392</externalIdentifier>
     </identity>
   </identification>
   <fileinfo>

diff --git a/testfiles/output/2339337_not_well_formed.jp2_XmlUnitExpectedOutput.xml b/testfiles/output/2339337_not_well_formed.jp2_XmlUnitExpectedOutput.xml
@@ -0,0 +1,93 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<fits xmlns="http://hul.harvard.edu/ois/xml/ns/fits/fits_output" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://hul.harvard.edu/ois/xml/ns/fits/fits_output http://hul.harvard.edu/ois/xml/xsd/fits/fits_output.xsd" version="1.4.0-SNAPSHOT" timestamp="4/2/19 4:22 PM">
+  <identification>
+    <identity format="JPEG 2000 JP2" mimetype="image/jp2" toolname="FITS" toolversion="1.4.1">
+      <tool toolname="Droid" toolversion="6.4" />
+      <tool toolname="Jhove" toolversion="1.20.1" />
+      <tool toolname="file utility" toolversion="5.31" />
+      <tool toolname="Exiftool" toolversion="11.14" />
+      <tool toolname="Tika" toolversion="1.19.1" />
+      <externalIdentifier toolname="Droid" toolversion="6.4" type="puid">x-fmt/392</externalIdentifier>
+    </identity>
+  </identification>
+  <fileinfo>
+    <size toolname="Jhove" toolversion="1.20.1">120521</size>
+    <filepath toolname="OIS File Information" toolversion="0.2" status="SINGLE_RESULT">/Users/dan179/git/git-daveneiman/fits/testfiles/2339337_not_well_formed.jp2</filepath>
+    <filename toolname="OIS File Information" toolversion="0.2" status="SINGLE_RESULT">2339337_not_well_formed.jp2</filename>
+    <md5checksum toolname="OIS File Information" toolversion="0.2" status="SINGLE_RESULT">1c9c7abb1cf187de78f550ec474f9952</md5checksum>
+    <fslastmodified toolname="OIS File Information" toolversion="0.2" status="SINGLE_RESULT">1554236769000</fslastmodified>
+  </fileinfo>
+  <filestatus>
+    <well-formed toolname="Jhove" toolversion="1.20.1" status="SINGLE_RESULT">false</well-formed>
+    <valid toolname="Jhove" toolversion="1.20.1" status="SINGLE_RESULT">false</valid>
+    <message toolname="Jhove" toolversion="1.20.1" status="SINGLE_RESULT">First box of JP2 header must be image header severity=error offset=48</message>
+  </filestatus>
+  <metadata>
+    <image>
+      <compressionScheme toolname="Exiftool" toolversion="11.14" status="SINGLE_RESULT">JPEG 2000</compressionScheme>
+      <imageWidth toolname="Exiftool" toolversion="11.14" status="SINGLE_RESULT">3039</imageWidth>
+      <imageHeight toolname="Exiftool" toolversion="11.14" status="SINGLE_RESULT">2490</imageHeight>
+      <iccProfileName toolname="Exiftool" toolversion="11.14" status="SINGLE_RESULT">Adobe RGB (1998)</iccProfileName>
+      <samplingFrequencyUnit toolname="Exiftool" toolversion="11.14" status="SINGLE_RESULT">cm</samplingFrequencyUnit>
+      <xSamplingFrequency toolname="Exiftool" toolversion="11.14" status="SINGLE_RESULT">3</xSamplingFrequency>
+      <ySamplingFrequency toolname="Exiftool" toolversion="11.14" status="SINGLE_RESULT">3</ySamplingFrequency>
+      <iccProfileVersion toolname="Exiftool" toolversion="11.14" status="SINGLE_RESULT">2.1.0</iccProfileVersion>
+      <standard>
+        <mix:mix xmlns:mix="http://www.loc.gov/mix/v20">
+          <mix:BasicDigitalObjectInformation>
+            <mix:Compression>
+              <mix:compressionScheme>JPEG 2000</mix:compressionScheme>
+            </mix:Compression>
+          </mix:BasicDigitalObjectInformation>
+          <mix:BasicImageInformation>
+            <mix:BasicImageCharacteristics>
+              <mix:imageWidth>3039</mix:imageWidth>
+              <mix:imageHeight>2490</mix:imageHeight>
+              <mix:PhotometricInterpretation>
+                <mix:ColorProfile>
+                  <mix:IccProfile>
+                    <mix:iccProfileName>Adobe RGB (1998)</mix:iccProfileName>
+                    <mix:iccProfileVersion>2.1.0</mix:iccProfileVersion>
+                  </mix:IccProfile>
+                </mix:ColorProfile>
+              </mix:PhotometricInterpretation>
+            </mix:BasicImageCharacteristics>
+          </mix:BasicImageInformation>
+          <mix:ImageCaptureMetadata>
+            <mix:GeneralCaptureInformation />
+          </mix:ImageCaptureMetadata>
+          <mix:ImageAssessmentMetadata>
+            <mix:SpatialMetrics>
+              <mix:samplingFrequencyUnit>cm</mix:samplingFrequencyUnit>
+              <mix:xSamplingFrequency>
+                <mix:numerator>3</mix:numerator>
+                <mix:denominator>1</mix:denominator>
+              </mix:xSamplingFrequency>
+              <mix:ySamplingFrequency>
+                <mix:numerator>3</mix:numerator>
+                <mix:denominator>1</mix:denominator>
+              </mix:ySamplingFrequency>
+            </mix:SpatialMetrics>
+            <mix:ImageColorEncoding />
+          </mix:ImageAssessmentMetadata>
+        </mix:mix>
+      </standard>
+    </image>
+  </metadata>
+  <statistics fitsExecutionTime="713">
+    <tool toolname="MediaInfo" toolversion="0.7.75" status="did not run" />
+    <tool toolname="OIS Audio Information" toolversion="0.1" status="did not run" />
+    <tool toolname="ADL Tool" toolversion="0.1" status="did not run" />
+    <tool toolname="VTT Tool" toolversion="0.1" status="did not run" />
+    <tool toolname="Droid" toolversion="6.4" executionTime="145" />
+    <tool toolname="Jhove" toolversion="1.20.1" executionTime="583" />
+    <tool toolname="file utility" toolversion="5.31" executionTime="643" />
+    <tool toolname="Exiftool" toolversion="11.14" executionTime="660" />
+    <tool toolname="NLNZ Metadata Extractor" toolversion="3.6GA" status="did not run" />
+    <tool toolname="OIS File Information" toolversion="0.2" executionTime="145" />
+    <tool toolname="OIS XML Metadata" toolversion="0.2" status="did not run" />
+    <tool toolname="ffident" toolversion="0.2" executionTime="497" />
+    <tool toolname="Tika" toolversion="1.19.1" executionTime="185" />
+  </statistics>
+</fits>
+
diff --git a/testfiles/output/test.jp2_XmlUnitExpectedOutput.xml b/testfiles/output/test.jp2_XmlUnitExpectedOutput.xml
@@ -1,50 +1,51 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<fits xmlns="http://hul.harvard.edu/ois/xml/ns/fits/fits_output" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://hul.harvard.edu/ois/xml/ns/fits/fits_output http://hul.harvard.edu/ois/xml/xsd/fits/fits_output.xsd" version="1.2.x" timestamp="2/13/18 2:53 PM">
+<fits xmlns="http://hul.harvard.edu/ois/xml/ns/fits/fits_output" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://hul.harvard.edu/ois/xml/ns/fits/fits_output http://hul.harvard.edu/ois/xml/xsd/fits/fits_output.xsd" version="1.4.1-SNAPSHOT" timestamp="4/2/19 4:06 PM">
   <identification>
-    <identity format="JPEG 2000 JP2" mimetype="image/jp2" toolname="FITS" toolversion="1.2.x">
-      <tool toolname="Droid" toolversion="6.3" />
-      <tool toolname="Jhove" toolversion="1.16" />
-      <tool toolname="Exiftool" toolversion="10.00" />
-      <tool toolname="Tika" toolversion="1.10" />
-      <externalIdentifier toolname="Droid" toolversion="6.3" type="puid">x-fmt/392</externalIdentifier>
+    <identity format="JPEG 2000 JP2" mimetype="image/jp2" toolname="FITS" toolversion="1.4.1-SNAPSHOT">
+      <tool toolname="Droid" toolversion="6.4" />
+      <tool toolname="Jhove" toolversion="1.20.1" />
+      <tool toolname="file utility" toolversion="5.31" />
+      <tool toolname="Exiftool" toolversion="11.14" />
+      <tool toolname="Tika" toolversion="1.19.1" />
+      <externalIdentifier toolname="Droid" toolversion="6.4" type="puid">x-fmt/392</externalIdentifier>
     </identity>
   </identification>
   <fileinfo>
-    <size toolname="Jhove" toolversion="1.16">253214</size>
-    <creatingApplicationName toolname="Exiftool" toolversion="10.00" status="SINGLE_RESULT">Photostation v1</creatingApplicationName>
-    <created toolname="Exiftool" toolversion="10.00" status="SINGLE_RESULT">2007:09:21 00:00:00+08:00</created>
+    <size toolname="Jhove" toolversion="1.20.1">253214</size>
+    <creatingApplicationName toolname="Exiftool" toolversion="11.14" status="SINGLE_RESULT">Photostation v1</creatingApplicationName>
+    <created toolname="Exiftool" toolversion="11.14" status="SINGLE_RESULT">2007:09:21 00:00:00+08:00</created>
     <filepath toolname="OIS File Information" toolversion="0.2" status="SINGLE_RESULT">/Users/dan179/git/git-daveneiman/fits/testfiles/test.jp2</filepath>
     <filename toolname="OIS File Information" toolversion="0.2" status="SINGLE_RESULT">test.jp2</filename>
     <md5checksum toolname="OIS File Information" toolversion="0.2" status="SINGLE_RESULT">a9f2c77997e438c97639ad3a843f4fdb</md5checksum>
     <fslastmodified toolname="OIS File Information" toolversion="0.2" status="SINGLE_RESULT">1446674963000</fslastmodified>
   </fileinfo>
   <filestatus>
-    <well-formed toolname="Jhove" toolversion="1.16" status="SINGLE_RESULT">true</well-formed>
-    <valid toolname="Jhove" toolversion="1.16" status="SINGLE_RESULT">true</valid>
+    <well-formed toolname="Jhove" toolversion="1.20.1" status="SINGLE_RESULT">true</well-formed>
+    <valid toolname="Jhove" toolversion="1.20.1" status="SINGLE_RESULT">true</valid>
   </filestatus>
   <metadata>
     <image>
-      <byteOrder toolname="Jhove" toolversion="1.16" status="SINGLE_RESULT">big endian</byteOrder>
-      <compressionScheme toolname="Jhove" toolversion="1.16" status="CONFLICT">JPEG 2000 Lossy</compressionScheme>
-      <compressionScheme toolname="Exiftool" toolversion="10.00" status="CONFLICT">JPEG 2000</compressionScheme>
-      <imageWidth toolname="Jhove" toolversion="1.16">1542</imageWidth>
-      <imageHeight toolname="Jhove" toolversion="1.16">2464</imageHeight>
-      <colorSpace toolname="Jhove" toolversion="1.16">sRGB</colorSpace>
-      <tileWidth toolname="Jhove" toolversion="1.16" status="SINGLE_RESULT">1024</tileWidth>
-      <tileHeight toolname="Jhove" toolversion="1.16" status="SINGLE_RESULT">1024</tileHeight>
-      <qualityLayers toolname="Jhove" toolversion="1.16" status="SINGLE_RESULT">8</qualityLayers>
-      <resolutionLevels toolname="Jhove" toolversion="1.16" status="SINGLE_RESULT">5</resolutionLevels>
-      <orientation toolname="Exiftool" toolversion="10.00" status="SINGLE_RESULT">normal*</orientation>
-      <samplingFrequencyUnit toolname="Jhove" toolversion="1.16" status="CONFLICT">cm</samplingFrequencyUnit>
-      <samplingFrequencyUnit toolname="Exiftool" toolversion="10.00" status="CONFLICT">0.1 mm</samplingFrequencyUnit>
-      <xSamplingFrequency toolname="Jhove" toolversion="1.16" status="CONFLICT">118</xSamplingFrequency>
-      <xSamplingFrequency toolname="Exiftool" toolversion="10.00" status="CONFLICT">1.181102</xSamplingFrequency>
-      <ySamplingFrequency toolname="Jhove" toolversion="1.16" status="CONFLICT">118</ySamplingFrequency>
-      <ySamplingFrequency toolname="Exiftool" toolversion="10.00" status="CONFLICT">1.181102</ySamplingFrequency>
-      <bitsPerSample toolname="Jhove" toolversion="1.16">8 8 8</bitsPerSample>
-      <samplesPerPixel toolname="Jhove" toolversion="1.16">3</samplesPerPixel>
-      <scannerManufacturer toolname="Exiftool" toolversion="10.00" status="SINGLE_RESULT">Google</scannerManufacturer>
-      <scannerModelName toolname="Exiftool" toolversion="10.00" status="SINGLE_RESULT">Photostation v1</scannerModelName>
+      <byteOrder toolname="Jhove" toolversion="1.20.1" status="SINGLE_RESULT">big endian</byteOrder>
+      <compressionScheme toolname="Jhove" toolversion="1.20.1" status="CONFLICT">JPEG 2000 Lossy</compressionScheme>
+      <compressionScheme toolname="Exiftool" toolversion="11.14" status="CONFLICT">JPEG 2000</compressionScheme>
+      <imageWidth toolname="Jhove" toolversion="1.20.1">1542</imageWidth>
+      <imageHeight toolname="Jhove" toolversion="1.20.1">2464</imageHeight>
+      <colorSpace toolname="Jhove" toolversion="1.20.1">sRGB</colorSpace>
+      <tileWidth toolname="Jhove" toolversion="1.20.1" status="SINGLE_RESULT">1024</tileWidth>
+      <tileHeight toolname="Jhove" toolversion="1.20.1" status="SINGLE_RESULT">1024</tileHeight>
+      <qualityLayers toolname="Jhove" toolversion="1.20.1" status="SINGLE_RESULT">8</qualityLayers>
+      <resolutionLevels toolname="Jhove" toolversion="1.20.1" status="SINGLE_RESULT">5</resolutionLevels>
+      <orientation toolname="Exiftool" toolversion="11.14" status="SINGLE_RESULT">normal*</orientation>
+      <samplingFrequencyUnit toolname="Jhove" toolversion="1.20.1" status="CONFLICT">cm</samplingFrequencyUnit>
+      <samplingFrequencyUnit toolname="Exiftool" toolversion="11.14" status="CONFLICT">0.1 mm</samplingFrequencyUnit>
+      <xSamplingFrequency toolname="Jhove" toolversion="1.20.1" status="CONFLICT">118</xSamplingFrequency>
+      <xSamplingFrequency toolname="Exiftool" toolversion="11.14" status="CONFLICT">1.181102</xSamplingFrequency>
+      <ySamplingFrequency toolname="Jhove" toolversion="1.20.1" status="CONFLICT">118</ySamplingFrequency>
+      <ySamplingFrequency toolname="Exiftool" toolversion="11.14" status="CONFLICT">1.181102</ySamplingFrequency>
+      <bitsPerSample toolname="Jhove" toolversion="1.20.1">8 8 8</bitsPerSample>
+      <samplesPerPixel toolname="Jhove" toolversion="1.20.1">3</samplesPerPixel>
+      <scannerManufacturer toolname="Exiftool" toolversion="11.14" status="SINGLE_RESULT">Google</scannerManufacturer>
+      <scannerModelName toolname="Exiftool" toolversion="11.14" status="SINGLE_RESULT">Photostation v1</scannerModelName>
       <standard>
         <mix:mix xmlns:mix="http://www.loc.gov/mix/v20">
           <mix:BasicDigitalObjectInformation>
@@ -112,20 +113,20 @@
       </standard>
     </image>
   </metadata>
-  <statistics fitsExecutionTime="725">
+  <statistics fitsExecutionTime="214">
     <tool toolname="MediaInfo" toolversion="0.7.75" status="did not run" />
     <tool toolname="OIS Audio Information" toolversion="0.1" status="did not run" />
     <tool toolname="ADL Tool" toolversion="0.1" status="did not run" />
     <tool toolname="VTT Tool" toolversion="0.1" status="did not run" />
-    <tool toolname="Droid" toolversion="6.3" executionTime="106" />
-    <tool toolname="Jhove" toolversion="1.16" executionTime="670" />
-    <tool toolname="file utility" toolversion="5.31" executionTime="488" />
-    <tool toolname="Exiftool" toolversion="10.00" executionTime="573" />
+    <tool toolname="Droid" toolversion="6.4" executionTime="9" />
+    <tool toolname="Jhove" toolversion="1.20.1" executionTime="171" />
+    <tool toolname="file utility" toolversion="5.31" executionTime="128" />
+    <tool toolname="Exiftool" toolversion="11.14" executionTime="201" />
     <tool toolname="NLNZ Metadata Extractor" toolversion="3.6GA" status="did not run" />
-    <tool toolname="OIS File Information" toolversion="0.2" executionTime="102" />
+    <tool toolname="OIS File Information" toolversion="0.2" executionTime="5" />
     <tool toolname="OIS XML Metadata" toolversion="0.2" status="did not run" />
-    <tool toolname="ffident" toolversion="0.2" executionTime="446" />
-    <tool toolname="Tika" toolversion="1.10" executionTime="150" />
+    <tool toolname="ffident" toolversion="0.2" executionTime="9" />
+    <tool toolname="Tika" toolversion="1.19.1" executionTime="102" />
   </statistics>
 </fits>
 
diff --git a/xml/fileutility/fileutility_to_fits.xslt b/xml/fileutility/fileutility_to_fits.xslt
@@ -271,7 +271,7 @@
 					<!--  JPEG 2000 image data -->
 					<xsl:when test="$mime='image/jp2'">
 						<xsl:attribute name="format">
-							<xsl:value-of select="string('JPEG 2000')"/>
+							<xsl:value-of select="string('JPEG 2000 JP2')"/>
 						</xsl:attribute>				
 					</xsl:when>							
 					<!-- RIFF (little-endian) data, WAVE audio, Microsoft PCM, 24 bit, mono 96000 Hz -->