Merge pull request #30 from qqia/release_1.0.6

Release 1.0.6

README.md

@@ -105,12 +105,17 @@ with the AWS SDK for the Kinesis Video. This example provides examples for
 
 ## Release Notes
 
+### Release 1.0.6 (Sep 2018)
+* Introduce handling for empty fragment metadata
+* Added a new SimpleFrame Visitor to handle video with no tags
+* Refactored H264FrameDecoder, so that base class method can be reused by child class
+
 ### Release 1.0.5 (May 2018)
-* Introduce `GetMediaResponseStreamConsumer` as an abstract class used to consume the output of a GetMedia* call 
+* Introduce `GetMediaResponseStreamConsumer` as an abstract class used to consume the output of a GetMedia* call
 to Kinesis Video in a streaming fashion. Child classes will use visitors to implement different consumers.
 * The `MergedOutputPiper` extends `GetMediaResponseStreamConsumer` to merge consecutive mkv streams in the output of GetMedia
-   and pipes the merged stream to the stdin of a child process. 
-* Add the capability and example to pipe the output of GetMedia calls to GStreamer using `MergedOutputPiper`. 
+   and pipes the merged stream to the stdin of a child process.
+* Add the capability and example to pipe the output of GetMedia calls to GStreamer using `MergedOutputPiper`.
 
 ### Release 1.0.4 (April 2018)
 * Add example for KinesisVideo Streams integration with Rekognition and draw Bounding Boxes for every sampled frame.

pom.xml

@@ -11,7 +11,7 @@
     <artifactId>amazon-kinesis-video-streams-parser-library</artifactId>
     <packaging>jar</packaging>
     <name>Amazon Kinesis Video Streams Parser Library</name>
-    <version>1.0.5-SNAPSHOT</version>
+    <version>1.0.6</version>
     <description>The Amazon Kinesis Video Streams Parser Library for Java enables Java developers to parse the streams
         returned by GetMedia calls to Amazon Kinesis Video.
     </description>
@@ -107,6 +107,13 @@
             <version>2.8.1</version>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.powermock</groupId>
+            <artifactId>powermock-mockito-release-full</artifactId>
+            <version>1.6.3</version>
+            <type>pom</type>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
 
     <developers>

src/main/java/com/amazonaws/kinesisvideo/parser/utilities/FragmentMetadata.java

@@ -94,11 +94,12 @@ static FragmentMetadata createFromtagNametoValueMap(final Map<String, String> ta
             return new FragmentMetadata(getValueForTag(tagNameToTagValueMap, FRAGMENT_NUMBER_KEY),
                     Double.parseDouble(getValueForTag(tagNameToTagValueMap, SERVER_SIDE_TIMESTAMP_KEY)),
                     Double.parseDouble(getValueForTag(tagNameToTagValueMap, PRODCUER_SIDE_TIMESTAMP_KEY)));
-        } else {
+        } else if (tagNameToTagValueMap.containsKey(FRAGMENT_NUMBER_KEY)) {
             return new FragmentMetadata(getValueForTag(tagNameToTagValueMap, FRAGMENT_NUMBER_KEY),
                     Long.parseLong(getValueForTag(tagNameToTagValueMap, ERROR_ID_KEY)),
                     getValueForTag(tagNameToTagValueMap, ERROR_CODE_KEY));
         }
+        return null;
     }
 
     private static String getValueForTag(Map<String, String> tagNameToTagValueMap, String tagName) {

src/main/java/com/amazonaws/kinesisvideo/parser/utilities/FragmentMetadataVisitor.java

@@ -207,7 +207,7 @@ private void setMillisBehindLatestAndContinuationToken() {
     private void collectPreClusterInfo() {
         final Map<String, String> tagNameToTagValueMap = getTagNameToValueMap();
 
-        currentFragmentMetadata = Optional.of(FragmentMetadata.createFromtagNametoValueMap(tagNameToTagValueMap));
+        currentFragmentMetadata = Optional.ofNullable(FragmentMetadata.createFromtagNametoValueMap(tagNameToTagValueMap));
 
         final Map<Long, List<MkvElement>> trackEntryElementNumberToMkvElement = getTrackEntryMap();
         trackEntryElementNumberToMkvElement.values().stream().forEach(this::createTrackMetadata);

src/main/java/com/amazonaws/kinesisvideo/parser/utilities/FrameVisitor.java

@@ -74,4 +74,4 @@ public void visit(com.amazonaws.kinesisvideo.parser.mkv.MkvDataElement dataEleme
             }
         }
     }
-}
+}

src/main/java/com/amazonaws/kinesisvideo/parser/utilities/H264FrameDecoder.java

@@ -0,0 +1,90 @@
+/*
+Copyright 2017-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"). 
+You may not use this file except in compliance with the License. 
+A copy of the License is located at
+
+   http://aws.amazon.com/apache2.0/
+
+or in the "license" file accompanying this file. 
+This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and limitations under the License.
+*/
+package com.amazonaws.kinesisvideo.parser.utilities;
+
+import com.amazonaws.kinesisvideo.parser.mkv.Frame;
+import lombok.Getter;
+import lombok.extern.slf4j.Slf4j;
+import org.jcodec.codecs.h264.H264Decoder;
+import org.jcodec.codecs.h264.mp4.AvcCBox;
+import org.jcodec.common.model.ColorSpace;
+import org.jcodec.common.model.Picture;
+import org.jcodec.scale.AWTUtil;
+import org.jcodec.scale.Transform;
+import org.jcodec.scale.Yuv420jToRgb;
+
+import java.awt.image.BufferedImage;
+import java.nio.ByteBuffer;
+import java.util.List;
+import java.util.Optional;
+
+import static org.jcodec.codecs.h264.H264Utils.splitMOVPacket;
+
+@Slf4j
+public class H264FrameDecoder implements FrameVisitor.FrameProcessor  {
+
+    private final H264Decoder decoder = new H264Decoder();
+    private final Transform transform = new Yuv420jToRgb();
+
+    @Getter
+    private int frameCount;
+
+    private byte[] codecPrivateData;
+
+    @Override
+    public void process(Frame frame, MkvTrackMetadata trackMetadata, Optional<FragmentMetadata> fragmentMetadata) {
+        decodeH264Frame(frame, trackMetadata);
+    }
+
+    protected BufferedImage decodeH264Frame(Frame frame, MkvTrackMetadata trackMetadata) {
+        ByteBuffer frameBuffer = frame.getFrameData();
+        int pixelWidth = trackMetadata.getPixelWidth().get().intValue();
+        int pixelHeight = trackMetadata.getPixelHeight().get().intValue();
+        codecPrivateData = trackMetadata.getCodecPrivateData().array();
+        log.debug("Decoding frames ... ");
+        // Read the bytes that appear to comprise the header
+        // See: https://www.matroska.org/technical/specs/index.html#simpleblock_structure
+
+        Picture rgb = Picture.create(pixelWidth, pixelHeight, ColorSpace.RGB);
+        BufferedImage bufferedImage = new BufferedImage(pixelWidth, pixelHeight, BufferedImage.TYPE_3BYTE_BGR);
+        AvcCBox avcC = AvcCBox.parseAvcCBox(ByteBuffer.wrap(codecPrivateData));
+
+        decoder.addSps(avcC.getSpsList());
+        decoder.addPps(avcC.getPpsList());
+
+        Picture buf = Picture.create(pixelWidth, pixelHeight, ColorSpace.YUV420J);
+        List<ByteBuffer> byteBuffers = splitMOVPacket(frameBuffer, avcC);
+        Picture pic = decoder.decodeFrameFromNals(byteBuffers, buf.getData());
+
+        if (pic != null) {
+            // Work around for color issues in JCodec
+            // https://github.com/jcodec/jcodec/issues/59
+            // https://github.com/jcodec/jcodec/issues/192
+            byte[][] dataTemp = new byte[3][pic.getData().length];
+            dataTemp[0] = pic.getPlaneData(0);
+            dataTemp[1] = pic.getPlaneData(2);
+            dataTemp[2] = pic.getPlaneData(1);
+
+            Picture tmpBuf = Picture.createPicture(pixelWidth, pixelHeight, dataTemp, ColorSpace.YUV420J);
+            transform.transform(tmpBuf, rgb);
+            AWTUtil.toBufferedImage(rgb, bufferedImage);
+            frameCount++;
+        }
+        return bufferedImage;
+    }
+
+    public ByteBuffer getCodecPrivateData() {
+        return ByteBuffer.wrap(codecPrivateData);
+    }
+}

src/main/java/com/amazonaws/kinesisvideo/parser/utilities/H264FrameRenderer.java

@@ -33,18 +33,12 @@
 import static org.jcodec.codecs.h264.H264Utils.splitMOVPacket;
 
 @Slf4j
-public class H264FrameRenderer implements FrameVisitor.FrameProcessor {
+public class H264FrameRenderer extends H264FrameDecoder {
 
     private final KinesisVideoFrameViewer kinesisVideoFrameViewer;
-    private final H264Decoder decoder = new H264Decoder();
-    private final Transform transform = new Yuv420jToRgb();
-
-    @Getter
-    private int frameCount;
-
-    private byte[] codecPrivateData;
 
     protected H264FrameRenderer(final KinesisVideoFrameViewer kinesisVideoFrameViewer) {
+        super();
         this.kinesisVideoFrameViewer = kinesisVideoFrameViewer;
         this.kinesisVideoFrameViewer.setVisible(true);
     }
@@ -58,46 +52,4 @@ public void process(Frame frame, MkvTrackMetadata trackMetadata, Optional<Fragme
         final BufferedImage bufferedImage = decodeH264Frame(frame, trackMetadata);
         kinesisVideoFrameViewer.update(bufferedImage);
     }
-
-    protected BufferedImage decodeH264Frame(Frame frame, MkvTrackMetadata trackMetadata) {
-        ByteBuffer frameBuffer = frame.getFrameData();
-        int pixelWidth = trackMetadata.getPixelWidth().get().intValue();
-        int pixelHeight = trackMetadata.getPixelHeight().get().intValue();
-        codecPrivateData = trackMetadata.getCodecPrivateData().array();
-        log.debug("Decoding frames ... ");
-        // Read the bytes that appear to comprise the header
-        // See: https://www.matroska.org/technical/specs/index.html#simpleblock_structure
-
-        Picture rgb = Picture.create(pixelWidth, pixelHeight, ColorSpace.RGB);
-        BufferedImage bufferedImage = new BufferedImage(pixelWidth, pixelHeight, BufferedImage.TYPE_3BYTE_BGR);
-        AvcCBox avcC = AvcCBox.parseAvcCBox(ByteBuffer.wrap(codecPrivateData));
-
-        decoder.addSps(avcC.getSpsList());
-        decoder.addPps(avcC.getPpsList());
-
-        Picture buf = Picture.create(pixelWidth, pixelHeight, ColorSpace.YUV420J);
-        List<ByteBuffer> byteBuffers = splitMOVPacket(frameBuffer, avcC);
-        Picture pic = decoder.decodeFrameFromNals(byteBuffers, buf.getData());
-
-        if (pic != null) {
-            // Work around for color issues in JCodec
-            // https://github.com/jcodec/jcodec/issues/59
-            // https://github.com/jcodec/jcodec/issues/192
-            byte[][] dataTemp = new byte[3][pic.getData().length];
-            dataTemp[0] = pic.getPlaneData(0);
-            dataTemp[1] = pic.getPlaneData(2);
-            dataTemp[2] = pic.getPlaneData(1);
-
-            Picture tmpBuf = Picture.createPicture(pixelWidth, pixelHeight, dataTemp, ColorSpace.YUV420J);
-            transform.transform(tmpBuf, rgb);
-            AWTUtil.toBufferedImage(rgb, bufferedImage);
-            frameCount++;
-        }
-        return bufferedImage;
-    }
-
-    public ByteBuffer getCodecPrivateData() {
-        return ByteBuffer.wrap(codecPrivateData);
-    }
-
 }

src/main/java/com/amazonaws/kinesisvideo/parser/utilities/SimpleFrameVisitor.java

@@ -0,0 +1,91 @@
+/*
+Copyright 2017-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"). 
+You may not use this file except in compliance with the License. 
+A copy of the License is located at
+
+   http://aws.amazon.com/apache2.0/
+
+or in the "license" file accompanying this file. 
+This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and limitations under the License.
+*/
+package com.amazonaws.kinesisvideo.parser.utilities;
+
+import com.amazonaws.kinesisvideo.parser.ebml.MkvTypeInfos;
+import com.amazonaws.kinesisvideo.parser.mkv.Frame;
+import com.amazonaws.kinesisvideo.parser.mkv.MkvElementVisitException;
+import com.amazonaws.kinesisvideo.parser.mkv.MkvElementVisitor;
+import com.amazonaws.kinesisvideo.parser.mkv.MkvValue;
+import com.amazonaws.kinesisvideo.parser.mkv.visitors.CompositeMkvElementVisitor;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.NotImplementedException;
+import org.apache.commons.lang3.Validate;
+
+import java.math.BigInteger;
+
+/**
+ * Fragment metdata tags will not be present as there is no tag when the data (mkv/webm) is not
+ * retrieved from Kinesis video.
+ * As a result, user cannot get the timestamp from fragment metadata tags.
+ * This class is used to provide a FrameVisitor to process frame as well as time elements for timestamp.
+ **/
+
+@Slf4j
+public class SimpleFrameVisitor extends CompositeMkvElementVisitor {
+    private final FrameVisitorInternal frameVisitorInternal;
+    private final FrameProcessor frameProcessor;
+
+    private SimpleFrameVisitor(FrameProcessor frameProcessor) {
+        this.frameProcessor = frameProcessor;
+        frameVisitorInternal = new FrameVisitorInternal();
+        childVisitors.add(frameVisitorInternal);
+
+    }
+    public static SimpleFrameVisitor create(FrameProcessor frameProcessor) {
+        return new SimpleFrameVisitor(frameProcessor);
+    }
+    public interface FrameProcessor {
+        default void process(Frame frame, long clusterTimeCode, long timeCodeScale) {
+            throw new NotImplementedException("Default FrameVisitor with No Fragement MetaData");
+        }
+    }
+
+    private class FrameVisitorInternal extends MkvElementVisitor {
+        private long clusterTimeCode = -1;
+        private long timeCodeScale = -1;
+        @Override
+        public void visit(com.amazonaws.kinesisvideo.parser.mkv.MkvStartMasterElement startMasterElement)
+                throws com.amazonaws.kinesisvideo.parser.mkv.MkvElementVisitException {
+        }
+
+        @Override
+        public void visit(com.amazonaws.kinesisvideo.parser.mkv.MkvEndMasterElement endMasterElement)
+                throws com.amazonaws.kinesisvideo.parser.mkv.MkvElementVisitException {
+        }
+
+        @Override
+        public void visit(com.amazonaws.kinesisvideo.parser.mkv.MkvDataElement dataElement)
+                throws com.amazonaws.kinesisvideo.parser.mkv.MkvElementVisitException {
+
+            if (MkvTypeInfos.TIMECODE.equals(dataElement.getElementMetaData().getTypeInfo())) {
+                clusterTimeCode = ((BigInteger) dataElement.getValueCopy().getVal()).longValue();
+            }
+
+            if (MkvTypeInfos.TIMECODESCALE.equals(dataElement.getElementMetaData().getTypeInfo())) {
+                timeCodeScale = ((BigInteger) dataElement.getValueCopy().getVal()).longValue();
+            }
+
+            if (MkvTypeInfos.SIMPLEBLOCK.equals(dataElement.getElementMetaData().getTypeInfo())) {
+                if (clusterTimeCode == -1 || timeCodeScale == -1) {
+                    throw new MkvElementVisitException("No timeCodeScale or timeCode found", new RuntimeException());
+                }
+                final MkvValue<Frame> frame = dataElement.getValueCopy();
+                Validate.notNull(frame);
+                frameProcessor.process(frame.getVal(), clusterTimeCode, timeCodeScale);
+            }
+        }
+
+    }
+}

src/test/java/com/amazonaws/kinesisvideo/parser/utilities/FragmentMetadataVisitorTest.java

@@ -51,7 +51,7 @@ public void basicTest() throws IOException, MkvElementVisitException {
         continuationTokens.add("91343852333181432397633822764885441725874549018");
         continuationTokens.add("91343852333181432402585582922026963247510532162");
 
-        FragmentMetadataVisitor fragmentVisitor = FragmentMetadataVisitor.create();
+        final FragmentMetadataVisitor fragmentVisitor = FragmentMetadataVisitor.create();
         StreamingMkvReader mkvStreamReader =
                 StreamingMkvReader.createDefault(new InputStreamParserByteSource(in));
         int segmentCount = 0;
@@ -102,7 +102,7 @@ private void assertTrackAndFragmentInfo(FragmentMetadataVisitor fragmentVisitor,
 
     @Test
     public void withOutputSegmentMergerTest() throws IOException, MkvElementVisitException {
-        FragmentMetadataVisitor fragmentVisitor = FragmentMetadataVisitor.create();
+        final FragmentMetadataVisitor fragmentVisitor = FragmentMetadataVisitor.create();
 
         ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
 
@@ -133,7 +133,7 @@ public void withOutputSegmentMergerTest() throws IOException, MkvElementVisitExc
 
     @Test
     public void testFragmentNumbers_NoClusterData() throws IOException, MkvElementVisitException {
-        FragmentMetadataVisitor fragmentVisitor = FragmentMetadataVisitor.create();
+        final FragmentMetadataVisitor fragmentVisitor = FragmentMetadataVisitor.create();
         String testFile = "empty-mkv-with-tags.mkv";        
         Set<String> expectedFragmentNumbers = new HashSet<>(
                 Arrays.asList(
@@ -159,6 +159,46 @@ public void testFragmentNumbers_NoClusterData() throws IOException, MkvElementVi
         Assert.assertEquals(expectedFragmentNumbers, visitedFragmentNumbers);
     }
 
+    @Test
+    public void testFragmentMetadata_NoFragementMetadata_withWebm() throws IOException, MkvElementVisitException {
+        final FragmentMetadataVisitor fragmentMetadataVisitor = FragmentMetadataVisitor.create();
+        final String testFile = "big-buck-bunny_trailer.webm";
+        int metadataCount = 0;
+        final StreamingMkvReader mkvStreamReader =  StreamingMkvReader
+                        .createDefault(new InputStreamParserByteSource(TestResourceUtil.getTestInputStream(testFile)));
+        while (mkvStreamReader.mightHaveNext()) {
+            Optional<MkvElement> mkvElement = mkvStreamReader.nextIfAvailable();
+            if (mkvElement.isPresent()) {
+                mkvElement.get().accept(fragmentMetadataVisitor);
+                Optional<FragmentMetadata> fragmentMetadata = fragmentMetadataVisitor.getCurrentFragmentMetadata();
+                if(fragmentMetadata.isPresent()) {
+                    metadataCount ++;
+                }
+            }
+        }
+        Assert.assertEquals(0, metadataCount);
+    }
+
+    @Test
+    public void testFragmentMetadata_NoFragementMetadata_withMkv() throws IOException, MkvElementVisitException {
+        final FragmentMetadataVisitor fragmentMetadataVisitor = FragmentMetadataVisitor.create();
+        final String testFile = "clusters.mkv";
+        int metadataCount = 0;
+        final StreamingMkvReader mkvStreamReader =  StreamingMkvReader
+                .createDefault(new InputStreamParserByteSource(TestResourceUtil.getTestInputStream(testFile)));
+        while (mkvStreamReader.mightHaveNext()) {
+            Optional<MkvElement> mkvElement = mkvStreamReader.nextIfAvailable();
+            if (mkvElement.isPresent()) {
+                mkvElement.get().accept(fragmentMetadataVisitor);
+                Optional<FragmentMetadata> fragmentMetadata = fragmentMetadataVisitor.getCurrentFragmentMetadata();
+                if(fragmentMetadata.isPresent()) {
+                    metadataCount ++;
+                }
+            }
+        }
+        Assert.assertEquals(0, metadataCount);
+    }
+
     private static class TestCompositeVisitor extends CompositeMkvElementVisitor {
         public TestCompositeVisitor(FragmentMetadataVisitor fragmentMetadataVisitor, OutputSegmentMerger merger) {
             super(fragmentMetadataVisitor, merger);