Changes to preserve dfDateTime object #910

config/dpkg/control

@@ -17,7 +17,7 @@ Description: Data files for plaso (log2timeline)
 
 Package: python3-plaso
 Architecture: all
-Depends: plaso-data (>= ${binary:Version}), libbde-python3 (>= 20140531), libcreg-python3 (>= 20200725), libesedb-python3 (>= 20150409), libevt-python3 (>= 20191104), libevtx-python3 (>= 20141112), libewf-python3 (>= 20131210), libfsapfs-python3 (>= 20181205), libfsext-python3 (>= 20200819), libfsntfs-python3 (>= 20200805), libfvde-python3 (>= 20160719), libfwnt-python3 (>= 20180117), libfwsi-python3 (>= 20150606), liblnk-python3 (>= 20150830), libluksde-python3 (>= 20200101), libmsiecf-python3 (>= 20150314), libolecf-python3 (>= 20151223), libqcow-python3 (>= 20131204), libregf-python3 (>= 20190714), libscca-python3 (>= 20190605), libsigscan-python3 (>= 20190629), libsmdev-python3 (>= 20140529), libsmraw-python3 (>= 20140612), libvhdi-python3 (>= 20131210), libvmdk-python3 (>= 20140421), libvshadow-python3 (>= 20160109), libvslvm-python3 (>= 20160109), python3-artifacts (>= 20190305), python3-bencode, python3-certifi (>= 2016.9.26), python3-cffi-backend (>= 1.9.1), python3-chardet (>= 2.0.1), python3-cryptography (>= 2.0.2), python3-dateutil (>= 1.5), python3-defusedxml (>= 0.5.0), python3-dfdatetime (>= 20200613), python3-dfvfs (>= 20200920), python3-dfwinreg (>= 20200927), python3-dtfabric (>= 20200621), python3-elasticsearch (>= 6.0), python3-future (>= 0.16.0), python3-idna (>= 2.5), python3-lz4 (>= 0.10.0), python3-pefile (>= 2018.8.8), python3-psutil (>= 5.4.3), python3-pyparsing (>= 2.3.0), python3-pytsk3 (>= 20160721), python3-redis (>= 3.4), python3-requests (>= 2.18.0), python3-six (>= 1.1.0), python3-tz, python3-urllib3 (>= 1.21.1), python3-xlsxwriter (>= 0.9.3), python3-yaml (>= 3.10), python3-yara (>= 3.4.0), python3-zmq (>= 2.1.11), ${python3:Depends}, ${misc:Depends}
+Depends: plaso-data (>= ${binary:Version}), libbde-python3 (>= 20140531), libcreg-python3 (>= 20200725), libesedb-python3 (>= 20150409), libevt-python3 (>= 20191104), libevtx-python3 (>= 20141112), libewf-python3 (>= 20131210), libfsapfs-python3 (>= 20181205), libfsext-python3 (>= 20200819), libfsntfs-python3 (>= 20200805), libfvde-python3 (>= 20160719), libfwnt-python3 (>= 20180117), libfwsi-python3 (>= 20150606), liblnk-python3 (>= 20150830), libluksde-python3 (>= 20200101), libmsiecf-python3 (>= 20150314), libolecf-python3 (>= 20151223), libqcow-python3 (>= 20131204), libregf-python3 (>= 20190714), libscca-python3 (>= 20190605), libsigscan-python3 (>= 20190629), libsmdev-python3 (>= 20140529), libsmraw-python3 (>= 20140612), libvhdi-python3 (>= 20131210), libvmdk-python3 (>= 20140421), libvshadow-python3 (>= 20160109), libvslvm-python3 (>= 20160109), python3-artifacts (>= 20190305), python3-bencode, python3-certifi (>= 2016.9.26), python3-cffi-backend (>= 1.9.1), python3-chardet (>= 2.0.1), python3-cryptography (>= 2.0.2), python3-dateutil (>= 1.5), python3-defusedxml (>= 0.5.0), python3-dfdatetime (>= 20200824), python3-dfvfs (>= 20200920), python3-dfwinreg (>= 20200927), python3-dtfabric (>= 20200621), python3-elasticsearch (>= 6.0), python3-future (>= 0.16.0), python3-idna (>= 2.5), python3-lz4 (>= 0.10.0), python3-pefile (>= 2018.8.8), python3-psutil (>= 5.4.3), python3-pyparsing (>= 2.3.0), python3-pytsk3 (>= 20160721), python3-redis (>= 3.4), python3-requests (>= 2.18.0), python3-six (>= 1.1.0), python3-tz, python3-urllib3 (>= 1.21.1), python3-xlsxwriter (>= 0.9.3), python3-yaml (>= 3.10), python3-yara (>= 3.4.0), python3-zmq (>= 2.1.11), ${python3:Depends}, ${misc:Depends}
 Description: Python 3 module of plaso (log2timeline)
  Plaso (log2timeline) is a framework to create super timelines. Its
  purpose is to extract timestamps from various files found on typical

dependencies.ini

@@ -50,7 +50,7 @@ version_property: __version__
 
 [dfdatetime]
 dpkg_name: python3-dfdatetime
-minimum_version: 20200613
+minimum_version: 20200824
 rpm_name: python3-dfdatetime
 version_property: __version__
 

plaso/containers/events.py

@@ -131,6 +131,7 @@ class EventObject(interface.AttributeContainer):
   attributes.
 
   Attributes:
+    date_time (dfdatetime.DateTimeValues): date and time values.
     parser (str): string identifying the parser that produced the event.
     timestamp (int): timestamp, which contains the number of microseconds
         since January 1, 1970, 00:00:00 UTC.
@@ -145,6 +146,7 @@ def __init__(self):
     super(EventObject, self).__init__()
     self._event_data_identifier = None
     self._event_data_row_identifier = None
+    self.date_time = None
     self.parser = None
     self.timestamp = None
     # TODO: rename timestamp_desc to timestamp_description

plaso/containers/time_events.py

@@ -11,13 +11,13 @@ class DateTimeValuesEvent(events.EventObject):
   """dfDateTime date time values-based event attribute container.
 
   Attributes:
+    date_time (dfdatetime.DateTimeValues): date and time values.
     timestamp (int): timestamp, which contains the number of microseconds
         since January 1, 1970, 00:00:00 UTC.
     timestamp_desc (str): description of the meaning of the timestamp.
   """
 
-  def __init__(
-      self, date_time, date_time_description, time_zone=None):
+  def __init__(self, date_time, date_time_description, time_zone=None):
     """Initializes an event.
 
     Args:
@@ -31,5 +31,6 @@ def __init__(
       timestamp = timelib.Timestamp.LocaltimeToUTC(timestamp, time_zone)
 
     super(DateTimeValuesEvent, self).__init__()
+    self.date_time = date_time
     self.timestamp = timestamp
     self.timestamp_desc = date_time_description

plaso/dependencies.py

@@ -26,7 +26,7 @@
     'cryptography': ('__version__', '2.0.2', None, True),
     'dateutil': ('__version__', '1.5', None, True),
     'defusedxml': ('__version__', '0.5.0', None, True),
-    'dfdatetime': ('__version__', '20200613', None, True),
+    'dfdatetime': ('__version__', '20200824', None, True),
     'dfvfs': ('__version__', '20200920', None, True),
     'dfwinreg': ('__version__', '20200927', None, True),
     'dtfabric': ('__version__', '20200621', None, True),

plaso/parsers/apt_history.py

@@ -115,13 +115,22 @@ def _BuildDateTime(time_elements_structure):
       dfdatetime.TimeElements: date and time extracted from the structure or
           None f the structure does not represent a valid string.
     """
+    # Ensure time_elements_tuple is not a pyparsing.ParseResults otherwise
+    # copy.deepcopy() of the dfDateTime object will fail on Python 3.8 with:
+    # "TypeError: 'str' object is not callable" due to pyparsing.ParseResults
+    # overriding __getattr__ with a function that returns an empty string when
+    # named token does not exists.
     try:
-      date_time = dfdatetime_time_elements.TimeElements(
-          time_elements_tuple=time_elements_structure)
+      year, month, day_of_month, hours, minutes, seconds = (
+          time_elements_structure)
+
+      date_time = dfdatetime_time_elements.TimeElements(time_elements_tuple=(
+          year, month, day_of_month, hours, minutes, seconds))
+
       # APT History logs store date and time values in local time.
       date_time.is_local_time = True
       return date_time
-    except ValueError:
+    except (TypeError, ValueError):
       return None
 
   def _ParseRecordStart(self, parser_mediator, structure):
@@ -133,12 +142,10 @@ def _ParseRecordStart(self, parser_mediator, structure):
       structure (pyparsing.ParseResults): structure of tokens derived from
           a log entry.
     """
-    time_elements_structure = self._GetValueFromStructure(
-        structure, 'start_date')
-    self._date_time = self._BuildDateTime(time_elements_structure)
+    self._date_time = self._BuildDateTime(structure.get('start_date', None))
     if not self._date_time:
       parser_mediator.ProduceExtractionWarning(
-          'invalid date time value: {0!s}'.format(time_elements_structure))
+          'invalid date time value: {0!s}'.format(self._date_time))
       return
 
     self._event_data = APTHistoryLogEventData()

plaso/parsers/dpkg.py

@@ -143,14 +143,23 @@ def ParseRecord(self, parser_mediator, key, structure):
       raise errors.ParseError(
           'Unable to parse record, unknown structure: {0:s}'.format(key))
 
-    time_elements_tuple = self._GetValueFromStructure(structure, 'date_time')
+    # Ensure time_elements_tuple is not a pyparsing.ParseResults otherwise
+    # copy.deepcopy() of the dfDateTime object will fail on Python 3.8 with:
+    # "TypeError: 'str' object is not callable" due to pyparsing.ParseResults
+    # overriding __getattr__ with a function that returns an empty string when
+    # named token does not exists.
+    time_elements_structure = structure.get('date_time', None)
 
     try:
-      date_time = dfdatetime_time_elements.TimeElements(
-          time_elements_tuple=time_elements_tuple)
-    except ValueError:
+      year, month, day_of_month, hours, minutes, seconds = (
+          time_elements_structure)
+
+      date_time = dfdatetime_time_elements.TimeElements(time_elements_tuple=(
+          year, month, day_of_month, hours, minutes, seconds))
+
+    except (TypeError, ValueError):
       parser_mediator.ProduceExtractionWarning(
-          'invalid date time value: {0!s}'.format(time_elements_tuple))
+          'invalid date time value: {0!s}'.format(time_elements_structure))
       return
 
     body_text = self._GetValueFromStructure(structure, 'body')

plaso/parsers/iis.py

@@ -207,8 +207,19 @@ def _ParseLogLine(self, parser_mediator, structure):
           and other components, such as storage and dfvfs.
       structure (pyparsing.ParseResults): structure parsed from the log file.
     """
-    time_elements_tuple = self._GetValueFromStructure(structure, 'date_time')
-    if not time_elements_tuple:
+    time_elements_structure = structure.get('date_time', None)
+    if time_elements_structure:
+      # Ensure time_elements_tuple is not a pyparsing.ParseResults otherwise
+      # copy.deepcopy() of the dfDateTime object will fail on Python 3.8 with:
+      # "TypeError: 'str' object is not callable" due to pyparsing.ParseResults
+      # overriding __getattr__ with a function that returns an empty string when
+      # named token does not exists.
+      year, month, day_of_month, hours, minutes, seconds = (
+          time_elements_structure)
+
+      time_elements_tuple = (year, month, day_of_month, hours, minutes, seconds)
+
+    else:
       time_tuple = self._GetValueFromStructure(structure, 'time')
       if not time_tuple:
         parser_mediator.ProduceExtractionWarning('missing time values')

plaso/parsers/sophos_av.py

@@ -74,15 +74,25 @@ def _ParseLogLine(self, parser_mediator, structure):
       structure (pyparsing.ParseResults): structure of tokens derived from
           a line of a text file.
     """
-    time_elements_tuple = self._GetValueFromStructure(structure, 'date_time')
+    # Ensure time_elements_tuple is not a pyparsing.ParseResults otherwise
+    # copy.deepcopy() of the dfDateTime object will fail on Python 3.8 with:
+    # "TypeError: 'str' object is not callable" due to pyparsing.ParseResults
+    # overriding __getattr__ with a function that returns an empty string when
+    # named token does not exists.
+    time_elements_structure = structure.get('date_time', None)
+
     try:
-      date_time = dfdatetime_time_elements.TimeElements(
-          time_elements_tuple=time_elements_tuple)
+      year, month, day_of_month, hours, minutes, seconds = (
+          time_elements_structure)
+
+      date_time = dfdatetime_time_elements.TimeElements(time_elements_tuple=(
+          year, month, day_of_month, hours, minutes, seconds))
       # TODO: check if date and time values are local time or in UTC.
       date_time.is_local_time = True
-    except ValueError:
+
+    except (TypeError, ValueError):
       parser_mediator.ProduceExtractionWarning(
-          'invalid date time value: {0!s}'.format(time_elements_tuple))
+          'invalid date time value: {0!s}'.format(time_elements_structure))
       return
 
     event_data = SophosAVLogEventData()

plaso/parsers/winfirewall.py

@@ -150,14 +150,24 @@ def _ParseLogLine(self, parser_mediator, structure):
       structure (pyparsing.ParseResults): structure of tokens derived from
           a line of a text file.
     """
-    time_elements_tuple = self._GetValueFromStructure(structure, 'date_time')
+    # Ensure time_elements_tuple is not a pyparsing.ParseResults otherwise
+    # copy.deepcopy() of the dfDateTime object will fail on Python 3.8 with:
+    # "TypeError: 'str' object is not callable" due to pyparsing.ParseResults
+    # overriding __getattr__ with a function that returns an empty string when
+    # named token does not exists.
+    time_elements_structure = structure.get('date_time', None)
+
     try:
-      date_time = dfdatetime_time_elements.TimeElements(
-          time_elements_tuple=time_elements_tuple)
+      year, month, day_of_month, hours, minutes, seconds = (
+          time_elements_structure)
+
+      date_time = dfdatetime_time_elements.TimeElements(time_elements_tuple=(
+          year, month, day_of_month, hours, minutes, seconds))
       date_time.is_local_time = True
-    except ValueError:
+
+    except (TypeError, ValueError):
       parser_mediator.ProduceExtractionWarning(
-          'invalid date time value: {0!s}'.format(time_elements_tuple))
+          'invalid date time value: {0!s}'.format(time_elements_structure))
       return
 
     event_data = WinFirewallEventData()

plaso/serializer/json_serializer.py

@@ -8,9 +8,16 @@
 import collections
 import json
 
+from dfdatetime import factory as dfdatetime_factory
+from dfdatetime import interface as dfdatetime_interface
+
 from dfvfs.path import path_spec as dfvfs_path_spec
 from dfvfs.path import factory as dfvfs_path_spec_factory
 
+# The following import is needed to make sure TSKTime is registered with
+# the dfDateTime factory.
+from dfvfs.vfs import tsk_file_entry  # pylint: disable=unused-import
+
 from plaso.containers import interface as containers_interface
 from plaso.containers import manager as containers_manager
 from plaso.serializer import interface
@@ -117,6 +124,9 @@ def _ConvertAttributeValueToDict(cls, attribute_value):
     elif isinstance(attribute_value, collections.Counter):
       attribute_value = cls._ConvertCollectionsCounterToDict(attribute_value)
 
+    elif isinstance(attribute_value, dfdatetime_interface.DateTimeValues):
+      attribute_value = cls._ConvertDateTimeValuesToDict(attribute_value)
+
     elif isinstance(attribute_value, dfvfs_path_spec.PathSpec):
       attribute_value = cls._ConvertPathSpecToDict(attribute_value)
 
@@ -215,6 +225,11 @@ def _ConvertDictToObject(cls, json_dict):
       # Use __container_type__ to indicate the attribute container type.
       container_type = json_dict.get('__container_type__', None)
 
+    # Since we would like the JSON as flat as possible we handle decoding
+    # date time values.
+    elif class_type == 'DateTimeValues':
+      return cls._ConvertDictToDateTimeValues(json_dict)
+
     # Since we would like the JSON as flat as possible we handle decoding
     # a path specification.
     elif class_type == 'PathSpec':
@@ -330,6 +345,45 @@ def _ConvertListToObject(cls, json_list):
 
     return list_value
 
+  @classmethod
+  def _ConvertDictToDateTimeValues(cls, json_dict):
+    """Converts a JSON dict into a date time values object.
+
+    The dictionary of the JSON serialized objects consists of:
+    {
+        '__type__': 'DateTimeValues'
+        '__class_name__': 'RFC2579DateTime'
+        ...
+    }
+
+    Here '__type__' indicates the object base type. In this case this should
+    be 'DateTimeValues'. The rest of the elements of the dictionary make up the
+    date time values object properties.
+
+    Args:
+      json_dict (dict[str, object]): JSON serialized objects.
+
+    Returns:
+      dfdatetime.DateTimeValues: date and time values.
+    """
+    class_name = json_dict.get('__class_name__', None)
+    if class_name:
+      del json_dict['__class_name__']
+
+    # Remove the class type from the JSON dict since we cannot pass it.
+    del json_dict['__type__']
+
+    is_local_time = json_dict.get('is_local_time', None)
+    if is_local_time is not None:
+      del json_dict['is_local_time']
+
+    date_time = dfdatetime_factory.Factory.NewDateTimeValues(
+        class_name, **json_dict)
+    if is_local_time:
+      date_time.is_local_time = is_local_time
+
+    return date_time
+
   @classmethod
   def _ConvertDictToPathSpec(cls, json_dict):
     """Converts a JSON dict into a path specification object.
@@ -350,7 +404,7 @@ def _ConvertDictToPathSpec(cls, json_dict):
       json_dict (dict[str, object]): JSON serialized objects.
 
     Returns:
-      path.PathSpec: path specification.
+      dfvfs.PathSpec: path specification.
     """
     type_indicator = json_dict.get('type_indicator', None)
     if type_indicator:
@@ -365,6 +419,81 @@ def _ConvertDictToPathSpec(cls, json_dict):
     return dfvfs_path_spec_factory.Factory.NewPathSpec(
         type_indicator, **json_dict)
 
+  @classmethod
+  def _ConvertDateTimeValuesToDict(cls, date_time_values):
+    """Converts a date and time values object into a JSON dictionary.
+
+    The resulting dictionary of the JSON serialized objects consists of:
+    {
+        '__type__': 'DateTimeValues'
+        '__class_name__': 'RFC2579DateTime'
+        ...
+    }
+
+    Here '__type__' indicates the object base type. In this case
+    'DateTimeValues'. The rest of the elements of the dictionary make up the
+    date and time value object properties.
+
+    Args:
+      date_time_values (dfdatetime.DateTimeValues): date and time values.
+
+    Returns:
+      dict[str, object]: JSON serialized objects.
+
+    Raises:
+      TypeError: if not an instance of dfvfs.PathSpec.
+    """
+    if not isinstance(date_time_values, dfdatetime_interface.DateTimeValues):
+      raise TypeError
+
+    class_name = type(date_time_values).__name__
+
+    json_dict = {
+        '__class_name__': class_name,
+        '__type__': 'DateTimeValues'}
+
+    if hasattr(date_time_values, 'timestamp'):
+      json_dict['timestamp'] = date_time_values.timestamp
+
+    elif hasattr(date_time_values, 'string'):
+      json_dict['string'] = date_time_values.string
+
+    elif class_name == 'FATDateTime':
+      # TODO: set fat_date_time value
+      pass
+
+    elif class_name == 'RFC2579DateTime':
+      json_dict['rfc2579_date_time_tuple'] = (
+          date_time_values.year, date_time_values.month,
+          date_time_values.day_of_month, date_time_values.hours,
+          date_time_values.minutes, date_time_values.seconds,
+          date_time_values.deciseconds)
+
+    elif class_name == 'TimeElements':
+      json_dict['time_elements_tuple'] = (
+          date_time_values.year, date_time_values.month,
+          date_time_values.day_of_month, date_time_values.hours,
+          date_time_values.minutes, date_time_values.seconds)
+
+    elif class_name == 'TimeElementsInMilliseconds':
+      json_dict['time_elements_tuple'] = (
+          date_time_values.year, date_time_values.month,
+          date_time_values.day_of_month, date_time_values.hours,
+          date_time_values.minutes, date_time_values.seconds,
+          date_time_values.milliseconds)
+
+    elif class_name == 'TimeElementsInMicroseconds':
+      json_dict['time_elements_tuple'] = (
+          date_time_values.year, date_time_values.month,
+          date_time_values.day_of_month, date_time_values.hours,
+          date_time_values.minutes, date_time_values.seconds,
+          date_time_values.microseconds)
+
+    if date_time_values.is_local_time:
+      json_dict['is_local_time'] = True
+
+    return json_dict
+
   @classmethod
   def _ConvertPathSpecToDict(cls, path_spec_object):
     """Converts a path specification object into a JSON dictionary.