diff --git a/config/dpkg/control b/config/dpkg/control index beb958fae6..fcc1f281f2 100644 --- a/config/dpkg/control +++ b/config/dpkg/control @@ -17,7 +17,7 @@ Description: Data files for plaso (log2timeline) Package: python3-plaso Architecture: all -Depends: plaso-data (>= ${binary:Version}), libbde-python3 (>= 20140531), libcreg-python3 (>= 20200725), libesedb-python3 (>= 20150409), libevt-python3 (>= 20191104), libevtx-python3 (>= 20141112), libewf-python3 (>= 20131210), libfsapfs-python3 (>= 20181205), libfsext-python3 (>= 20200819), libfsntfs-python3 (>= 20200805), libfvde-python3 (>= 20160719), libfwnt-python3 (>= 20180117), libfwsi-python3 (>= 20150606), liblnk-python3 (>= 20150830), libluksde-python3 (>= 20200101), libmsiecf-python3 (>= 20150314), libolecf-python3 (>= 20151223), libqcow-python3 (>= 20131204), libregf-python3 (>= 20190714), libscca-python3 (>= 20190605), libsigscan-python3 (>= 20190629), libsmdev-python3 (>= 20140529), libsmraw-python3 (>= 20140612), libvhdi-python3 (>= 20131210), libvmdk-python3 (>= 20140421), libvshadow-python3 (>= 20160109), libvslvm-python3 (>= 20160109), python3-artifacts (>= 20190305), python3-bencode, python3-certifi (>= 2016.9.26), python3-cffi-backend (>= 1.9.1), python3-chardet (>= 2.0.1), python3-cryptography (>= 2.0.2), python3-dateutil (>= 1.5), python3-defusedxml (>= 0.5.0), python3-dfdatetime (>= 20200613), python3-dfvfs (>= 20200920), python3-dfwinreg (>= 20200927), python3-dtfabric (>= 20200621), python3-elasticsearch (>= 6.0), python3-future (>= 0.16.0), python3-idna (>= 2.5), python3-lz4 (>= 0.10.0), python3-pefile (>= 2018.8.8), python3-psutil (>= 5.4.3), python3-pyparsing (>= 2.3.0), python3-pytsk3 (>= 20160721), python3-redis (>= 3.4), python3-requests (>= 2.18.0), python3-six (>= 1.1.0), python3-tz, python3-urllib3 (>= 1.21.1), python3-xlsxwriter (>= 0.9.3), python3-yaml (>= 3.10), python3-yara (>= 3.4.0), python3-zmq (>= 2.1.11), ${python3:Depends}, ${misc:Depends} +Depends: plaso-data (>= ${binary:Version}), libbde-python3 (>= 20140531), libcreg-python3 (>= 20200725), libesedb-python3 (>= 20150409), libevt-python3 (>= 20191104), libevtx-python3 (>= 20141112), libewf-python3 (>= 20131210), libfsapfs-python3 (>= 20181205), libfsext-python3 (>= 20200819), libfsntfs-python3 (>= 20200805), libfvde-python3 (>= 20160719), libfwnt-python3 (>= 20180117), libfwsi-python3 (>= 20150606), liblnk-python3 (>= 20150830), libluksde-python3 (>= 20200101), libmsiecf-python3 (>= 20150314), libolecf-python3 (>= 20151223), libqcow-python3 (>= 20131204), libregf-python3 (>= 20190714), libscca-python3 (>= 20190605), libsigscan-python3 (>= 20190629), libsmdev-python3 (>= 20140529), libsmraw-python3 (>= 20140612), libvhdi-python3 (>= 20131210), libvmdk-python3 (>= 20140421), libvshadow-python3 (>= 20160109), libvslvm-python3 (>= 20160109), python3-artifacts (>= 20190305), python3-bencode, python3-certifi (>= 2016.9.26), python3-cffi-backend (>= 1.9.1), python3-chardet (>= 2.0.1), python3-cryptography (>= 2.0.2), python3-dateutil (>= 1.5), python3-defusedxml (>= 0.5.0), python3-dfdatetime (>= 20200824), python3-dfvfs (>= 20200920), python3-dfwinreg (>= 20200927), python3-dtfabric (>= 20200621), python3-elasticsearch (>= 6.0), python3-future (>= 0.16.0), python3-idna (>= 2.5), python3-lz4 (>= 0.10.0), python3-pefile (>= 2018.8.8), python3-psutil (>= 5.4.3), python3-pyparsing (>= 2.3.0), python3-pytsk3 (>= 20160721), python3-redis (>= 3.4), python3-requests (>= 2.18.0), python3-six (>= 1.1.0), python3-tz, python3-urllib3 (>= 1.21.1), python3-xlsxwriter (>= 0.9.3), python3-yaml (>= 3.10), python3-yara (>= 3.4.0), python3-zmq (>= 2.1.11), ${python3:Depends}, ${misc:Depends} Description: Python 3 module of plaso (log2timeline) Plaso (log2timeline) is a framework to create super timelines. Its purpose is to extract timestamps from various files found on typical diff --git a/dependencies.ini b/dependencies.ini index b06a2f5702..0ff183f9b9 100644 --- a/dependencies.ini +++ b/dependencies.ini @@ -50,7 +50,7 @@ version_property: __version__ [dfdatetime] dpkg_name: python3-dfdatetime -minimum_version: 20200613 +minimum_version: 20200824 rpm_name: python3-dfdatetime version_property: __version__ diff --git a/plaso/containers/events.py b/plaso/containers/events.py index 60a17085ee..a2e61fb276 100644 --- a/plaso/containers/events.py +++ b/plaso/containers/events.py @@ -131,6 +131,7 @@ class EventObject(interface.AttributeContainer): attributes. Attributes: + date_time (dfdatetime.DateTimeValues): date and time values. parser (str): string identifying the parser that produced the event. timestamp (int): timestamp, which contains the number of microseconds since January 1, 1970, 00:00:00 UTC. @@ -145,6 +146,7 @@ def __init__(self): super(EventObject, self).__init__() self._event_data_identifier = None self._event_data_row_identifier = None + self.date_time = None self.parser = None self.timestamp = None # TODO: rename timestamp_desc to timestamp_description diff --git a/plaso/containers/time_events.py b/plaso/containers/time_events.py index 83aaa47a01..4ba828be4f 100644 --- a/plaso/containers/time_events.py +++ b/plaso/containers/time_events.py @@ -11,13 +11,13 @@ class DateTimeValuesEvent(events.EventObject): """dfDateTime date time values-based event attribute container. Attributes: + date_time (dfdatetime.DateTimeValues): date and time values. timestamp (int): timestamp, which contains the number of microseconds since January 1, 1970, 00:00:00 UTC. timestamp_desc (str): description of the meaning of the timestamp. """ - def __init__( - self, date_time, date_time_description, time_zone=None): + def __init__(self, date_time, date_time_description, time_zone=None): """Initializes an event. Args: @@ -31,5 +31,6 @@ def __init__( timestamp = timelib.Timestamp.LocaltimeToUTC(timestamp, time_zone) super(DateTimeValuesEvent, self).__init__() + self.date_time = date_time self.timestamp = timestamp self.timestamp_desc = date_time_description diff --git a/plaso/dependencies.py b/plaso/dependencies.py index 368a242d11..59b564c3d8 100644 --- a/plaso/dependencies.py +++ b/plaso/dependencies.py @@ -26,7 +26,7 @@ 'cryptography': ('__version__', '2.0.2', None, True), 'dateutil': ('__version__', '1.5', None, True), 'defusedxml': ('__version__', '0.5.0', None, True), - 'dfdatetime': ('__version__', '20200613', None, True), + 'dfdatetime': ('__version__', '20200824', None, True), 'dfvfs': ('__version__', '20200920', None, True), 'dfwinreg': ('__version__', '20200927', None, True), 'dtfabric': ('__version__', '20200621', None, True), diff --git a/plaso/parsers/apt_history.py b/plaso/parsers/apt_history.py index b7e00d8eea..7d06eaa5d6 100644 --- a/plaso/parsers/apt_history.py +++ b/plaso/parsers/apt_history.py @@ -115,13 +115,22 @@ def _BuildDateTime(time_elements_structure): dfdatetime.TimeElements: date and time extracted from the structure or None f the structure does not represent a valid string. """ + # Ensure time_elements_tuple is not a pyparsing.ParseResults otherwise + # copy.deepcopy() of the dfDateTime object will fail on Python 3.8 with: + # "TypeError: 'str' object is not callable" due to pyparsing.ParseResults + # overriding __getattr__ with a function that returns an empty string when + # named token does not exists. try: - date_time = dfdatetime_time_elements.TimeElements( - time_elements_tuple=time_elements_structure) + year, month, day_of_month, hours, minutes, seconds = ( + time_elements_structure) + + date_time = dfdatetime_time_elements.TimeElements(time_elements_tuple=( + year, month, day_of_month, hours, minutes, seconds)) + # APT History logs store date and time values in local time. date_time.is_local_time = True return date_time - except ValueError: + except (TypeError, ValueError): return None def _ParseRecordStart(self, parser_mediator, structure): @@ -133,12 +142,10 @@ def _ParseRecordStart(self, parser_mediator, structure): structure (pyparsing.ParseResults): structure of tokens derived from a log entry. """ - time_elements_structure = self._GetValueFromStructure( - structure, 'start_date') - self._date_time = self._BuildDateTime(time_elements_structure) + self._date_time = self._BuildDateTime(structure.get('start_date', None)) if not self._date_time: parser_mediator.ProduceExtractionWarning( - 'invalid date time value: {0!s}'.format(time_elements_structure)) + 'invalid date time value: {0!s}'.format(self._date_time)) return self._event_data = APTHistoryLogEventData() diff --git a/plaso/parsers/dpkg.py b/plaso/parsers/dpkg.py index eda25e0463..fb40f276be 100644 --- a/plaso/parsers/dpkg.py +++ b/plaso/parsers/dpkg.py @@ -143,14 +143,23 @@ def ParseRecord(self, parser_mediator, key, structure): raise errors.ParseError( 'Unable to parse record, unknown structure: {0:s}'.format(key)) - time_elements_tuple = self._GetValueFromStructure(structure, 'date_time') + # Ensure time_elements_tuple is not a pyparsing.ParseResults otherwise + # copy.deepcopy() of the dfDateTime object will fail on Python 3.8 with: + # "TypeError: 'str' object is not callable" due to pyparsing.ParseResults + # overriding __getattr__ with a function that returns an empty string when + # named token does not exists. + time_elements_structure = structure.get('date_time', None) try: - date_time = dfdatetime_time_elements.TimeElements( - time_elements_tuple=time_elements_tuple) - except ValueError: + year, month, day_of_month, hours, minutes, seconds = ( + time_elements_structure) + + date_time = dfdatetime_time_elements.TimeElements(time_elements_tuple=( + year, month, day_of_month, hours, minutes, seconds)) + + except (TypeError, ValueError): parser_mediator.ProduceExtractionWarning( - 'invalid date time value: {0!s}'.format(time_elements_tuple)) + 'invalid date time value: {0!s}'.format(time_elements_structure)) return body_text = self._GetValueFromStructure(structure, 'body') diff --git a/plaso/parsers/iis.py b/plaso/parsers/iis.py index 6b7b92a4fd..0185dde60c 100644 --- a/plaso/parsers/iis.py +++ b/plaso/parsers/iis.py @@ -207,8 +207,19 @@ def _ParseLogLine(self, parser_mediator, structure): and other components, such as storage and dfvfs. structure (pyparsing.ParseResults): structure parsed from the log file. """ - time_elements_tuple = self._GetValueFromStructure(structure, 'date_time') - if not time_elements_tuple: + time_elements_structure = structure.get('date_time', None) + if time_elements_structure: + # Ensure time_elements_tuple is not a pyparsing.ParseResults otherwise + # copy.deepcopy() of the dfDateTime object will fail on Python 3.8 with: + # "TypeError: 'str' object is not callable" due to pyparsing.ParseResults + # overriding __getattr__ with a function that returns an empty string when + # named token does not exists. + year, month, day_of_month, hours, minutes, seconds = ( + time_elements_structure) + + time_elements_tuple = (year, month, day_of_month, hours, minutes, seconds) + + else: time_tuple = self._GetValueFromStructure(structure, 'time') if not time_tuple: parser_mediator.ProduceExtractionWarning('missing time values') diff --git a/plaso/parsers/sophos_av.py b/plaso/parsers/sophos_av.py index 1c9868468f..611c141b7e 100644 --- a/plaso/parsers/sophos_av.py +++ b/plaso/parsers/sophos_av.py @@ -74,15 +74,25 @@ def _ParseLogLine(self, parser_mediator, structure): structure (pyparsing.ParseResults): structure of tokens derived from a line of a text file. """ - time_elements_tuple = self._GetValueFromStructure(structure, 'date_time') + # Ensure time_elements_tuple is not a pyparsing.ParseResults otherwise + # copy.deepcopy() of the dfDateTime object will fail on Python 3.8 with: + # "TypeError: 'str' object is not callable" due to pyparsing.ParseResults + # overriding __getattr__ with a function that returns an empty string when + # named token does not exists. + time_elements_structure = structure.get('date_time', None) + try: - date_time = dfdatetime_time_elements.TimeElements( - time_elements_tuple=time_elements_tuple) + year, month, day_of_month, hours, minutes, seconds = ( + time_elements_structure) + + date_time = dfdatetime_time_elements.TimeElements(time_elements_tuple=( + year, month, day_of_month, hours, minutes, seconds)) # TODO: check if date and time values are local time or in UTC. date_time.is_local_time = True - except ValueError: + + except (TypeError, ValueError): parser_mediator.ProduceExtractionWarning( - 'invalid date time value: {0!s}'.format(time_elements_tuple)) + 'invalid date time value: {0!s}'.format(time_elements_structure)) return event_data = SophosAVLogEventData() diff --git a/plaso/parsers/winfirewall.py b/plaso/parsers/winfirewall.py index 618082ad52..25d6312abe 100644 --- a/plaso/parsers/winfirewall.py +++ b/plaso/parsers/winfirewall.py @@ -150,14 +150,24 @@ def _ParseLogLine(self, parser_mediator, structure): structure (pyparsing.ParseResults): structure of tokens derived from a line of a text file. """ - time_elements_tuple = self._GetValueFromStructure(structure, 'date_time') + # Ensure time_elements_tuple is not a pyparsing.ParseResults otherwise + # copy.deepcopy() of the dfDateTime object will fail on Python 3.8 with: + # "TypeError: 'str' object is not callable" due to pyparsing.ParseResults + # overriding __getattr__ with a function that returns an empty string when + # named token does not exists. + time_elements_structure = structure.get('date_time', None) + try: - date_time = dfdatetime_time_elements.TimeElements( - time_elements_tuple=time_elements_tuple) + year, month, day_of_month, hours, minutes, seconds = ( + time_elements_structure) + + date_time = dfdatetime_time_elements.TimeElements(time_elements_tuple=( + year, month, day_of_month, hours, minutes, seconds)) date_time.is_local_time = True - except ValueError: + + except (TypeError, ValueError): parser_mediator.ProduceExtractionWarning( - 'invalid date time value: {0!s}'.format(time_elements_tuple)) + 'invalid date time value: {0!s}'.format(time_elements_structure)) return event_data = WinFirewallEventData() diff --git a/plaso/serializer/json_serializer.py b/plaso/serializer/json_serializer.py index 5cf8dac72f..b8913a923a 100644 --- a/plaso/serializer/json_serializer.py +++ b/plaso/serializer/json_serializer.py @@ -8,9 +8,16 @@ import collections import json +from dfdatetime import factory as dfdatetime_factory +from dfdatetime import interface as dfdatetime_interface + from dfvfs.path import path_spec as dfvfs_path_spec from dfvfs.path import factory as dfvfs_path_spec_factory +# The following import is needed to make sure TSKTime is registered with +# the dfDateTime factory. +from dfvfs.vfs import tsk_file_entry # pylint: disable=unused-import + from plaso.containers import interface as containers_interface from plaso.containers import manager as containers_manager from plaso.serializer import interface @@ -117,6 +124,9 @@ def _ConvertAttributeValueToDict(cls, attribute_value): elif isinstance(attribute_value, collections.Counter): attribute_value = cls._ConvertCollectionsCounterToDict(attribute_value) + elif isinstance(attribute_value, dfdatetime_interface.DateTimeValues): + attribute_value = cls._ConvertDateTimeValuesToDict(attribute_value) + elif isinstance(attribute_value, dfvfs_path_spec.PathSpec): attribute_value = cls._ConvertPathSpecToDict(attribute_value) @@ -215,6 +225,11 @@ def _ConvertDictToObject(cls, json_dict): # Use __container_type__ to indicate the attribute container type. container_type = json_dict.get('__container_type__', None) + # Since we would like the JSON as flat as possible we handle decoding + # date time values. + elif class_type == 'DateTimeValues': + return cls._ConvertDictToDateTimeValues(json_dict) + # Since we would like the JSON as flat as possible we handle decoding # a path specification. elif class_type == 'PathSpec': @@ -330,6 +345,45 @@ def _ConvertListToObject(cls, json_list): return list_value + @classmethod + def _ConvertDictToDateTimeValues(cls, json_dict): + """Converts a JSON dict into a date time values object. + + The dictionary of the JSON serialized objects consists of: + { + '__type__': 'DateTimeValues' + '__class_name__': 'RFC2579DateTime' + ... + } + + Here '__type__' indicates the object base type. In this case this should + be 'DateTimeValues'. The rest of the elements of the dictionary make up the + date time values object properties. + + Args: + json_dict (dict[str, object]): JSON serialized objects. + + Returns: + dfdatetime.DateTimeValues: date and time values. + """ + class_name = json_dict.get('__class_name__', None) + if class_name: + del json_dict['__class_name__'] + + # Remove the class type from the JSON dict since we cannot pass it. + del json_dict['__type__'] + + is_local_time = json_dict.get('is_local_time', None) + if is_local_time is not None: + del json_dict['is_local_time'] + + date_time = dfdatetime_factory.Factory.NewDateTimeValues( + class_name, **json_dict) + if is_local_time: + date_time.is_local_time = is_local_time + + return date_time + @classmethod def _ConvertDictToPathSpec(cls, json_dict): """Converts a JSON dict into a path specification object. @@ -350,7 +404,7 @@ def _ConvertDictToPathSpec(cls, json_dict): json_dict (dict[str, object]): JSON serialized objects. Returns: - path.PathSpec: path specification. + dfvfs.PathSpec: path specification. """ type_indicator = json_dict.get('type_indicator', None) if type_indicator: @@ -365,6 +419,81 @@ def _ConvertDictToPathSpec(cls, json_dict): return dfvfs_path_spec_factory.Factory.NewPathSpec( type_indicator, **json_dict) + @classmethod + def _ConvertDateTimeValuesToDict(cls, date_time_values): + """Converts a date and time values object into a JSON dictionary. + + The resulting dictionary of the JSON serialized objects consists of: + { + '__type__': 'DateTimeValues' + '__class_name__': 'RFC2579DateTime' + ... + } + + Here '__type__' indicates the object base type. In this case + 'DateTimeValues'. The rest of the elements of the dictionary make up the + date and time value object properties. + + Args: + date_time_values (dfdatetime.DateTimeValues): date and time values. + + Returns: + dict[str, object]: JSON serialized objects. + + Raises: + TypeError: if not an instance of dfvfs.PathSpec. + """ + if not isinstance(date_time_values, dfdatetime_interface.DateTimeValues): + raise TypeError + + class_name = type(date_time_values).__name__ + + json_dict = { + '__class_name__': class_name, + '__type__': 'DateTimeValues'} + + if hasattr(date_time_values, 'timestamp'): + json_dict['timestamp'] = date_time_values.timestamp + + elif hasattr(date_time_values, 'string'): + json_dict['string'] = date_time_values.string + + elif class_name == 'FATDateTime': + # TODO: set fat_date_time value + pass + + elif class_name == 'RFC2579DateTime': + json_dict['rfc2579_date_time_tuple'] = ( + date_time_values.year, date_time_values.month, + date_time_values.day_of_month, date_time_values.hours, + date_time_values.minutes, date_time_values.seconds, + date_time_values.deciseconds) + + elif class_name == 'TimeElements': + json_dict['time_elements_tuple'] = ( + date_time_values.year, date_time_values.month, + date_time_values.day_of_month, date_time_values.hours, + date_time_values.minutes, date_time_values.seconds) + + elif class_name == 'TimeElementsInMilliseconds': + json_dict['time_elements_tuple'] = ( + date_time_values.year, date_time_values.month, + date_time_values.day_of_month, date_time_values.hours, + date_time_values.minutes, date_time_values.seconds, + date_time_values.milliseconds) + + elif class_name == 'TimeElementsInMicroseconds': + json_dict['time_elements_tuple'] = ( + date_time_values.year, date_time_values.month, + date_time_values.day_of_month, date_time_values.hours, + date_time_values.minutes, date_time_values.seconds, + date_time_values.microseconds) + + if date_time_values.is_local_time: + json_dict['is_local_time'] = True + + return json_dict + @classmethod def _ConvertPathSpecToDict(cls, path_spec_object): """Converts a path specification object into a JSON dictionary. diff --git a/requirements.txt b/requirements.txt index ae85fe253c..aba193b0f4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,7 +8,7 @@ cffi >= 1.9.1 chardet >= 2.0.1 cryptography >= 2.0.2 defusedxml >= 0.5.0 -dfdatetime >= 20200613 +dfdatetime >= 20200824 dfvfs >= 20200920 dfwinreg >= 20200927 dtfabric >= 20200621 diff --git a/setup.cfg b/setup.cfg index 3514e5c06b..0adb482a26 100644 --- a/setup.cfg +++ b/setup.cfg @@ -52,7 +52,7 @@ requires = libbde-python3 >= 20140531 python3-cryptography >= 2.0.2 python3-dateutil >= 1.5 python3-defusedxml >= 0.5.0 - python3-dfdatetime >= 20200613 + python3-dfdatetime >= 20200824 python3-dfvfs >= 20200920 python3-dfwinreg >= 20200927 python3-dtfabric >= 20200621 diff --git a/tests/containers/events.py b/tests/containers/events.py index 5319241678..4035c9e733 100644 --- a/tests/containers/events.py +++ b/tests/containers/events.py @@ -84,6 +84,7 @@ def testGetAttributeNames(self): expected_attribute_names = [ '_event_data_row_identifier', + 'date_time', 'parser', 'timestamp', 'timestamp_desc'] diff --git a/tests/containers/time_events.py b/tests/containers/time_events.py index 42be279607..8efa478fb4 100644 --- a/tests/containers/time_events.py +++ b/tests/containers/time_events.py @@ -22,7 +22,11 @@ def testGetAttributeNames(self): attribute_container = time_events.DateTimeValuesEvent(posix_time, 'usage') expected_attribute_names = [ - '_event_data_row_identifier', 'parser', 'timestamp', 'timestamp_desc'] + '_event_data_row_identifier', + 'date_time', + 'parser', + 'timestamp', + 'timestamp_desc'] attribute_names = sorted(attribute_container.GetAttributeNames())