Processes a metadata file.

    Args:
      mediator (ParserMediator): mediates the interactions between
          parsers and other components, such as storage and abort signals.
      file_entry (dfvfs.FileEntry): file entry of the metadata file.
546395
def _ProcessMetadataFile(self, mediator, file_entry):
    
    self.processing_status = definitions.STATUS_INDICATOR_EXTRACTING

    self._event_extractor.ParseFileEntryMetadata(mediator, file_entry)
    for data_stream in file_entry.data_streams:
      if self._abort:
        break
      self.last_activity_timestamp = time.time()

      self._event_extractor.ParseMetadataFile(
          mediator, file_entry, data_stream.name)
641351
def GetEntries(
      self, parser_mediator, cookie_data=None, url=None, **kwargs):
    
    fields = cookie_data.split('.')
    number_of_fields = len(fields)

    if number_of_fields not in (1, 6):
      parser_mediator.ProduceExtractionWarning(
          'unsupported number of fields: {0:d} in cookie: {1:s}'.format(
              number_of_fields, self.COOKIE_NAME))
      return

    if number_of_fields == 1:
      domain_hash = None
      visitor_identifier = None
      first_visit_posix_time = None
      previous_visit_posix_time = None

      try:
        # TODO: fix that we're losing precision here use dfdatetime.
        last_visit_posix_time = int(fields[0], 10) / 10000000
      except ValueError:
        last_visit_posix_time = None

      number_of_sessions = None

    elif number_of_fields == 6:
      domain_hash = fields[0]
      visitor_identifier = fields[1]

      # TODO: Double check this time is stored in UTC and not local time.
      try:
        first_visit_posix_time = int(fields[2], 10)
      except ValueError:
        first_visit_posix_time = None

      try:
        previous_visit_posix_time = int(fields[3], 10)
      except ValueError:
        previous_visit_posix_time = None

      try:
        last_visit_posix_time = int(fields[4], 10)
      except ValueError:
        last_visit_posix_time = None

      try:
        number_of_sessions = int(fields[5], 10)
      except ValueError:
        number_of_sessions = None

    event_data = GoogleAnalyticsEventData('utma')
    event_data.cookie_name = self.COOKIE_NAME
    event_data.domain_hash = domain_hash
    event_data.sessions = number_of_sessions
    event_data.url = url
    event_data.visitor_id = visitor_identifier

    if first_visit_posix_time is not None:
      date_time = dfdatetime_posix_time.PosixTime(
          timestamp=first_visit_posix_time)
      event = time_events.DateTimeValuesEvent(
          date_time, 'Analytics Creation Time')
      parser_mediator.ProduceEventWithEventData(event, event_data)

    if previous_visit_posix_time is not None:
      date_time = dfdatetime_posix_time.PosixTime(
          timestamp=previous_visit_posix_time)
      event = time_events.DateTimeValuesEvent(
          date_time, 'Analytics Previous Time')
      parser_mediator.ProduceEventWithEventData(event, event_data)

    date_time = None
    if last_visit_posix_time is not None:
      date_time = dfdatetime_posix_time.PosixTime(
          timestamp=last_visit_posix_time)
      timestamp_description = definitions.TIME_DESCRIPTION_LAST_VISITED
    elif first_visit_posix_time is None and previous_visit_posix_time is None:
      # If both creation_time and written_time are None produce an event
      # object without a timestamp.
      date_time = dfdatetime_semantic_time.SemanticTime('Not set')
      timestamp_description = definitions.TIME_DESCRIPTION_NOT_A_TIME

    if date_time is not None:
      event = time_events.DateTimeValuesEvent(date_time, timestamp_description)
      parser_mediator.ProduceEventWithEventData(event, event_data)
25249
def _ProcessDirectory(self, mediator, file_entry):
    
    self.processing_status = definitions.STATUS_INDICATOR_COLLECTING

    if self._processing_profiler:
      self._processing_profiler.StartTiming('collecting')

    for sub_file_entry in file_entry.sub_file_entries:
      if self._abort:
        break

      try:
        if not sub_file_entry.IsAllocated():
          continue

      except dfvfs_errors.BackEndError as exception:
        warning_message = (
            'unable to process directory entry: {0:s} with error: '
            '{1!s}').format(sub_file_entry.name, exception)
        mediator.ProduceExtractionWarning(
            warning_message, path_spec=file_entry.path_spec)
        continue

      # For TSK-based file entries only, ignore the virtual /$OrphanFiles
      # directory.
      if sub_file_entry.type_indicator == dfvfs_definitions.TYPE_INDICATOR_TSK:
        if file_entry.IsRoot() and sub_file_entry.name == '$OrphanFiles':
          continue

      event_source = event_sources.FileEntryEventSource(
          path_spec=sub_file_entry.path_spec)

      # TODO: move this into a dfVFS file entry property.
      stat_object = sub_file_entry.GetStat()
      if stat_object:
        event_source.file_entry_type = stat_object.type

      mediator.ProduceEventSource(event_source)

      self.last_activity_timestamp = time.time()

    if self._processing_profiler:
      self._processing_profiler.StopTiming('collecting')

    self.processing_status = definitions.STATUS_INDICATOR_RUNNING
760935
def _process_incoming(self, xmlstream, queue_entry):
        

        stanza_obj, exc = queue_entry

        # first, handle SM stream objects
        if isinstance(stanza_obj, nonza.SMAcknowledgement):
            self._logger.debug("received SM ack: %r", stanza_obj)
            if not self._sm_enabled:
                self._logger.warning("received SM ack, but SM not enabled")
                return
            self.sm_ack(stanza_obj.counter)
            return
        elif isinstance(stanza_obj, nonza.SMRequest):
            self._logger.debug("received SM request: %r", stanza_obj)
            if not self._sm_enabled:
                self._logger.warning("received SM request, but SM not enabled")
                return
            response = nonza.SMAcknowledgement()
            response.counter = self._sm_inbound_ctr
            self._logger.debug("sending SM ack: %r", response)
            xmlstream.send_xso(response)
            return

        # raise if it is not a stanza
        if not isinstance(stanza_obj, stanza.StanzaBase):
            raise RuntimeError(
                "unexpected stanza class: {}".format(stanza_obj))

        # now handle stanzas, these always increment the SM counter
        if self._sm_enabled:
            self._sm_inbound_ctr += 1
            self._sm_inbound_ctr &= 0xffffffff

        # check if the stanza has errors
        if exc is not None:
            self._process_incoming_erroneous_stanza(stanza_obj, exc)
            return

        if isinstance(stanza_obj, stanza.IQ):
            self._process_incoming_iq(stanza_obj)
        elif isinstance(stanza_obj, stanza.Message):
            self._process_incoming_message(stanza_obj)
        elif isinstance(stanza_obj, stanza.Presence):
            self._process_incoming_presence(stanza_obj)
1358042
def ParseFileObject(self, parser_mediator, file_object):
    
    esedb_file = pyesedb.file()

    try:
      esedb_file.open_file_object(file_object)
    except IOError as exception:
      parser_mediator.ProduceExtractionWarning(
          'unable to open file with error: {0!s}'.format(exception))
      return

    # Compare the list of available plugin objects.
    cache = ESEDBCache()
    try:
      table_names = frozenset(self._GetTableNames(esedb_file))

      for plugin in self._plugins:
        if parser_mediator.abort:
          break

        if not plugin.required_tables.issubset(table_names):
          continue

        try:
          plugin.UpdateChainAndProcess(
              parser_mediator, cache=cache, database=esedb_file)

        except Exception as exception:  # pylint: disable=broad-except
          parser_mediator.ProduceExtractionWarning((
              'plugin: {0:s} unable to parse ESE database with error: '
              '{1!s}').format(plugin.NAME, exception))

    finally:
      # TODO: explicitly clean up cache.

      esedb_file.close()
484862
public void updateEntry(final Entry entry, String fsid) throws Exception {
        synchronized (FileStore.getFileStore()) {

            final Feed f = getFeedDocument();

            if (fsid.endsWith(".media-link")) {
                fsid = fsid.substring(0, fsid.length() - ".media-link".length());
            }

            updateTimestamps(entry);

            updateEntryAppLinks(entry, fsid, false);
            updateFeedDocumentWithExistingEntry(f, entry);

            final String entryPath = getEntryPath(fsid);
            final OutputStream os = FileStore.getFileStore().getFileOutputStream(entryPath);
            updateEntryAppLinks(entry, fsid, true);
            Atom10Generator.serializeEntry(entry, new OutputStreamWriter(os, "UTF-8"));
            os.flush();
            os.close();
        }
    }
325224
def ParseFileObject(self, parser_mediator, file_object):
    
    volume = pyfsntfs.volume()
    try:
      volume.open_file_object(file_object)
    except IOError as exception:
      parser_mediator.ProduceExtractionWarning(
          'unable to open NTFS volume with error: {0!s}'.format(exception))

    try:
      usn_change_journal = volume.get_usn_change_journal()
      self._ParseUSNChangeJournal(parser_mediator, usn_change_journal)
    finally:
      volume.close()
522933
def _ParseRecords(self, parser_mediator, evtx_file):
    
    # To handle errors when parsing a Windows XML EventLog (EVTX) file in the
    # most granular way the following code iterates over every event record.
    # The call to evt_file.get_record() and access to members of evt_record
    # should be called within a try-except.

    for record_index in range(evtx_file.number_of_records):
      if parser_mediator.abort:
        break

      try:
        evtx_record = evtx_file.get_record(record_index)
        self._ParseRecord(parser_mediator, record_index, evtx_record)

      except IOError as exception:
        parser_mediator.ProduceExtractionWarning(
            'unable to parse event record: {0:d} with error: {1!s}'.format(
                record_index, exception))

    for record_index in range(evtx_file.number_of_recovered_records):
      if parser_mediator.abort:
        break

      try:
        evtx_record = evtx_file.get_recovered_record(record_index)
        self._ParseRecord(
            parser_mediator, record_index, evtx_record, recovered=True)

      except IOError as exception:
        parser_mediator.ProduceExtractionWarning((
            'unable to parse recovered event record: {0:d} with error: '
            '{1!s}').format(record_index, exception))
1298830
def GetEntries(
      self, parser_mediator, cookie_data=None, url=None, **kwargs):
    
    fields = cookie_data.split('.')
    number_of_fields = len(fields)

    if number_of_fields > 5:
      variables = '.'.join(fields[4:])
      fields = fields[0:4]
      fields.append(variables)
      number_of_fields = len(fields)

    if number_of_fields not in (1, 5):
      parser_mediator.ProduceExtractionWarning(
          'unsupported number of fields: {0:d} in cookie: {1:s}'.format(
              number_of_fields, self.COOKIE_NAME))
      return

    if number_of_fields == 1:
      domain_hash = None

      try:
        # TODO: fix that we're losing precision here use dfdatetime.
        last_visit_posix_time = int(fields[0], 10) / 10000000
      except ValueError:
        last_visit_posix_time = None

      number_of_sessions = None
      number_of_sources = None
      extra_attributes = {}

    elif number_of_fields == 5:
      domain_hash = fields[0]

      try:
        last_visit_posix_time = int(fields[1], 10)
      except ValueError:
        last_visit_posix_time = None

      try:
        number_of_sessions = int(fields[2], 10)
      except ValueError:
        number_of_sessions = None

      try:
        number_of_sources = int(fields[3], 10)
      except ValueError:
        number_of_sources = None

      extra_variables = fields[4].split('|')

      extra_attributes = {}
      for variable in extra_variables:
        key, _, value = variable.partition('=')

        # Urllib2 in Python 2 requires a 'str' argument, not 'unicode'. We thus
        # need to convert the value argument to 'str" and back again, but only
        # in Python 2.
        if isinstance(value, py2to3.UNICODE_TYPE) and py2to3.PY_2:
          try:
            value = codecs.decode(value, 'ascii')
          except UnicodeEncodeError:
            value = codecs.decode(value, 'ascii', errors='replace')
            parser_mediator.ProduceExtractionWarning(
                'Cookie contains non 7-bit ASCII characters, which have been '
                'replaced with a "?".')

        value = urlparse.unquote(value)

        if py2to3.PY_2:
          try:
            value = codecs.encode(value, 'utf-8')
          except UnicodeDecodeError:
            value = codecs.encode(value, 'utf-8', errors='replace')
            parser_mediator.ProduceExtractionWarning(
                'Cookie value did not contain a Unicode string. Non UTF-8 '
                'characters have been replaced.')

        extra_attributes[key] = value

    if last_visit_posix_time is not None:
      date_time = dfdatetime_posix_time.PosixTime(
          timestamp=last_visit_posix_time)
      timestamp_description = definitions.TIME_DESCRIPTION_LAST_VISITED
    else:
      date_time = dfdatetime_semantic_time.SemanticTime('Not set')
      timestamp_description = definitions.TIME_DESCRIPTION_NOT_A_TIME

    event_data = GoogleAnalyticsEventData('utmz')
    event_data.cookie_name = self.COOKIE_NAME
    event_data.domain_hash = domain_hash
    event_data.sessions = number_of_sessions
    event_data.sources = number_of_sources
    event_data.url = url

    for key, value in iter(extra_attributes.items()):
      setattr(event_data, key, value)

    event = time_events.DateTimeValuesEvent(date_time, timestamp_description)
    parser_mediator.ProduceEventWithEventData(event, event_data)
912155
def GetEntries(
      self, parser_mediator, cookie_data=None, url=None, **kwargs):
    
    fields = cookie_data.split('.')
    number_of_fields = len(fields)

    if number_of_fields not in (1, 4):
      parser_mediator.ProduceExtractionWarning(
          'unsupported number of fields: {0:d} in cookie: {1:s}'.format(
              number_of_fields, self.COOKIE_NAME))
      return

    if number_of_fields == 1:
      domain_hash = None

      try:
        # TODO: fix that we're losing precision here use dfdatetime.
        last_visit_posix_time = int(fields[0], 10) / 10000000
      except ValueError:
        last_visit_posix_time = None

      number_of_pages_viewed = None

    elif number_of_fields == 4:
      domain_hash = fields[0]

      try:
        number_of_pages_viewed = int(fields[1], 10)
      except ValueError:
        number_of_pages_viewed = None

      try:
        if fields[2] in ('8', '9'):
          # TODO: fix that we're losing precision here use dfdatetime.
          last_visit_posix_time = int(fields[3], 10) / 1000
        else:
          last_visit_posix_time = int(fields[3], 10)
      except ValueError:
        last_visit_posix_time = None

    if last_visit_posix_time is not None:
      date_time = dfdatetime_posix_time.PosixTime(
          timestamp=last_visit_posix_time)
      timestamp_description = definitions.TIME_DESCRIPTION_LAST_VISITED
    else:
      date_time = dfdatetime_semantic_time.SemanticTime('Not set')
      timestamp_description = definitions.TIME_DESCRIPTION_NOT_A_TIME

    event_data = GoogleAnalyticsEventData('utmb')
    event_data.cookie_name = self.COOKIE_NAME
    event_data.domain_hash = domain_hash
    event_data.pages_viewed = number_of_pages_viewed
    event_data.url = url

    event = time_events.DateTimeValuesEvent(date_time, timestamp_description)
    parser_mediator.ProduceEventWithEventData(event, event_data)
878052
public Entry updateMediaEntry(final String fileName, final String contentType, final InputStream is) throws Exception {
        synchronized (FileStore.getFileStore()) {

            final File tempFile = File.createTempFile(fileName, "tmp");
            final FileOutputStream fos = new FileOutputStream(tempFile);
            Utilities.copyInputToOutput(is, fos);
            fos.close();

            // Update media file
            final FileInputStream fis = new FileInputStream(tempFile);
            saveMediaFile(fileName, contentType, tempFile.length(), fis);
            fis.close();
            final File resourceFile = new File(getEntryMediaPath(fileName));

            // Load media-link entry to return
            final String entryPath = getEntryPath(fileName);
            final InputStream in = FileStore.getFileStore().getFileInputStream(entryPath);
            final Entry atomEntry = loadAtomResourceEntry(in, resourceFile);

            updateTimestamps(atomEntry);
            updateMediaEntryAppLinks(atomEntry, fileName, false);

            // Update feed with new entry
            final Feed f = getFeedDocument();
            updateFeedDocumentWithExistingEntry(f, atomEntry);

            // Save updated media-link entry
            final OutputStream os = FileStore.getFileStore().getFileOutputStream(entryPath);
            updateMediaEntryAppLinks(atomEntry, fileName, true);
            Atom10Generator.serializeEntry(atomEntry, new OutputStreamWriter(os, "UTF-8"));
            os.flush();
            os.close();

            return atomEntry;
        }
    }
798238
public Object process(BinaryEntry entry) {
        Object result = processEntry(entry);
        return result == NO_RESULT ? null : result;
    }
11875
def ParseFileObject(self, parser_mediator, file_object):
    
    msiecf_file = pymsiecf.file()
    msiecf_file.set_ascii_codepage(parser_mediator.codepage)

    try:
      msiecf_file.open_file_object(file_object)
    except IOError as exception:
      parser_mediator.ProduceExtractionWarning(
          'unable to open file with error: {0!s}'.format(exception))
      return

    try:
      self._ParseItems(parser_mediator, msiecf_file)
    finally:
      msiecf_file.close()
729214
def _ProcessFileEntry(self, mediator, file_entry):
    
    display_name = mediator.GetDisplayName()
    logger.debug(
        '[ProcessFileEntry] processing file entry: {0:s}'.format(display_name))

    reference_count = mediator.resolver_context.GetFileObjectReferenceCount(
        file_entry.path_spec)

    try:
      if self._IsMetadataFile(file_entry):
        self._ProcessMetadataFile(mediator, file_entry)

      else:
        file_entry_processed = False
        for data_stream in file_entry.data_streams:
          if self._abort:
            break

          if self._CanSkipDataStream(file_entry, data_stream):
            logger.debug((
                '[ProcessFileEntry] Skipping datastream {0:s} for {1:s}: '
                '{2:s}').format(
                    data_stream.name, file_entry.type_indicator, display_name))
            continue

          self._ProcessFileEntryDataStream(mediator, file_entry, data_stream)

          file_entry_processed = True

        if not file_entry_processed:
          # For when the file entry does not contain a data stream.
          self._ProcessFileEntryDataStream(mediator, file_entry, None)

    finally:
      new_reference_count = (
          mediator.resolver_context.GetFileObjectReferenceCount(
              file_entry.path_spec))
      if reference_count != new_reference_count:
        # Clean up after parsers that do not call close explicitly.
        if mediator.resolver_context.ForceRemoveFileObject(
            file_entry.path_spec):
          logger.warning(
              'File-object not explicitly closed for file: {0:s}'.format(
                  display_name))

    logger.debug(
        '[ProcessFileEntry] done processing file entry: {0:s}'.format(
            display_name))
887653
def CompileReport(self, mediator):
    
    report_text = 'Tagging plugin produced {0:d} tags.\n'.format(
        self._number_of_event_tags)
    self._number_of_event_tags = 0
    return reports.AnalysisReport(plugin_name=self.NAME, text=report_text)
507563
def ParseFileObject(self, parser_mediator, file_object):
    
    evt_file = pyevt.file()
    evt_file.set_ascii_codepage(parser_mediator.codepage)

    try:
      evt_file.open_file_object(file_object)
    except IOError as exception:
      parser_mediator.ProduceExtractionWarning(
          'unable to open file with error: {0!s}'.format(exception))
      return

    try:
      self._ParseRecords(parser_mediator, evt_file)
    finally:
      evt_file.close()
395846
static <K, V> Map.Entry<K, V> parseEntry(CodedInputStream input, Metadata<K, V> metadata,
            ExtensionRegistryLite extensionRegistry) throws IOException {
        K key = metadata.defaultKey;
        V value = metadata.defaultValue;
        while (true) {
            int tag = input.readTag();
            if (tag == 0) {
                break;
            }
            if (tag == CodedConstant.makeTag(KEY_FIELD_NUMBER, metadata.keyType.getWireType())) {
                key = parseField(input, extensionRegistry, metadata.keyType, key);
            } else if (tag == CodedConstant.makeTag(VALUE_FIELD_NUMBER, metadata.valueType.getWireType())) {
                value = parseField(input, extensionRegistry, metadata.valueType, value);
            } else {
                if (!input.skipField(tag)) {
                    break;
                }
            }
        }
        return new AbstractMap.SimpleImmutableEntry<K, V>(key, value);
    }