convert attachment metadata from archivant to es format
329183
def denormalize_attachment(attachment):
        ''''''
        res = dict()
        ext = ['id', 'url']
        for k in ext:
            if k in attachment['metadata']:
                raise ValueError("metadata section could not contain special key '{}'".format(k))
            res[k] = attachment[k]
        res.update(attachment['metadata'])
        return res
1260807
function migrateMetadata(txn) {

    function decodeMetadataCompat(storedObject) {
      if (!storedObject.data) {
        // old format, when we didn't store it stringified
        storedObject.deleted = storedObject.deletedOrLocal === '1';
        return storedObject;
      }
      return decodeMetadata(storedObject);
    }

    // ensure that every metadata has a winningRev and seq,
    // which was previously created on-the-fly but better to migrate
    var bySeqStore = txn.objectStore(BY_SEQ_STORE);
    var docStore = txn.objectStore(DOC_STORE);
    var cursor = docStore.openCursor();
    cursor.onsuccess = function (e) {
      var cursor = e.target.result;
      if (!cursor) {
        return; // done
      }
      var metadata = decodeMetadataCompat(cursor.value);

      metadata.winningRev = metadata.winningRev ||
        pouchdbMerge.winningRev(metadata);

      function fetchMetadataSeq() {
        // metadata.seq was added post-3.2.0, so if it's missing,
        // we need to fetch it manually
        var start = metadata.id + '::';
        var end = metadata.id + '::\uffff';
        var req = bySeqStore.index('_doc_id_rev').openCursor(
          IDBKeyRange.bound(start, end));

        var metadataSeq = 0;
        req.onsuccess = function (e) {
          var cursor = e.target.result;
          if (!cursor) {
            metadata.seq = metadataSeq;
            return onGetMetadataSeq();
          }
          var seq = cursor.primaryKey;
          if (seq > metadataSeq) {
            metadataSeq = seq;
          }
          cursor.continue();
        };
      }

      function onGetMetadataSeq() {
        var metadataToStore = encodeMetadata(metadata,
          metadata.winningRev, metadata.deleted);

        var req = docStore.put(metadataToStore);
        req.onsuccess = function () {
          cursor.continue();
        };
      }

      if (metadata.seq) {
        return onGetMetadataSeq();
      }

      fetchMetadataSeq();
    };

  }
266934
def Convert(self, metadata, yara_match, token=None):
    """"""

    conv = ProcessToExportedProcessConverter(options=self.options)
    process = list(
        conv.Convert(ExportedMetadata(), yara_match.process, token=token))[0]

    seen_rules = set()
    for m in yara_match.match:
      if m.rule_name in seen_rules:
        continue

      seen_rules.add(m.rule_name)
      yield ExportedYaraProcessScanMatch(
          metadata=metadata,
          process=process,
          rule_name=m.rule_name,
          scan_time_us=yara_match.scan_time_us)
727719
@Override
    public <X extends Archive<X>> X getAsType(Class<X> type, ArchivePath path) {
        Validate.notNull(type, "Type must be specified");
        Validate.notNull(path, "ArchivePath must be specified");

        Node content = get(path);
        if (content == null) {
            return null;
        }
        Asset asset = content.getAsset();
        if (asset == null) {
            return null;
        }

        if (asset instanceof ArchiveAsset) {
            ArchiveAsset archiveAsset = (ArchiveAsset) asset;
            return archiveAsset.getArchive().as(type);
        }

        ArchiveFormat archiveFormat = this.configuration.getExtensionLoader()
            .getArchiveFormatFromExtensionMapping(type);
        return getAsType(type, path, archiveFormat);
    }
765116
function exportFilesystemToArchive (opts, cb) {
  return maybe(cb, async function () {
    assert(opts && typeof opts === 'object', 'opts object is required')

    // core arguments, srcPath and dstArchive
    var srcPath = opts.srcPath
    var dstArchive = opts.dstArchive
    assert(srcPath && typeof srcPath === 'string', 'srcPath is required')
    assert(dstArchive && typeof dstArchive === 'object', 'dstArchive is required')

    // options
    var dstPath = typeof opts.dstPath === 'string' ? opts.dstPath : '/'
    var ignore = Array.isArray(opts.ignore) ? opts.ignore : DEFAULT_IGNORE
    var inplaceImport = opts.inplaceImport === true
    var dryRun = opts.dryRun === true

    // ensure we have the archive's private key
    if (!dstArchive.writable && !dryRun) {
      throw new ArchiveNotWritableError()
    }

    // make source the source exists
    var srcStat
    try {
      srcStat = await fse.stat(srcPath)
    } catch (e) {
      throw new SourceNotFoundError(e.toString())
    }

    // if reading from a directory, and not doing an implace-import,
    // then put the dstPath at a subpath so that the folder's contents dont
    // get imported in-place into the archive
    if (srcStat.isDirectory() && !inplaceImport) {
      dstPath = path.join(dstPath, path.basename(srcPath))
    }

    // make sure the destination is a folder
    var dstStat
    try { dstStat = await stat(dstArchive, dstPath) } catch (e) {}
    if (!dstStat) {
      try { dstStat = await stat(dstArchive, path.dirname(dstPath)) } catch (e) {}
    }
    if (!dstStat || !dstStat.isDirectory()) {
      throw new ParentFolderDoesntExistError()
    }

    // dont overwrite directories with files
    if (!srcStat.isDirectory() && dstStat.isDirectory()) {
      dstPath = path.join(dstPath, path.basename(srcPath))
    }

    const statThenExport = async function (srcPath, dstPath) {
      // apply ignore filter
      if (ignore && match(ignore, srcPath)) {
        return
      }

      // export by type
      var srcStat = await fse.stat(srcPath)
      if (srcStat.isFile()) {
        await exportFile(srcPath, srcStat, dstPath)
      } else if (srcStat.isDirectory()) {
        await exportDirectory(srcPath, dstPath)
      }
    }

    const exportFile = async function (srcPath, srcStat, dstPath) {
      // fetch dest stats
      var dstFileStats = null
      try {
        dstFileStats = await stat(dstArchive, dstPath)
      } catch (e) {}

      // track the stats
      stats.fileCount++
      stats.totalSize += srcStat.size || 0
      if (dstFileStats) {
        if (dstFileStats.isDirectory()) {
          // delete the directory-tree
          if (!dryRun) await rmdir(dstArchive, dstPath, {recursive: true})
          stats.removedFolders.push(dstPath)
          stats.addedFiles.push(dstPath)
        } else {
          stats.updatedFiles.push(dstPath)
        }
      } else {
        stats.addedFiles.push(dstPath)
      }

      // write the file
      if (dryRun) return
      return new Promise((resolve, reject) => {
        pump(
          fse.createReadStream(srcPath),
          dstArchive.createWriteStream(dstPath),
          err => {
            if (err) reject(err)
            else resolve()
          }
        )
      })
    }

    const exportDirectory = async function (srcPath, dstPath) {
      // make sure the destination folder exists
      var dstStat
      try { dstStat = await stat(dstArchive, dstPath) } catch (e) {}
      if (!dstStat) {
        if (!dryRun) await mkdir(dstArchive, dstPath)
        stats.addedFolders.push(dstPath)
      } else if (dstStat.isFile()) {
        // a file is at the same location, remove it and add the folder
        if (!dryRun) {
          await unlink(dstArchive, dstPath)
          await mkdir(dstArchive, dstPath)
        }
        stats.addedFolders.push(dstPath)
        stats.removedFiles.push(dstPath)
        if (dryRun) {
          // dont recurse on a dry run -- not possible because the actual changes arent being made
          return
        }
      }

      // list the directory
      var fileNames = await fse.readdir(srcPath)

      // recurse into each
      var promises = fileNames.map(name => {
        return statThenExport(path.join(srcPath, name), path.join(dstPath, name))
      })
      await Promise.all(promises)
    }

    // recursively export
    var stats = {
      addedFiles: [],
      addedFolders: [],
      updatedFiles: [],
      removedFiles: [],
      removedFolders: [],
      skipCount: 0,
      fileCount: 0,
      totalSize: 0
    }
    await statThenExport(srcPath, dstPath)
    return stats
  })
}
290807
def extract_chm (archive, compression, cmd, verbosity, interactive, outdir):
    """"""
    # archmage can only extract in non-existing directories
    # so a nice dirname is created
    name = util.get_single_outfile("", archive)
    outfile = os.path.join(outdir, name)
    return [cmd, '-x', os.path.abspath(archive), outfile]
175427
private function makePrettyArchivedOnText()
    {
        $dateText = $this->metadata[DataTable::ARCHIVED_DATE_METADATA_NAME];
        $date     = Date::factory($dateText);
        $today    = mktime(0, 0, 0);

        if ($date->getTimestamp() > $today) {
            $elapsedSeconds = time() - $date->getTimestamp();
            $timeAgo        = $this->metricsFormatter->getPrettyTimeFromSeconds($elapsedSeconds);

            return Piwik::translate('CoreHome_ReportGeneratedXAgo', $timeAgo);
        }

        $prettyDate = $date->getLocalized(Date::DATE_FORMAT_SHORT);

        $timezoneAppend = ' (UTC)';
        return Piwik::translate('CoreHome_ReportGeneratedOn', $prettyDate) . $timezoneAppend;
    }
614192
function processFile(dirMetas) {
  return (file, callback) => {
    const filePath = `.${path.sep}${file.path}`;
    const dirMeta = path.parse(filePath).dir.split(path.sep)
      .map((dirName, i, arr) => path.join(...arr.slice(0, i), dirName))
      .filter(dirPath => dirPath in dirMetas)
      .reduce((obj, dirPath) => Object.assign(obj, dirMetas[dirPath]), {});
    const meta = Object.assign({}, dirMeta, file.meta);
    async.nextTick(callback, null, Object.assign({}, file, { meta }));
  };
}
446116
function initMetadataConversions() {
  var mdc = this.metadataConversions

  mdc['publish'] = {
    deserialize: function (dateString) {
      if (dateString)
        return new Date(Date.parse(dateString))
      else
        return dateString
    },
    serialize: function (date) {
      if (date instanceof Date) {
        return date.getFullYear() + '-' 
          + ('0' + (date.getMonth()+1)).slice(-2) + '-'
          + ('0' + date.getDate())
      } else {
        return date.toString()
      }
    }
  }

  mdc['tags'] = {
    deserialize: function (tagString) { //prob not very readable
      if (S(tagString).contains(',')) return tagString.split(',').map(function(tag){ return tag.trim() })
      //else if (S(tagString).contains(' ')) return tagString.split(' ').map(function(tag){ return tag.trim() })
      else return [tagString]
    },
    serialize: function (tagArray) {
      return tagArray.join(', ')
    }
  }
}
1310271
function convertForPackage(context, pkg) {
	var name = pkg.name;
	var version = pkg.version;

	var conv = context.deferredConversions;
	var pkgConv = conv[name];
	var depPkg, fns, keys = 0;
	if(pkgConv) {
		for(var range in pkgConv) {
			depPkg = crawl.matchedVersion(context, name, range);
			if(depPkg) {
				fns = pkgConv[range];
				for(var i = 0, len = fns.length; i < len; i++) {
					fns[i].call(context);
				}
				delete pkgConv[range];
			} else {
				keys++;
			}
		}
		if(keys === 0) {
			delete conv[name];
		}
	}
}
435940
def download(ctx, archive_name, filepath, version):
    '''
    
    '''

    _generate_api(ctx)
    var = ctx.obj.api.get_archive(archive_name)

    if version is None:
        version = var.get_default_version()

    var.download(filepath, version=version)

    archstr = var.archive_name +\
        '' if (not var.versioned) else ' v{}'.format(version)

    click.echo('downloaded{} to {}'.format(archstr, filepath))
141158
def ParseMetadataFile(
      self, parser_mediator, file_entry, data_stream_name):
    
    parent_path_spec = getattr(file_entry.path_spec, 'parent', None)
    filename_upper = file_entry.name.upper()
    if (self._mft_parser and parent_path_spec and
        filename_upper in ('$MFT', '$MFTMIRR') and not data_stream_name):
      self._ParseDataStreamWithParser(
          parser_mediator, self._mft_parser, file_entry, '')

    elif (self._usnjrnl_parser and parent_path_spec and
          filename_upper == '$USNJRNL' and data_stream_name == '$J'):
      # To be able to ignore the sparse data ranges the UsnJrnl parser
      # needs to read directly from the volume.
      volume_file_object = path_spec_resolver.Resolver.OpenFileObject(
          parent_path_spec, resolver_context=parser_mediator.resolver_context)

      try:
        self._ParseFileEntryWithParser(
            parser_mediator, self._usnjrnl_parser, file_entry,
            file_object=volume_file_object)
      finally:
        volume_file_object.close()
616136
function cleanDocs(docs) {
  for (var i = 0; i < docs.length; i++) {
    var doc = docs[i];
    if (doc._deleted) {
      delete doc._attachments; // ignore atts for deleted docs
    } else if (doc._attachments) {
      // filter out extraneous keys from _attachments
      var atts = Object.keys(doc._attachments);
      for (var j = 0; j < atts.length; j++) {
        var att = atts[j];
        doc._attachments[att] = pouchdbUtils.pick(doc._attachments[att],
          ['data', 'digest', 'content_type', 'length', 'revpos', 'stub']);
      }
    }
  }
}
562584
function convertFileType (data) {
  data['type'] = 'string'
  data['media'] = { 'binaryEncoding': 'binary' }
  if (data.fileTypes) {
    data['media']['anyOf'] = []
    data.fileTypes.forEach(function (el) {
      data['media']['anyOf'].push({ 'mediaType': el })
    })
    delete data.fileTypes
  }
  return data
}
1313737
public function generateAttachmentMetadata($metadata, $attachment_id) {
		if(!$this->enabledAlternativeFormats) {
			return $metadata;
		}

		$mime = get_post_mime_type($attachment_id);
		if($mime == 'application/vnd.adobe.illustrator') {
			$file = get_attached_file($attachment_id);
			$fileName = pathinfo($file, PATHINFO_BASENAME);
			$fallback_sizes = array(
				'thumbnail',
				'medium',
				'medium_large',
				'large',
			);
			$fallback_sizes = apply_filters('fallback_intermediate_image_sizes', $fallback_sizes, $metadata);
			$additional_sizes = wp_get_additional_image_sizes();

			$metadata = [
				'sizes' => [
					'full' => [
						'file' => $fileName,
						'width' => null,
						'height' => null,
						'mime-type' => 'image/jpeg'
					]
				]
			];

			foreach($fallback_sizes as $sz) {
				$metadata['sizes'][$sz] = [
					'file' => $fileName,
					'width' => get_option("{$sz}_size_w"),
					'height' => get_option("{$sz}_size_h"),
					'mime-type' => 'image/jpeg'
				];
			}

			foreach($additional_sizes as $size => $sizeInfo) {
				if(isset($sizeInfo['crop']) && $sizeInfo['crop']) {
					$metadata['sizes'][$size] = [
						'file' => $fileName,
						'width' => $sizeInfo['width'],
						'height' => $sizeInfo['height'],
						'mime-type' => 'image/jpeg'
					];
				}
			}

			$upload_info = wp_upload_dir();
			$upload_path = $upload_info['basedir'];
			$metadata['file'] = trim(str_replace($upload_path, '', $file), '/');
		}

		return $metadata;
	}
879540
private synchronized void readAttachmentsFromAsset(Asset ass) {
        Collection<Attachment> attachments = ass.getAttachments();

        _attachments = new HashMap<String, AttachmentResourceImpl>();
        if (attachments != null) {
            for (Attachment at : attachments) {
                _attachments.put(at.getName(), new AttachmentResourceImpl(at));

                if (at.getType() == AttachmentType.CONTENT) {
                    _contentAttached = true;
                }
            }
        }

    }
66497
def convert_content(self) -> dict:
        """"""
        source_content = self.load_content()

        converted = {}

        md_extenstions = [
            'full_yaml_metadata',
            'markdown.extensions.fenced_code',
        ]
        md = markdown.Markdown(md_extenstions)
        content = md.convert(source_content)
        meta = md.Meta

        if meta:
            converted.update(meta)

        if content:
            converted['content'] = content.strip()

        return converted