Files extractor command line
334691
def main():
    """"""

    parser = argparse.ArgumentParser(
        description=__doc__,
        formatter_class=argparse.RawDescriptionHelpFormatter)
    parser.add_argument('source', help="location of the downloaded repos")
    parser.add_argument('destination', help="location of the extracted files")
    parser.add_argument(
        '-t', '--nb-test-files', help="number of testing files per language",
        type=int, default=5000)
    parser.add_argument(
        '-l', '--nb-learn-files', help="number of learning files per language",
        type=int, default=10000)
    parser.add_argument(
        '-r', '--remove', help="remove repos that cannot be read",
        action='store_true', default=False)
    parser.add_argument(
        '-d', '--debug', default=False, action='store_true',
        help="show debug messages")

    args = parser.parse_args()
    config_logging(args.debug)

    source = Path(args.source)
    destination = Path(args.destination)
    nb_test = args.nb_test_files
    nb_learn = args.nb_learn_files
    remove = args.remove

    repos = _find_repos(source)
    split_repos = _split_repos(repos, nb_test, nb_learn)
    split_files = _find_files(*split_repos, nb_test, nb_learn, remove)
    _unzip_all(*split_files, destination)
    LOGGER.info("Files saved into %s", destination)
    LOGGER.debug("Exit OK")
1018718
def _extract_file(zip_fp, info, path):
    """"""
    zip_fp.extract(info.filename, path=path)
    out_path = os.path.join(path, info.filename)

    perm = info.external_attr >> 16
    perm |= stat.S_IREAD  # make sure we're not accidentally setting this to 0
    os.chmod(out_path, perm)
341761
def _extract_archive(archive_fpath, archive_file, archive_namelist, output_dir,
                     force_commonprefix=True, prefix=None,
                     dryrun=False, verbose=not QUIET, overwrite=None):
    
    # force extracted components into a subdirectory if force_commonprefix is
    # on return_path = output_diG
    # FIXMpathE doesn't work right
    if prefix is not None:
        output_dir = join(output_dir, prefix)
        util_path.ensurepath(output_dir)

    archive_basename, ext = split_archive_ext(basename(archive_fpath))
    if force_commonprefix and commonprefix(archive_namelist) == '':
        # use the archivename as the default common prefix
        output_dir = join(output_dir, archive_basename)
        util_path.ensurepath(output_dir)

    for member in archive_namelist:
        (dname, fname) = split(member)
        dpath = join(output_dir, dname)
        util_path.ensurepath(dpath)
        if verbose:
            print('[utool] Unarchive ' + fname + ' in ' + dpath)

        if not dryrun:
            if overwrite is False:
                if exists(join(output_dir, member)):
                    continue
            archive_file.extract(member, path=output_dir)
    return output_dir
115527
private static CommandLine parseArgs(String[] args) throws ParseException {
    Options options = new Options();

    // Input
    Option o = new Option("i", "input", true,
        "input directory as hdfs path, or local as file://");
    o.setArgName("input-path");
    o.setRequired(true);
    options.addOption(o);

    // Input
    o = new Option("o", "output", true, "output directory");
    o.setArgName("input-path");
    o.setRequired(true);
    options.addOption(o);

    // Whether to skip existing files or not.
    o = new Option("s", "skipExisting", false,
        "skip existing files. Cannot be used together with m for move.");
    o.setRequired(false);
    options.addOption(o);

    // Maximum number of files to retain in the specified input directory.
    o = new Option(
        "x",
        "maXretention",
        true,
        "The maximum number of the most recent files to retain in the input directory after processing."
            + " Can be used by HDFS input paths only. Mutually exclusive with s (move),"
            + " but can be used in combination with s (skipExisting)");
    o.setRequired(false);
    options.addOption(o);

    // Whether files need to be moved
    o = new Option("m", "move", false, "move all files rather than copying."
        + "Delete source if target already exists."
        + " Can be used with HDFS input paths only. "
        + " Mutually exlusive with s (skipExisting)");
    o.setRequired(false);
    options.addOption(o);

    // Debugging
    options.addOption("d", "debug", false, "switch on DEBUG log level");
    o.setRequired(false);
    options.addOption(o);

    CommandLineParser parser = new PosixParser();
    CommandLine commandLine = null;
    try {
      commandLine = parser.parse(options, args);
    } catch (Exception e) {
      System.err.println("ERROR: " + e.getMessage() + "\n");
      HelpFormatter formatter = new HelpFormatter();
      formatter.printHelp(NAME + " ", options, true);
      System.exit(-1);
    }

    return commandLine;
  }
290807
def extract_chm (archive, compression, cmd, verbosity, interactive, outdir):
    """"""
    # archmage can only extract in non-existing directories
    # so a nice dirname is created
    name = util.get_single_outfile("", archive)
    outfile = os.path.join(outdir, name)
    return [cmd, '-x', os.path.abspath(archive), outfile]
218877
def extract(args):
    
    p = OptionParser(extract.__doc__)
    p.add_option("--format", default=False, action="store_true",
            help="enable flag to reformat header into a symbol separated list of constituent reads "+ \
            "[default: %default]")
    p.add_option("--singlets", default=False, action="store_true",
            help="ask the program to look in the singlets file (should be in the same folder) for " +\
            "unused reads and put them in the resultant fasta file [default: %default]")
    p.set_sep(sep="|", help="Separator used to list the reads in the FASTA header")
    opts, args = p.parse_args(args)

    if len(args) != 1:
        sys.exit(not p.print_help())

    acefile, = args
    ace = Ace.read(must_open(acefile))
    logging.debug('Loaded ace file {0}'.format(acefile))

    fastafile = acefile.rsplit(".", 1)[0] + ".fasta"
    fw = open(fastafile, "w")
    for c in ace.contigs:
        id = c.name
        if opts.format:
           id = opts.sep.join([read.name for read in c.af])

        seqrec = SeqRecord(Seq(c.sequence), id=id, description="")
        SeqIO.write([seqrec], fw, "fasta")

    if opts.singlets:
        singletsfile = acefile.rsplit(".", 1)[0] + ".singlets"
        if os.path.getsize(singletsfile) > 0:
            fp = SeqIO.parse(must_open(singletsfile), "fasta")
            for rec in fp:
                SeqIO.write(rec, fw, "fasta")

    fw.close()
    logging.debug('Wrote contigs to fasta file {0}'.format(fastafile))
239998
def _extract_archive(file, target_dir):
    
    if file.endswith('.gz') or file.endswith('.tar') or file.endswith('.tgz'):
        archive = tarfile.open(file, 'r')
    elif file.endswith('.zip'):
        archive = zipfile.ZipFile(file, 'r')
    else:
        raise Exception('Unrecognized file type: ' + file)
    archive.extractall(path=target_dir)
    archive.close()
1232756
def cli_main():
    """"""
    import argparse
    import os

    def is_file_or_pipe(arg):
        if not os.path.exists(arg) or os.path.isdir(arg):
            parser.error('The file {0} does not exist!'.format(arg))
        else:
            return arg

    def is_dir(arg):
        if not os.path.isdir(arg):
            parser.error('The directory {0} does not exist!'.format(arg))
        else:
            return arg

    parser = argparse.ArgumentParser(description=__doc__)

    parser.add_argument('-v', '--version', action='version',
                        version=version)

    parser.add_argument('template', help='The mustache file',
                        type=is_file_or_pipe)

    parser.add_argument('-d', '--data', dest='data',
                        help='The json data file',
                        type=is_file_or_pipe, default={})

    parser.add_argument('-p', '--path', dest='partials_path',
                        help='The directory where your partials reside',
                        type=is_dir, default='.')

    parser.add_argument('-e', '--ext', dest='partials_ext',
                        help='The extension for your mustache\
                              partials, \'mustache\' by default',
                        default='mustache')

    parser.add_argument('-l', '--left-delimiter', dest='def_ldel',
                        help='The default left delimiter, "{{" by default.',
                        default='{{')

    parser.add_argument('-r', '--right-delimiter', dest='def_rdel',
                        help='The default right delimiter, "}}" by default.',
                        default='}}')

    args = vars(parser.parse_args())

    try:
        sys.stdout.write(main(**args))
        sys.stdout.flush()
    except SyntaxError as e:
        print('Chevron: syntax error')
        print('    ' + '\n    '.join(e.args[0].split('\n')))
        exit(1)
192774
def extractall(archive, filename, dstdir):
    
    
    if zipfile.is_zipfile(archive):
        z = zipfile.ZipFile(archive)
        for name in z.namelist():
            targetname = name
            # directories ends with '/' (on Windows as well)
            if targetname.endswith('/'):
                targetname = targetname[:-1]

            # don't include leading "/" from file name if present
            if targetname.startswith(os.path.sep):
                targetname = os.path.join(dstdir, targetname[1:])
            else:
                targetname = os.path.join(dstdir, targetname)                
            targetname = os.path.normpath(targetname)

            # Create all upper directories if necessary.    
            upperdirs = os.path.dirname(targetname)
            if upperdirs and not os.path.exists(upperdirs):
                os.makedirs(upperdirs)

            # directories ends with '/' (on Windows as well)
            if not name.endswith('/'):
                # copy file
                file(targetname, 'wb').write(z.read(name))

    elif tarfile.is_tarfile(archive):
        tar = tarfile.open(archive)
        tar.extractall(path=dstdir)
    else:
        # seems to be a single file, save it
        shutil.copyfile(archive, os.path.join(dstdir, filename))
687041
def extract_gzip (archive, compression, cmd, verbosity, interactive, outdir):
    """"""
    targetname = util.get_single_outfile(outdir, archive)
    try:
        with gzip.GzipFile(archive) as gzipfile:
            with open(targetname, 'wb') as targetfile:
                data = gzipfile.read(READ_SIZE_BYTES)
                while data:
                    targetfile.write(data)
                    data = gzipfile.read(READ_SIZE_BYTES)
    except Exception as err:
        msg = "error extracting %s to %s: %s" % (archive, targetname, err)
        raise util.PatoolError(msg)
    return None
1155498
function(archive, env) {

    // The extraction command
    var extract = [
      'tar',
      '-zxvf',
      archive,
      '-C',
      '/media',
      '&&',
      'mv',
      '/media/files_' + env + '/*',
      '/media'
    ];

    // Extract and remove
    return run('usermap', extract)
    .then(function() {
      return run('rm', ['-rf', '/media/files_' + env]);
    })
    .then(function() {
      return run('rm', ['-f', archive]);
    });

  }
123003
def cli(list_files, config, ignore, path):
    
    files = MarkdownFileFinder.find_files(path)
    if list_files:
        echo_files(files)

    lint_config = get_lint_config(config)
    lint_config.apply_on_csv_string(ignore, lint_config.disable_rule)

    linter = MarkdownLinter(lint_config)
    error_count = linter.lint_files(files)
    exit(error_count)
4781
def _extract(filename):
        
        random_string = '{:0d}'.format(randint(0, 10**6))
        tmp = '/tmp/kytos-napp-' + Path(filename).stem + '-' + random_string
        os.mkdir(tmp)
        with tarfile.open(filename, 'r:xz') as tar:
            tar.extractall(tmp)
        return Path(tmp)
680434
def extract_dirs(mod, path, dst, verbose=False, exclude=None, exclude_ext=None, recursion=True, replace=True):
    
    default_exclude = ['.svn', '_svn', '.git']
    default_exclude_ext = ['.pyc', '.pyo', '.bak', '.tmp']
    exclude = exclude or []
    exclude_ext = exclude_ext or []
#    log = logging.getLogger('uliweb')
    if not os.path.exists(dst):
        os.makedirs(dst)
        if verbose:
            print 'Make directory %s' % dst
    for r in pkg.resource_listdir(mod, path):
        if r in exclude or r in default_exclude:
            continue
        fpath = os.path.join(path, r)
        if pkg.resource_isdir(mod, fpath):
            if recursion:
                extract_dirs(mod, fpath, os.path.join(dst, r), verbose, exclude, exclude_ext, recursion, replace)
        else:
            ext = os.path.splitext(fpath)[1]
            if ext in exclude_ext or ext in default_exclude_ext:
                continue
            extract_file(mod, fpath, dst, verbose, replace)
1026307
def main():
    """"""
    description = ('PATH can be either the path to a scratch file, or a '
                   'directory containing scratch files. Multiple PATH '
                   'arguments can be provided.')
    parser = OptionParser(usage='%prog -p PLUGIN_NAME [options] PATH...',
                          description=description,
                          version='%prog {}'.format(__version__))
    parser.add_option('-d', '--plugin-dir', metavar='DIR',
                      help=('Specify the path to a directory containing '
                            'plugins. Plugins in this directory take '
                            'precedence over similarly named plugins '
                            'included with Hairball.'))
    parser.add_option('-p', '--plugin', action='append',
                      help=('Use the named plugin to perform analysis. '
                            'This option can be provided multiple times.'))
    parser.add_option('-k', '--kurt-plugin', action='append',
                      help=('Provide either a python import path (e.g, '
                            'kelp.octopi) to a package/module, or the path'
                            ' to a python file, which will be loaded as a '
                            'Kurt plugin. This option can be provided '
                            'multiple times.'))
    parser.add_option('-q', '--quiet', action='store_true',
                      help=('Prevent output from Hairball. Plugins may still '
                            'produce output.'))
    parser.add_option('-C', '--no-cache', action='store_true',
                      help='Do not use Hairball\'s cache.', default=False)
    options, args = parser.parse_args(sys.argv[1:])

    if not options.plugin:
        parser.error('At least one plugin must be specified via -p.')
    if not args:
        parser.error('At least one PATH must be provided.')

    if options.plugin_dir:
        if os.path.isdir(options.plugin_dir):
            sys.path.append(options.plugin_dir)
        else:
            parser.error('{} is not a directory'.format(options.plugin_dir))

    hairball = Hairball(options, args, cache=not options.no_cache)
    hairball.initialize_plugins()
    hairball.process()
    hairball.finalize()
900933
function extractText( filePath, options, cb ) {
  var result = ''
    , error = null
    , textutil = spawn( 'textutil', ['-convert', 'txt', '-stdout', filePath] )
    ;

  textutil.stdout.on( 'data', function( buffer ) {
    result += buffer.toString();
  });

  textutil.stderr.on( 'error', function( buffer ) {
    if ( !error ) {
      error = '';
    }
    error += buffer.toString();
  });

  textutil.on( 'close', function( /* code */ ) {
    if ( error ) {
      error = new Error( 'textutil read of file named [[ ' +
        path.basename( filePath ) + ' ]] failed: ' + error );
      cb( error, null );
      return;
    }
    cb( null, result.trim() );
  });
}
780251
def extractall(self, directory, auto_create_dir=False, patool_path=None):
        '''
        
        '''
        log.debug('extracting %s into %s (backend=%s)', self.filename, directory, self.backend)
        is_zipfile = zipfile.is_zipfile(self.filename)
        directory = _fullpath(directory)
        if not os.path.exists(self.filename):
            raise ValueError(
                'archive file does not exist:' + str(self.filename))
        if not os.path.exists(directory):
            if auto_create_dir:
                os.makedirs(directory)
            else:
                raise ValueError('directory does not exist:' + str(directory))

        if self.backend == 'auto':
            if is_zipfile:
                self.extractall_zipfile(directory)
            else:
                self.extractall_patool(directory, patool_path)

        if self.backend == 'zipfile':
            if not is_zipfile:
                raise ValueError('file is not zip file:' + str(self.filename))
            self.extractall_zipfile(directory)

        if self.backend == 'patool':
            self.extractall_patool(directory, patool_path)