Reformat input data files to a format the tasks can use
551739
def reformat_file(self, input_file, input_format, output_format):
        
        #Return none if input_file or input_format do not exist
        if input_file is None or input_format is None:
            return None
        #Find the needed input class and read the input stream
        try:
            input_cls = self.find_input(input_format)
            input_inst = input_cls()
        except TypeError:
            #Return none if input_cls is a Nonetype
            return None
        #If the input file cannot be found, return None
        try:
            input_inst.read_input(self.absolute_filepath(input_file))
        except IOError:
            return None

        formatter = find_needed_formatter(input_format, output_format)
        if formatter is None:
            raise Exception("Cannot find a formatter that can convert from {0} to {1}".format(self.input_format, output_format))
        formatter_inst = formatter()
        formatter_inst.read_input(input_inst.get_data(), input_format)
        data = formatter_inst.get_data(output_format)
        return data
1023486
def _prepare_sample(data, run_folder):
    
    want = set(["description", "files", "genome_build", "name", "analysis", "upload", "algorithm"])
    out = {}
    for k, v in data.items():
        if k in want:
            out[k] = _relative_paths(v, run_folder)
    if "algorithm" not in out:
        analysis, algorithm = _select_default_algorithm(out.get("analysis"))
        out["algorithm"] = algorithm
        out["analysis"] = analysis
    description = "%s-%s" % (out["name"], clean_name(out["description"]))
    out["name"] = [out["name"], description]
    out["description"] = description
    return out
723772
def prepare_input_data(config):
    

    if not dd.get_disambiguate(config):
        return dd.get_input_sequence_files(config)

    work_bam = dd.get_work_bam(config)
    logger.info("Converting disambiguated reads to fastq...")
    fq_files = convert_bam_to_fastq(
        work_bam, dd.get_work_dir(config), None, None, config
    )
    return fq_files
6327
def getinputfiles(self, outputfile, loadmetadata=True, client=None,requiremetadata=False):
        """"""
        if isinstance(outputfile, CLAMOutputFile):
            outputfilename = str(outputfile).replace(os.path.join(self.projectpath,'output/'),'')
        else:
            outputfilename = outputfile
        outputtemplate, inputfiles = self[outputfilename]
        for inputfilename, inputtemplate in inputfiles.items():
            yield CLAMInputFile(self.projectpath, inputfilename, loadmetadata,client,requiremetadata), inputtemplate
521590
function pluck(path, cb) {
  var tasks = [];
  fs.readFile(path, 'utf8', function (err, str) {
    if (err) return cb(err);

    /**
     * Skip binary-files
     */

    if (isBinary(str)) return;

    /**
     * Pointer within the file
     */

    var lines = str.split('\n'),
        linum = 0;

    while (lines.length) {
      var line = lines.shift(); linum++;

      /**
       * Check for task-comment
       */

      if (!pattern.test(line)) continue;

      /**
       * Old-style comments
       * @see http://www.riedquat.de/TR/trunk/TODO_Syntax/
       */

      var matches = line.match(isoPattern);
      if (matches != null) {
        matches = matches.toString().split(':');
        var labels = ['#'+matches[0].toLowerCase()],
            added = new Date(matches[1]),
            assignee = '@'+matches[2].replace(/[\\s]*/, '-'),
            linestart = linum,
            title = matches[3],
            body = '';
      }

      /**
       * New-style comments
       *
       * TODO Write a comment specification
       *    | Should be located at the address below
       *
       * TODO Share patterns between modules
       *    | Perhaps even make all of them global
       *
       * @see http://hkjels.github.com/ntask/syntax.html
       */

      else {
        var added = new Date(),
            assignee = (line.match(/@[\w\[\d\]-_]*/g)||['@none'])[0],
            title = line.replace(new RegExp('.*('+KEYWORDS+')'), '')
              .replace(assignee, '')
              .replace(/[^a-z0-9\!\]\)]*$/i, '').trim(),
            linestart = linum,
            body = (function body() {
              var next = lines.shift();
              if (/[\\s]*[\\\*\%\-(" )"]{0,3}[\|]/g.test(next)) {
                linum++;
                return next.split('|').slice(1).join('').trim()+'\n'+body();
              }
              else {
                lines.unshift(next);
                return '';
              }
            })(),
            labels = ((title+body).match(/#[\w]*[\[\]a-f0-9\.]*/gi) || [])
              .map(lowertrim).filter(minlength),
            keyword = line.match(new RegExp('.*('+KEYWORDS+')'))[1].toLowerCase();
            labels.unshift('#'+keyword);
      }
      var done = labels.indexOf('#done') != -1;

      /**
       * Parsed task-object ready for model-validation
       */

      var task = {
          added: added
        , assignee: assignee
        , body: body
        , done: done
        , file: path
        , labels: labels
        , line: linestart
        , title: title
      };
      tasks.push(task);
    }
    cb(false, tasks);
  });
}
600183
def organize_noalign(data):
    
    data = utils.to_single_data(data[0])
    work_dir = utils.safe_makedir(os.path.join(dd.get_work_dir(data), "align", dd.get_sample_name(data)))
    work_bam = os.path.join(work_dir, "%s-input.bam" % dd.get_sample_name(data))
    if data.get("files"):
        if data["files"][0].endswith(".cram"):
            work_bam = cram.to_bam(data["files"][0], work_bam, data)
        else:
            assert data["files"][0].endswith(".bam"), data["files"][0]
            utils.copy_plus(data["files"][0], work_bam)
        bam.index(work_bam, data["config"])
    else:
        work_bam = None
    data["align_bam"] = work_bam
    return data
572174
def prep_files(paths, extensions):
    """"""

    from batchpath import GeneratePaths

    filenames = GeneratePaths().files(paths, os.W_OK, extensions, 0, True)

    if filenames:
        return filenames
    else:
        LOGGER.error('No valid targets were specified')
        sys.exit(1)
1186046
def _to_cwl(val, input_files):
    
    if isinstance(val, six.string_types):
        if _file_and_exists(val, input_files):
            val = {"class": "File", "path": val}
            secondary = []
            for idx in [".bai", ".tbi", ".gbi", ".fai", ".crai", ".db"]:
                idx_file = val["path"] + idx
                if _file_and_exists(idx_file, input_files):
                    secondary.append({"class": "File", "path": idx_file})
            for idx in [".dict"]:
                idx_file = os.path.splitext(val["path"])[0] + idx
                if _file_and_exists(idx_file, input_files):
                    secondary.append({"class": "File", "path": idx_file})
            cur_dir, cur_file = os.path.split(val["path"])
            # Handle relative paths
            if not cur_dir:
                cur_dir = os.getcwd()
            if cur_file.endswith(cwlutils.DIR_TARGETS):
                if os.path.exists(cur_dir):
                    for fname in os.listdir(cur_dir):
                        if fname != cur_file and not os.path.isdir(os.path.join(cur_dir, fname))\
                                and fname != 'sbg.worker.log':
                            secondary.append({"class": "File", "path": os.path.join(cur_dir, fname)})
                else:
                    for f in input_files:
                        if f.startswith(cur_dir) and f != cur_file and not os.path.isdir(f):
                            secondary.append({"class": "File", "path": f})
            if secondary:
                val["secondaryFiles"] = _remove_duplicate_files(secondary)
    elif isinstance(val, (list, tuple)):
        val = [_to_cwl(x, input_files) for x in val]
    elif isinstance(val, dict):
        # File representation with secondary files
        if "base" in val and "secondary" in val:
            out = {"class": "File", "path": val["base"]}
            secondary = [{"class": "File", "path": x} for x in val["secondary"] if not os.path.isdir(x)]
            if secondary:
                out["secondaryFiles"] = _remove_duplicate_files(secondary)
            val = out
        else:
            val = json.dumps(val, sort_keys=True, separators=(',', ':'))
    return val
231593
def format_listall_output(format_file, format_item_dir, format, rule, option_prefix = None, template = None, skip_options = False):
    
    # Set the list of keys if printing from a file spec
    # _LINE_(whatever)_EOL_
    # _ITEM_(resource)_METI_
    # _KEY_(path_to_value)
    if format_file and os.path.isfile(format_file):
        if not template:
            with open(format_file, 'rt') as f:
                template = f.read()
        # Optional files
        if not skip_options:
            re_option = re.compile(r'(%_OPTION_\((.*?)\)_NOITPO_)')
            optional_files = re_option.findall(template)
            for optional_file in optional_files:
                if optional_file[1].startswith(option_prefix + '-'):
                    with open(os.path.join(format_item_dir, optional_file[1].strip()), 'rt') as f:
                        template = template.replace(optional_file[0].strip(), f.read())
        # Include files if needed
        re_file = re.compile(r'(_FILE_\((.*?)\)_ELIF_)')
        while True:
            requested_files = re_file.findall(template)
            available_files = os.listdir(format_item_dir) if format_item_dir else []
            for requested_file in requested_files:
                if requested_file[1].strip() in available_files:
                    with open(os.path.join(format_item_dir, requested_file[1].strip()), 'rt') as f:
                        template = template.replace(requested_file[0].strip(), f.read())
            # Find items and keys to be printed
            re_line = re.compile(r'(_ITEM_\((.*?)\)_METI_)')
            re_key = re.compile(r'_KEY_\(*(.*?)\)', re.DOTALL|re.MULTILINE) # Remove the multiline ?
            lines = re_line.findall(template)
            for (i, line) in enumerate(lines):
                lines[i] = line + (re_key.findall(line[1]),)
            requested_files = re_file.findall(template)
            if len(requested_files) == 0:
                break
    elif format and format[0] == 'csv':
        keys = rule.keys
        line = ', '.join('_KEY_(%s)' % k for k in keys)
        lines = [ (line, line, keys) ]
        template = line
    return (lines, template)
61553
def compile_data(tmp_dir, datasets, filename, datatypes_to_clean=None):
  """"""
  datatypes_to_clean = datatypes_to_clean or []
  filename = os.path.join(tmp_dir, filename)
  lang1_fname = filename + ".lang1"
  lang2_fname = filename + ".lang2"
  if tf.gfile.Exists(lang1_fname) and tf.gfile.Exists(lang2_fname):
    tf.logging.info("Skipping compile data, found files:\n%s\n%s", lang1_fname,
                    lang2_fname)
    return filename
  with tf.gfile.GFile(lang1_fname, mode="w") as lang1_resfile:
    with tf.gfile.GFile(lang2_fname, mode="w") as lang2_resfile:
      for dataset in datasets:
        url = dataset[0]
        compressed_filename = os.path.basename(url)
        compressed_filepath = os.path.join(tmp_dir, compressed_filename)
        if url.startswith("http"):
          generator_utils.maybe_download(tmp_dir, compressed_filename, url)

        if dataset[1][0] == "tmx":
          cleaning_requested = "tmx" in datatypes_to_clean
          tmx_filename = os.path.join(tmp_dir, dataset[1][1])
          if tmx_filename.endswith(".gz"):
            with gzip.open(tmx_filename, "rb") as tmx_file:
              _tmx_to_source_target(tmx_file, lang1_resfile, lang2_resfile,
                                    do_cleaning=cleaning_requested)
          else:
            with tf.gfile.Open(tmx_filename) as tmx_file:
              _tmx_to_source_target(tmx_file, lang1_resfile, lang2_resfile,
                                    do_cleaning=cleaning_requested)

        elif dataset[1][0] == "tsv":
          _, src_column, trg_column, glob_pattern = dataset[1]
          filenames = tf.gfile.Glob(os.path.join(tmp_dir, glob_pattern))
          if not filenames:
            # Capture *.tgz and *.tar.gz too.
            mode = "r:gz" if compressed_filepath.endswith("gz") else "r"
            with tarfile.open(compressed_filepath, mode) as corpus_tar:
              corpus_tar.extractall(tmp_dir)
            filenames = tf.gfile.Glob(os.path.join(tmp_dir, glob_pattern))
          for tsv_filename in filenames:
            if tsv_filename.endswith(".gz"):
              new_filename = tsv_filename.strip(".gz")
              generator_utils.gunzip_file(tsv_filename, new_filename)
              tsv_filename = new_filename
            with tf.gfile.Open(tsv_filename) as tsv_file:
              for line in tsv_file:
                if line and "\t" in line:
                  parts = line.split("\t")
                  source, target = parts[src_column], parts[trg_column]
                  source, target = source.strip(), target.strip()
                  clean_pairs = [(source, target)]
                  if "tsv" in datatypes_to_clean:
                    clean_pairs = cleaner_en_xx.clean_en_xx_pairs(clean_pairs)
                  for source, target in clean_pairs:
                    if source and target:
                      lang1_resfile.write(source)
                      lang1_resfile.write("\n")
                      lang2_resfile.write(target)
                      lang2_resfile.write("\n")

        else:
          lang1_filename, lang2_filename = dataset[1]
          lang1_filepath = os.path.join(tmp_dir, lang1_filename)
          lang2_filepath = os.path.join(tmp_dir, lang2_filename)
          is_sgm = (
              lang1_filename.endswith("sgm") and lang2_filename.endswith("sgm"))

          if not (tf.gfile.Exists(lang1_filepath) and
                  tf.gfile.Exists(lang2_filepath)):
            # For .tar.gz and .tgz files, we read compressed.
            mode = "r:gz" if compressed_filepath.endswith("gz") else "r"
            with tarfile.open(compressed_filepath, mode) as corpus_tar:
              corpus_tar.extractall(tmp_dir)
          if lang1_filepath.endswith(".gz"):
            new_filepath = lang1_filepath.strip(".gz")
            generator_utils.gunzip_file(lang1_filepath, new_filepath)
            lang1_filepath = new_filepath
          if lang2_filepath.endswith(".gz"):
            new_filepath = lang2_filepath.strip(".gz")
            generator_utils.gunzip_file(lang2_filepath, new_filepath)
            lang2_filepath = new_filepath

          for example in text_problems.text2text_txt_iterator(
              lang1_filepath, lang2_filepath):
            line1res = _preprocess_sgm(example["inputs"], is_sgm)
            line2res = _preprocess_sgm(example["targets"], is_sgm)
            clean_pairs = [(line1res, line2res)]
            if "txt" in datatypes_to_clean:
              clean_pairs = cleaner_en_xx.clean_en_xx_pairs(clean_pairs)
            for line1res, line2res in clean_pairs:
              if line1res and line2res:
                lang1_resfile.write(line1res)
                lang1_resfile.write("\n")
                lang2_resfile.write(line2res)
                lang2_resfile.write("\n")

  return filename
552177
def parsef!(file: "")
      types = @type ? [@type] : VALID_TASKS

      File.open(file, "r") do |f|
        line_number = 1
        while line = f.gets
          types.each do |type|
            result = line.match(/#{Regexp.escape(type)}[\s,:-]+(\S.*)\Z/) rescue nil

            unless result.nil?
              task = {
                file: file,
                line_number: line_number,
                task: result.to_a.last
              }

              @tasks[type.to_sym] << task
            end
          end

          line_number += 1
        end
      end
    end
706977
def _prepare_inputs(ma_fn, bam_file, out_dir):
    
    fixed_fa = os.path.join(out_dir, "file_reads.fa")
    count_name =dict()
    with file_transaction(fixed_fa) as out_tx:
        with open(out_tx, 'w') as out_handle:
            with open(ma_fn) as in_handle:
                h = next(in_handle)
                for line in in_handle:
                    cols = line.split("\t")
                    name_with_counts = "%s_x%s" % (cols[0], sum(map(int, cols[2:])))
                    count_name[cols[0]] = name_with_counts
                    out_handle.write(">%s\n%s\n" % (name_with_counts, cols[1]))
    fixed_bam = os.path.join(out_dir, "align.bam")
    bam_handle = pysam.AlignmentFile(bam_file, "rb")
    with pysam.AlignmentFile(fixed_bam, "wb", template=bam_handle) as out_handle:
        for read in bam_handle.fetch():
            read.query_name = count_name[read.query_name]
            out_handle.write(read)

    return fixed_fa, fixed_bam
455690
def _gen_tasks(self):
        """"""
        for filename in self.args:
            path = os.path.abspath(filename)
            yield {
                'name': path,
                # 'file_dep': [path],
                'actions': [(self.fun, (filename,))],
            }
1223125
def transform(self, data):
        
        out=[]
        keys = sorted(data.keys())
        for k in keys:
            out.append("%s=%s" % (k, data[k]))
        return "\n".join(out)
177646
function format(data) {
	var time = util.formatTime(new Date());
  var output_array = data.toString().replace(/^\s+|\s+$/g,"").split(/\s+/);
  var output_values = [];
  for (var i = 0, counter = 0; i < output_array.length; i++) {
    if(!isNaN(output_array[i])) {
      output_values[counter] = parseFloat(output_array[i]);
      counter++;
    }
  }
  if (output_values.length > 0) {
    output_hash = {
      date: time,
      disk: {
        kb_read: output_values[9],
        kb_wrtn: output_values[10],
        kb_read_per: output_values[7],
        kb_wrtn_per: output_values[8],
        tps: output_values[6]
      },
      cpu: {
        cpu_user: output_values[0],
        cpu_nice: output_values[1],
        cpu_system: output_values[2],
        cpu_iowait: output_values[3],
        cpu_steal: output_values[4],
        cpu_idle: output_values[5]
      }
    }
    return output_hash;
  }
}
560437
def parse_input_file(text, variables=None):
    
    text = find_includes(text)
    lines = text.splitlines()
    tasks, linenumbers = find_tasks(lines)
    preamble = [line for line in lines[:linenumbers[0]]]
    logging.debug("Preamble:\n{}".format("\n".join(preamble)))
    if variables is not None:
        preamble += "\n" + "\n".join(variables)
    environment = create_environment(preamble)
    code_sections = []
    for n in range(len(linenumbers) - 1):
        code_sections.append((linenumbers[n], linenumbers[n+1]))
    for n, task in zip(code_sections, tasks):
        task["code"] = lines[n[0]: n[1]]
        task["environment"] = environment
    clean_tasks = []
    for task in tasks:
        clean_tasks.append(Task(**task))
    return clean_tasks
1072808
def encode_task(task):
    
    # First, clean the task:
    task = task.copy()
    if 'tags' in task:
        task['tags'] = ','.join(task['tags'])
    for k in task:
        for unsafe, safe in six.iteritems(encode_replacements):
            if isinstance(task[k], six.string_types):
                task[k] = task[k].replace(unsafe, safe)

        if isinstance(task[k], datetime.datetime):
            task[k] = task[k].strftime("%Y%m%dT%M%H%SZ")

    # Then, format it as a string
    return "[%s]\n" % " ".join([
        "%s:\"%s\"" % (k, v)
        for k, v in sorted(task.items(), key=itemgetter(0))
    ])