Generates tuples of specfile and quant element for general formats
329056
def specfn_quant_generator(specfiles, quantfiles, tag, ignore_tags):
    """"""
    for specfn, qfn in zip(specfiles, quantfiles):
        for quant_el in basereader.generate_xmltags(qfn, tag, ignore_tags):
            yield os.path.basename(specfn), quant_el
1013670
def _generate_examples(self, filepath):
    """"""
    rows_per_pair_id = collections.defaultdict(list)

    with tf.io.gfile.GFile(filepath) as f:
      reader = csv.DictReader(f, delimiter='\t', quoting=csv.QUOTE_NONE)
      for row in reader:
        rows_per_pair_id[row['pairID']].append(row)

    for rows in six.itervalues(rows_per_pair_id):
      premise = {row['language']: row['sentence1'] for row in rows}
      hypothesis = {row['language']: row['sentence2'] for row in rows}
      yield {
          'premise': premise,
          'hypothesis': hypothesis,
          'label': rows[0]['gold_label'],
      }
725621
def _generate_examples(self, images_dir_path, labels_path, setid_path,
                         split_name):
    """"""
    with tf.io.gfile.GFile(labels_path, "rb") as f:
      labels = tfds.core.lazy_imports.scipy.io.loadmat(f)["labels"][0]
    with tf.io.gfile.GFile(setid_path, "rb") as f:
      examples = tfds.core.lazy_imports.scipy.io.loadmat(f)[split_name][0]

    for image_id in examples:
      file_name = "image_%05d.jpg" % image_id
      yield {
          "image": os.path.join(images_dir_path, file_name),
          "label": labels[image_id - 1] - 1,
          "file_name": file_name,
      }
1157123
def gen_properties(output, ascii_props=False, append=False):
    """"""

    files = {
        'gc': os.path.join(output, 'generalcategory.py'),
        'blk': os.path.join(output, 'block.py'),
        'sc': os.path.join(output, 'script.py'),
        'bc': os.path.join(output, 'bidiclass.py'),
        'binary': os.path.join(output, 'binary.py'),
        'posix': os.path.join(output, 'posix.py'),
        'age': os.path.join(output, 'age.py'),
        'ea': os.path.join(output, 'eastasianwidth.py'),
        'gcb': os.path.join(output, 'graphemeclusterbreak.py'),
        'lb': os.path.join(output, 'linebreak.py'),
        'sb': os.path.join(output, 'sentencebreak.py'),
        'wb': os.path.join(output, 'wordbreak.py'),
        'hst': os.path.join(output, 'hangulsyllabletype.py'),
        'dt': os.path.join(output, 'decompositiontype.py'),
        'jt': os.path.join(output, 'joiningtype.py'),
        'jg': os.path.join(output, 'joininggroup.py'),
        'nt': os.path.join(output, 'numerictype.py'),
        'nv': os.path.join(output, 'numericvalue.py'),
        'ccc': os.path.join(output, 'canonicalcombiningclass.py'),
        'qc': os.path.join(output, 'quickcheck.py'),
        'alias': os.path.join(output, 'alias.py')
    }

    files['scx'] = os.path.join(output, 'scriptextensions.py')
    files['insc'] = os.path.join(output, 'indicsyllabiccategory.py')
    if PY34:
        files['bpt'] = os.path.join(output, 'bidipairedbrackettype.py')
    if PY35:
        files['inpc'] = os.path.join(output, 'indicpositionalcategory.py')
    else:
        files['inmc'] = os.path.join(output, 'indicmatracategory.py')
    if PY37:
        files['vo'] = os.path.join(output, 'verticalorientation.py')

    prefix = "ascii" if ascii_props else 'unicode'

    # `L&` or `Lc` won't be found in the table,
    # so initialize 'c' at the start. `&` will have to be converted to 'c'
    # before sending it through.
    categories = [
        'generalcategory', 'script', 'block',
        'bidiclass', 'eastasianwidth', 'linebreak',
        'hangulsyllabletype', 'wordbreak', 'sentencebreak',
        'graphemeclusterbreak', 'decompositiontype', 'joiningtype',
        'joininggroup', 'numerictype', 'numericvalue',
        'canonicalcombiningclass', 'age'
    ]
    categories.append('scriptextensions')
    categories.append('indicsyllabiccategory')
    if PY34:
        categories.append('bidipairedbrackettype')
    if PY35:
        categories.append('indicpositionalcategory')
    else:
        categories.append('indicmatracategory')
    if PY37:
        categories.append('verticalorientation')
    if ascii_props:
        print('=========Ascii Tables=========')
    else:
        print('========Unicode Tables========')
    print('Building: General Category')
    max_range = ASCII_RANGE if ascii_props else UNICODE_RANGE
    all_chars = ALL_ASCII if ascii_props else ALL_CHARS
    table = {'l': {'c': []}}
    itable = {'l': {}}
    with codecs.open(os.path.join(HOME, 'unicodedata', UNIVERSION, 'UnicodeData.txt'), 'r', 'utf-8') as uf:
        for line in uf:
            data = line.strip().split(';')
            if data:
                i = int(data[0], 16)
                if i > max_range[1]:
                    continue
                p = data[2].lower()
                if p[0] not in table:
                    table[p[0]] = {}
                    itable[p[0]] = {}
                if p[1] not in table[p[0]]:
                    table[p[0]][p[1]] = []
                table[p[0]][p[1]].append(i)
                # Add LC which is a combo of Ll, Lu, and Lt
                if p[0] == 'l' and p[1] in ('l', 'u', 't'):
                    table['l']['c'].append(i)

    # Create inverse of each category
    for k1, v1 in table.items():
        inverse_category = set()
        for k2, v2 in v1.items():
            s = set(v2)
            inverse_category |= s
        itable[k1]['^'] = list(all_chars - inverse_category)

    # Generate Unicode blocks
    print('Building: Blocks')
    gen_blocks(files['blk'], ascii_props, append, prefix)

    # Generate Unicode scripts
    print('Building: Scripts & Script Extensions')
    gen_scripts(
        'Scripts.txt', 'ScriptExtensions.txt', 'scripts', 'script_extensions', files['sc'], files['scx'],
        notexplicit='zzzz', ascii_props=ascii_props, append=append, prefix=prefix
    )

    # Generate Unicode bidi classes
    print('Building: Bidi Classes')
    gen_bidi(files['bc'], ascii_props, append, prefix)

    if PY34:
        print('Building: Bidi Paired Bracket Type')
        gen_enum(
            'BidiBrackets.txt', 'bidi_paired_bracket_type', files['bpt'], notexplicit='n',
            field=2, ascii_props=ascii_props, append=append, prefix=prefix
        )

    # Generate Unicode binary
    print('Building: Binary')
    binary = gen_binary(table, files['binary'], ascii_props, append, prefix)

    # Generate posix table and write out to file.
    print('Building: Posix')
    gen_posix(files['posix'], is_bytes=ascii_props, append=append, prefix=prefix)

    print('Building: Age')
    gen_age(files['age'], ascii_props, append, prefix)

    print('Building: East Asian Width')
    gen_enum(
        'EastAsianWidth.txt', 'east_asian_width', files['ea'],
        notexplicit='n', ascii_props=ascii_props, append=append, prefix=prefix
    )

    print('Building: Grapheme Cluster Break')
    gen_enum(
        'GraphemeBreakProperty.txt', 'grapheme_cluster_break', files['gcb'], notexplicit='other',
        ascii_props=ascii_props, append=append, prefix=prefix
    )

    print('Building: Line Break')
    gen_enum(
        'LineBreak.txt', 'line_break', files['lb'], notexplicit='unknown',
        ascii_props=ascii_props, append=append, prefix=prefix
    )

    print('Building: Sentence Break')
    gen_enum(
        'SentenceBreakProperty.txt', 'sentence_break', files['sb'], notexplicit='other',
        ascii_props=ascii_props, append=append, prefix=prefix
    )

    print('Building: Word Break')
    gen_enum(
        'WordBreakProperty.txt', 'word_break', files['wb'], notexplicit='other',
        ascii_props=ascii_props, append=append, prefix=prefix
    )

    if PY35:
        print('Building: Indic Positional Category')
        gen_enum(
            'IndicPositionalCategory.txt', 'indic_positional_category', files['inpc'], notexplicit='na',
            ascii_props=ascii_props, append=append, prefix=prefix
        )
    else:
        print('Building: Indic Matra Category')
        gen_enum(
            'IndicMatraCategory.txt', 'indic_matra_category', files['inmc'], notexplicit='na',
            ascii_props=ascii_props, append=append, prefix=prefix
        )
    print('Building: Indic Syllabic Category')
    gen_enum(
        'IndicSyllabicCategory.txt', 'indic_syllabic_category', files['insc'], notexplicit='other',
        ascii_props=ascii_props, append=append, prefix=prefix
    )

    print('Building: Hangul Syllable Type')
    gen_enum(
        'HangulSyllableType.txt', 'hangul_syllable_type', files['hst'], notexplicit='na',
        ascii_props=ascii_props, append=append, prefix=prefix
    )

    print('Building: Decomposition Type')
    gen_enum(
        'DerivedDecompositionType.txt', 'decomposition_type', files['dt'], notexplicit='none',
        ascii_props=ascii_props, append=append, prefix=prefix
    )

    print('Building: Joining Type')
    gen_enum(
        'DerivedJoiningType.txt', 'joining_type', files['jt'], notexplicit='u',
        ascii_props=ascii_props, append=append, prefix=prefix
    )

    print('Building: Joining Group')
    gen_enum(
        'DerivedJoiningGroup.txt', 'joining_group', files['jg'], notexplicit='nonjoining',
        ascii_props=ascii_props, append=append, prefix=prefix
    )

    print('Building: Numeric Type')
    gen_enum(
        'DerivedNumericType.txt', 'numeric_type', files['nt'], notexplicit='none',
        ascii_props=ascii_props, append=append, prefix=prefix
    )

    print('Building: Numeric Value')
    gen_enum(
        'DerivedNumericValues.txt', 'numeric_values', files['nv'], field=3, notexplicit='nan',
        ascii_props=ascii_props, append=append, prefix=prefix
    )

    print('Building: Canonical Combining Class')
    gen_ccc(files['ccc'], ascii_props, append, prefix)

    print('Building: NF* Quick Check')
    categories.extend(gen_nf_quick_check(files['qc'], ascii_props, append, prefix))

    if PY37:
        print('Building: Vertical Orientation')
        gen_enum(
            'VerticalOrientation.txt', 'vertical_orientation', files['vo'], notexplicit='R',
            ascii_props=ascii_props, append=append, prefix=prefix
        )

    if not ascii_props:
        print('Building: Aliases')
        gen_alias(categories, binary, files['alias'], ascii_props, append, prefix)

    # Convert char values to string ranges.
    char2range(table, is_bytes=ascii_props)
    char2range(itable, is_bytes=ascii_props, invert=False)
    for k1, v1 in itable.items():
        table[k1]['^'] = v1['^']

    with codecs.open(files['gc'], 'a' if append else 'w', 'utf-8') as f:
        if not append:
            f.write(HEADER)
        # Write out the Unicode properties
        f.write('%s_properties = {\n' % prefix)
        count = len(table) - 1
        i = 0
        for k1, v1 in sorted(table.items()):
            f.write('    "%s": {\n' % k1)
            count2 = len(v1) - 1
            j = 0
            for k2, v2 in sorted(v1.items()):
                f.write('        "%s": "%s"' % (k2, v2))
                if j == count2:
                    f.write('\n    }')
                else:
                    f.write(',\n')
                j += 1
            if i == count:
                f.write('\n}\n')
            else:
                f.write(',\n')
            i += 1

    if not append:
        with codecs.open(os.path.join(output, '__init__.py'), 'w') as f:
            f.write(HEADER)
            for x in sorted(files):
                f.write('from .%s import *  # noqa\n' % os.path.basename(files[x])[:-3])
949749
def _compile_mothur_script(self):
        """"""
        def format_opts(*opts):
            """Formats a series of options for a Mothur script"""
            return ', '.join(filter(None, map(str, opts)))
        vars = {
            'in': self._input_filename,
            'unique': self._derive_unique_path(),
            'dist': self._derive_dist_path(),
            'names': self._derive_names_path(),
            'cluster_opts': format_opts(
                self.Parameters['method'],
                self.Parameters['cutoff'],
                self.Parameters['precision'],
            ),
        }
        script = (
            '#'
            'unique.seqs(fasta=%(in)s); '
            'dist.seqs(fasta=%(unique)s); '
            'read.dist(column=%(dist)s, name=%(names)s); '
            'cluster(%(cluster_opts)s)' % vars
        )
        return script
737426
def generate_tsv_pep_protein_quants(fns):
    """"""
    for fn in fns:
        header = get_tsv_header(fn)
        for pquant in generate_split_tsv_lines(fn, header):
            yield os.path.basename(fn), header, pquant
938769
def _generate_data_for_format(fmt):
    """"""
    # finally try some data, create some random data for the fmt.
    data = {}
    # keep track of how many "free_size" (wildcard) parameters we have
    # if we get two in a row then we know the pattern is invalid, meaning
    # we'll never be able to match the second wildcard field
    free_size_start = False
    for literal_text, field_name, format_spec, conversion in formatter.parse(fmt):
        if literal_text:
            free_size_start = False

        if not field_name:
            free_size_start = False
            continue

        # encapsulating free size keys,
        # e.g. {:s}{:s} or {:s}{:4s}{:d}
        if not format_spec or format_spec == "s" or format_spec == "d":
            if free_size_start:
                return None
            else:
                free_size_start = True

        # make some data for this key and format
        if format_spec and '%' in format_spec:
            # some datetime
            t = dt.datetime.now()
            # run once through format to limit precision
            t = parse(
                "{t:" + format_spec + "}", compose("{t:" + format_spec + "}", {'t': t}))['t']
            data[field_name] = t
        elif format_spec and 'd' in format_spec:
            # random number (with n sign. figures)
            if not format_spec.isalpha():
                n = _get_number_from_fmt(format_spec)
            else:
                # clearly bad
                return None
            data[field_name] = random.randint(0, 99999999999999999) % (10 ** n)
        else:
            # string type
            if format_spec is None:
                n = 4
            elif format_spec.isalnum():
                n = _get_number_from_fmt(format_spec)
            else:
                n = 4
            randstri = ''
            for x in range(n):
                randstri += random.choice(string.ascii_letters)
            data[field_name] = randstri
    return data
1171493
def toConsole(self):
        '''
        
        '''
        fmt_spec = {
            "class":     utils.AnsiColors.BOLD_MAGENTA,
            "struct":    utils.AnsiColors.BOLD_CYAN,
            "define":    utils.AnsiColors.BOLD_YELLOW,
            "enum":      utils.AnsiColors.BOLD_MAGENTA,
            "enumvalue": utils.AnsiColors.BOLD_RED,     # red means unused in framework
            "function":  utils.AnsiColors.BOLD_CYAN,
            "file":      utils.AnsiColors.BOLD_YELLOW,
            "dir":       utils.AnsiColors.BOLD_MAGENTA,
            "group":     utils.AnsiColors.BOLD_RED,     # red means unused in framework
            "namespace": utils.AnsiColors.BOLD_CYAN,
            "typedef":   utils.AnsiColors.BOLD_YELLOW,
            "union":     utils.AnsiColors.BOLD_MAGENTA,
            "variable":  utils.AnsiColors.BOLD_CYAN
        }

        self.consoleFormat(
            "{0} and {1}".format(
                utils._use_color("Classes", fmt_spec["class"],  sys.stderr),
                utils._use_color("Structs", fmt_spec["struct"], sys.stderr),
            ),
            self.class_like,
            fmt_spec
        )
        self.consoleFormat(
            utils._use_color("Defines", fmt_spec["define"], sys.stderr),
            self.defines,
            fmt_spec
        )
        self.consoleFormat(
            utils._use_color("Enums", fmt_spec["enum"], sys.stderr),
            self.enums,
            fmt_spec
        )
        self.consoleFormat(
            utils._use_color("Enum Values (unused)", fmt_spec["enumvalue"], sys.stderr),
            self.enum_values,
            fmt_spec
        )
        self.consoleFormat(
            utils._use_color("Functions", fmt_spec["function"], sys.stderr),
            self.functions,
            fmt_spec
        )
        self.consoleFormat(
            utils._use_color("Files", fmt_spec["file"], sys.stderr),
            self.files,
            fmt_spec
        )
        self.consoleFormat(
            utils._use_color("Directories", fmt_spec["dir"], sys.stderr),
            self.dirs,
            fmt_spec
        )
        self.consoleFormat(
            utils._use_color("Groups (unused)", fmt_spec["group"], sys.stderr),
            self.groups,
            fmt_spec
        )
        self.consoleFormat(
            utils._use_color("Namespaces", fmt_spec["namespace"], sys.stderr),
            self.namespaces,
            fmt_spec
        )
        self.consoleFormat(
            utils._use_color("Typedefs", fmt_spec["typedef"], sys.stderr),
            self.typedefs,
            fmt_spec
        )
        self.consoleFormat(
            utils._use_color("Unions", fmt_spec["union"], sys.stderr),
            self.unions,
            fmt_spec
        )
        self.consoleFormat(
            utils._use_color("Variables", fmt_spec["variable"], sys.stderr),
            self.variables,
            fmt_spec
        )
522113
def generate_cxx(module_name, code, specs=None, optimizations=None,
                 module_dir=None):
    '''

    '''

    pm, ir, renamings, docstrings = front_middle_end(module_name, code,
                                                     optimizations, module_dir)

    # back-end
    content = pm.dump(Cxx, ir)

    # instantiate the meta program
    if specs is None:

        class Generable(object):
            def __init__(self, content):
                self.content = content

            def __str__(self):
                return str(self.content)

            generate = __str__

        mod = Generable(content)

        def error_checker():
            tog.typecheck(ir)

    else:

        # uniform typing
        if isinstance(specs, dict):
            specs = Spec(specs, {})

        def error_checker():
            types = tog.typecheck(ir)
            check_specs(specs, renamings, types)

        specs.to_docstrings(docstrings)
        check_exports(ir, specs, renamings)

        if isinstance(code, bytes):
            code_bytes = code
        else:
            code_bytes = code.encode('ascii', 'ignore')
        metainfo = {'hash': hashlib.sha256(code_bytes).hexdigest(),
                    'version': __version__,
                    'date': datetime.now()}

        mod = PythonModule(module_name, docstrings, metainfo)
        mod.add_to_includes(
            Include("pythonic/core.hpp"),
            Include("pythonic/python/core.hpp"),
            # FIXME: only include these when needed
            Include("pythonic/types/bool.hpp"),
            Include("pythonic/types/int.hpp"),
            Line("#ifdef _OPENMP\n#include <omp.h>\n#endif")
        )
        mod.add_to_includes(*[Include(inc) for inc in
                              _extract_specs_dependencies(specs)])
        mod.add_to_includes(*content.body)
        mod.add_to_includes(
            Include("pythonic/python/exception_handler.hpp"),
        )

        def warded(module_name, internal_name):
            return pythran_ward + '{0}::{1}'.format(module_name, internal_name)

        for function_name, signatures in specs.functions.items():
            internal_func_name = renamings.get(function_name,
                                               function_name)
            # global variables are functions with no signatures :-)
            if not signatures:
                mod.add_global_var(function_name,
                                   "{}()()".format(warded(module_name,
                                                          internal_func_name)))

            for sigid, signature in enumerate(signatures):
                numbered_function_name = "{0}{1}".format(internal_func_name,
                                                         sigid)
                arguments_types = [pytype_to_ctype(t) for t in signature]
                arguments_names = HasArgument(internal_func_name).visit(ir)
                arguments = [n for n, _ in
                             zip(arguments_names, arguments_types)]
                name_fmt = pythran_ward + "{0}::{1}::type{2}"
                args_list = ", ".join(arguments_types)
                specialized_fname = name_fmt.format(module_name,
                                                    internal_func_name,
                                                    "<{0}>".format(args_list)
                                                    if arguments_names else "")
                result_type = "typename %s::result_type" % specialized_fname
                mod.add_pyfunction(
                    FunctionBody(
                        FunctionDeclaration(
                            Value(
                                result_type,
                                numbered_function_name),
                            [Value(t + '&&', a)
                             for t, a in zip(arguments_types, arguments)]),
                        Block([Statement("""
                            PyThreadState *_save = PyEval_SaveThread();
                            try {{
                                auto res = {0}()({1});
                                PyEval_RestoreThread(_save);
                                return res;
                            }}
                            catch(...) {{
                                PyEval_RestoreThread(_save);
                                throw;
                            }}
                            """.format(warded(module_name,
                                              internal_func_name),
                                       ', '.join(arguments)))])
                    ),
                    function_name,
                    arguments_types,
                    signature
                )

        for function_name, signature in specs.capsules.items():
            internal_func_name = renamings.get(function_name,
                                               function_name)

            arguments_types = [pytype_to_ctype(t) for t in signature]
            arguments_names = HasArgument(internal_func_name).visit(ir)
            arguments = [n for n, _ in
                         zip(arguments_names, arguments_types)]
            name_fmt = pythran_ward + "{0}::{1}::type{2}"
            args_list = ", ".join(arguments_types)
            specialized_fname = name_fmt.format(module_name,
                                                internal_func_name,
                                                "<{0}>".format(args_list)
                                                if arguments_names else "")
            result_type = "typename %s::result_type" % specialized_fname
            docstring = spec_to_string(function_name, signature)
            mod.add_capsule(
                FunctionBody(
                    FunctionDeclaration(
                        Value(result_type, function_name),
                        [Value(t, a)
                         for t, a in zip(arguments_types, arguments)]),
                    Block([ReturnStatement("{0}()({1})".format(
                        warded(module_name, internal_func_name),
                        ', '.join(arguments)))])
                ),
                function_name,
                docstring
            )

    return mod, error_checker
1109040
def canonicalize_spec(spec, parent_context):
  """"""

  test_specs = {k:v for (k,v) in spec.items() if k.startswith("Test")}
  local_context = {k:v for (k,v) in spec.items() if not k.startswith("Test")}

  context = reduce_contexts(parent_context, local_context)

  if test_specs:
    return {k: canonicalize_spec(v, context) for (k,v) in test_specs.items()}
  else:
    program_chunks = sum([resolve_module(m,context['Definitions']) for m in context['Modules']],[]) + [context['Program']]
    test_spec = {
      'Arguments': context['Arguments'],
      'Program': "\n".join(program_chunks),
      'Expect': context['Expect'],
    }
    return test_spec
267381
def _gen_txt_data(self, f):
		
		reader = iter(f)

		for line_num, line in enumerate(reader):
			if line_num == 0 and self.has_header:
				continue

			datum = line.rstrip('\r\n')

			yield datum, line_num+1
463340
function generateDocs(specjs) {
  var docs = GENERATE_MODULE(specjs.MODULE, '');
  GENERATE_TYPE = (function(enums) { 
    return function(type) {
      return (_.contains(enums, type) ? ('Enum ' + type) : type);  
    }
  })(_.keys(specjs.ENUMS_BY_GROUP));
  docs = _.reduce(specjs.METHODS, function(memo, methodSpec, methodName) {
    return memo += GENERATE_METHOD(methodName, methodSpec);
  }, docs);
  docs = _.reduce(specjs.ENUMS, function(memo, enumSpec, enumName) {
    return memo += GENERATE_ENUM(enumName, enumSpec);
  }, docs);
  if (_.isEmpty(specjs.CLASSGROUPS)) {
      docs += GENERATE_CLASSES(specjs.CLASSES);
  } else {
    docs += GENERATE_MODULE('common', '');
    var grouped = _.flatten(_.pluck(_.values(specjs.CLASSGROUPS), 'classes'));
    var ungrouped = _.difference(_.keys(specjs.CLASSES), grouped);
    docs += GENERATE_CLASSES(_.pick(specjs.CLASSES, ungrouped), 'common'); 
    _.each(specjs.CLASSGROUPS, function(groupSpec, groupName) {
      docs += GENERATE_CLASSES(_.pick(specjs.CLASSES, groupSpec.classes), groupName);
    });
    // TODO: figure out why yuidoc won't associate the class with the right module if module definitions are interspersed
    _.each(specjs.CLASSGROUPS, function(groupSpec, groupName) {
        docs += GENERATE_MODULE(groupName, groupSpec.description);
    });
  }
  return docs;
}
105746
def _generate_docstring_return_section(self, return_vals, header,
                                           return_element_name,
                                           return_element_type,
                                           placeholder, indent):
        """"""
        # If all return values are None, return none
        non_none_vals = [return_val for return_val in return_vals
                         if return_val and return_val != 'None']
        if not non_none_vals:
            return header + indent + 'None.'

        # Get only values with matching brackets that can be cleaned up
        non_none_vals = [return_val.strip(' ()\t\n').rstrip(',')
                         for return_val in non_none_vals]
        non_none_vals = [re.sub('([\"\'])(?:(?=(\\\\?))\\2.)*?\\1',
                                '"string"', return_val)
                         for return_val in non_none_vals]
        unambiguous_vals = []
        for return_val in non_none_vals:
            try:
                cleaned_val = self.find_top_level_bracket_locations(return_val)
            except IndexError:
                continue
            unambiguous_vals.append(cleaned_val)
        if not unambiguous_vals:
            return header + placeholder

        # If remaining are a mix of tuples and not, return single placeholder
        single_vals, tuple_vals = [], []
        for return_val in unambiguous_vals:
            (tuple_vals.append(return_val) if ',' in return_val
             else single_vals.append(return_val))
        if single_vals and tuple_vals:
            return header + placeholder

        # If return values are tuples of different length, return a placeholder
        if tuple_vals:
            num_elements = [return_val.count(',') + 1
                            for return_val in tuple_vals]
            if num_elements.count(num_elements[0]) != len(num_elements):
                return header + placeholder
            num_elements = num_elements[0]
        else:
            num_elements = 1

        # If all have the same len but some ambiguous return that placeholders
        if len(unambiguous_vals) != len(non_none_vals):
            return header + '\n'.join(
                [placeholder for __ in range(num_elements)])

        # Handle tuple (or single) values position by position
        return_vals_grouped = zip(*[
            [return_element.strip() for return_element in
             return_val.split(',')]
            for return_val in unambiguous_vals])
        return_elements_out = []
        for return_vals_group in return_vals_grouped:
            return_elements_out.append(
                self.parse_return_elements(return_vals_group,
                                           return_element_name,
                                           return_element_type,
                                           placeholder))

        return header + '\n'.join(return_elements_out)
7835
def dataset_generator(filepath,
                      dataset,
                      chunk_size=1,
                      start_idx=None,
                      end_idx=None):
  """"""
  encoder = dna_encoder.DNAEncoder(chunk_size=chunk_size)
  with h5py.File(filepath, "r") as h5_file:
    # Get input keys from h5_file
    src_keys = [s % dataset for s in ["%s_in", "%s_na", "%s_out"]]
    src_values = [h5_file[k] for k in src_keys]
    inp_data, mask_data, out_data = src_values
    assert len(set([v.len() for v in src_values])) == 1

    if start_idx is None:
      start_idx = 0
    if end_idx is None:
      end_idx = inp_data.len()

    for i in range(start_idx, end_idx):
      if i % 100 == 0:
        print("Generating example %d for %s" % (i, dataset))
      inputs, mask, outputs = inp_data[i], mask_data[i], out_data[i]
      ex_dict = to_example_dict(encoder, inputs, mask, outputs)
      # Original data has one output for every 128 input bases. Ensure that the
      # ratio has been maintained given the chunk size and removing EOS.
      assert (len(ex_dict["inputs"]) - 1) == ((
          128 // chunk_size) * ex_dict["targets_shape"][0])
      yield ex_dict
550031
def gen_template_files(path):
    "  "

    path = path.rstrip(op.sep)
    for root, _, files in walk(path):
        for f in filter(lambda x: not x in (TPLNAME, CFGNAME), files):
            yield op.relpath(op.join(root, f), path)
1239461
def get_quantcols(pattern, oldheader, coltype):
    """"""
    if pattern is None:
       return False
    if coltype == 'precur':
        return reader.get_cols_in_file(pattern, oldheader, single_col=True)
1365521
def _generate_examples(self, filepath):
    
    with tf.io.gfile.GFile(filepath, "rb") as f:
      data = tfds.core.lazy_imports.scipy.io.loadmat(f)

    # Maybe should shuffle ?

    assert np.max(data["y"]) <= 10  # Sanity check
    assert np.min(data["y"]) > 0

    for image, label in zip(np.rollaxis(data["X"], -1), data["y"]):
      yield {
          "image": image,
          "label": label % 10,  # digit 0 is saved as 0 (instead of 10)
      }