Generate source and target data from a single file.
323180
def tabbed_parsing_character_generator(tmp_dir, train):
  """"""
  character_vocab = text_encoder.ByteTextEncoder()
  filename = "parsing_{0}.pairs".format("train" if train else "dev")
  pair_filepath = os.path.join(tmp_dir, filename)
  return text_problems.text2text_generate_encoded(
      text_problems.text2text_txt_tab_iterator(pair_filepath), character_vocab)
550031
def gen_template_files(path):
    "  "

    path = path.rstrip(op.sep)
    for root, _, files in walk(path):
        for f in filter(lambda x: not x in (TPLNAME, CFGNAME), files):
            yield op.relpath(op.join(root, f), path)
1128328
def python_source(ast, file=sys.stdout):
    '''
    
    '''
    gen = SourceGen()
    gen.visit(ast)
    gen.dump(file)
647105
def _generate_examples(self, data_file):
    """"""
    target_language = self.builder_config.target_language

    with tf.io.gfile.GFile(data_file) as f:
      for i, line in enumerate(f):
        line_parts = line.strip().split("\t")
        if len(line_parts) != 2:
          raise ValueError(("Wrong data format in line {}. The line '{}' does "
                            "not have exactly one delimiter.").format(i, line))
        source, target = line_parts[0].strip(), line_parts[1].strip()

        yield {"en": source, target_language: target}
326720
def populate(content, **options)
      generator.create_file(RubyEdit::SOURCE_FILE_LOCATION,
                            content,
                            force: true,
                            verbose: false,
                            **options)
    end
723815
def GenerateFileData(self):
    """"""
    # Handle chunked encoding:
    # https://www.w3.org/Protocols/rfc2616/rfc2616-sec3.html#sec3.6.1
    while 1:
      line = self.rfile.readline()
      # We do not support chunked extensions, just ignore them.
      chunk_size = int(line.split(";")[0], 16)
      if chunk_size == 0:
        break

      for chunk in self._GenerateChunk(chunk_size):
        yield chunk

      # Chunk is followed by \r\n.
      lf = self.rfile.read(2)
      if lf != "\r\n":
        raise IOError("Unable to parse chunk.")

    # Skip entity headers.
    for header in self.rfile.readline():
      if not header:
        break
1061625
def generate(inputfilename, outputfilename='', dump=0, **flags):
    """"""

    if not outputfilename:
        if inputfilename[-2:] == '.g':
            outputfilename = inputfilename[:-2] + '.py'
        else:
            raise Exception("Missing output filename")

    print 'Input Grammar:', inputfilename
    print 'Output File:', outputfilename

    DIVIDER = '\n%%\n'  # This pattern separates the pre/post parsers
    preparser, postparser = None, None  # Code before and after the parser desc

    # Read the entire file
    s = open(inputfilename, 'r').read()

    # See if there's a separation between the pre-parser and parser
    f = find(s, DIVIDER)
    if f >= 0:
        preparser, s = s[:f] + '\n\n', s[f + len(DIVIDER):]

    # See if there's a separation between the parser and post-parser
    f = find(s, DIVIDER)
    if f >= 0:
        s, postparser = s[:f], '\n\n' + s[f + len(DIVIDER):]

    # Create the parser and scanner
    p = ParserDescription(ParserDescriptionScanner(s))
    if not p:
        return

    # Now parse the file
    t = wrap_error_reporter(p, 'Parser')
    if not t:
        return  # Error
    if preparser is not None:
        t.preparser = preparser
    if postparser is not None:
        t.postparser = postparser

    # Check the options
    for f in t.options.keys():
        for opt, _, _ in yapps_options:
            if f == opt:
                break
        else:
            print 'Warning: unrecognized option', f
    # Add command line options to the set
    for f in flags.keys():
        t.options[f] = flags[f]

    # Generate the output
    if dump:
        t.dump_information()
    else:
        t.output = open(outputfilename, 'w')
        t.generate_output()
272836
def batch_gen(data, batch_size):
    '''
    
    '''
    data = data or []
    for i in range(0, len(data), batch_size):
        yield data[i:i + batch_size]
244770
def build_file_lines(self):
    """"""
    build_file_lines = self._build_file_source_lines[:]
    target_begin, target_end = self._target_interval
    build_file_lines[target_begin:target_end] = self.target_lines()
    return build_file_lines
507337
def generate_file(fname, ns_func, dest_dir="."):
    """"""
    with open(pjoin(root, 'buildutils', 'templates', '%s' % fname), 'r') as f:
        tpl = f.read()
    out = tpl.format(**ns_func())
    dest = pjoin(dest_dir, fname)
    info("generating %s from template" % dest)
    with open(dest, 'w') as f:
        f.write(out)
902023
def generate_all_deps(self, target: Target):
        """"""
        yield from (self.targets[dep_name]
                    for dep_name in self.generate_dep_names(target))
884016
def generate(env):
   
   doxyfile_scanner = env.Scanner(
      DoxySourceScan,
      "DoxySourceScan",
      scan_check = DoxySourceScanCheck,
   )

   import SCons.Builder
   doxyfile_builder = SCons.Builder.Builder(
      action = "cd ${SOURCE.dir}  &&  ${DOXYGEN} ${SOURCE.file}",
      emitter = DoxyEmitter,
      target_factory = env.fs.Entry,
      single_source = True,
      source_scanner =  doxyfile_scanner,
   )

   env.Append(BUILDERS = {
      'Doxygen': doxyfile_builder,
   })

   env.AppendUnique(
      DOXYGEN = 'doxygen',
   )
66160
def _generator_file(self):
        """'"""
        for path in self.paths:
            if os.path.isfile(path):
                if isvalid(path, self.access, self.extensions,
                           minsize=self.minsize):
                    yield os.path.abspath(path)
            elif os.path.isdir(path):
                for root, _, fnames in self._walker(path):
                    yield from self._generator_rebase(fnames, root)
43714
def generate_contents(startpath, outfilename=DEFAULT_BUILDFILE):
    
    def _ignored_name(name):
        return (
            name.startswith('.') or
            name == PACKAGE_DIR_NAME or
            name.endswith('~') or
            name == outfilename
        )

    def _generate_contents(dir_path):
        safename_duplicates = defaultdict(list)
        for name in os.listdir(dir_path):
            if _ignored_name(name):
                continue

            path = os.path.join(dir_path, name)

            if os.path.isdir(path):
                nodename = name
                ext = None
            elif os.path.isfile(path):
                nodename, ext = splitext_no_dot(name)
            else:
                continue

            safename = to_identifier(nodename)
            safename_duplicates[safename].append((name, nodename, ext))

        safename_to_name = {}
        for safename, duplicates in iteritems(safename_duplicates):
            for name, nodename, ext in duplicates:
                if len(duplicates) > 1 and ext:
                    new_safename = to_identifier(name)  # Name with ext
                else:
                    new_safename = safename
                existing_name = safename_to_name.get(new_safename)
                if existing_name is not None:
                    message = "Duplicate node names in directory %r."
                    message += " %r was renamed to %r, which overlaps with %r"
                    raise BuildException(
                        message % (dir_path, name, new_safename, existing_name)
                    )
                safename_to_name[new_safename] = name

        contents = {}
        for safename, name in iteritems(safename_to_name):
            path = os.path.join(dir_path, name)

            if os.path.isdir(path):
                data = _generate_contents(path)
            else:
                rel_path = os.path.relpath(path, startpath)
                data = dict(file=rel_path)

            contents[safename] = data

        return contents

    return dict(
        contents=_generate_contents(startpath)
    )
1177062
def from_file(cls, file_path, compressed=False, encoded=False):
        """"""
        file_id = '.'.join(path.basename(file_path).split('.')[:-1])
        file_format = file_path.split('.')[-1]
        content = cls(file_id, file_format, compressed, encoded)
        content.file_exists = True
        content._location = path.dirname(file_path)
        return content
690758
def data_from_file(file):
    """"""
    fp = wave.open(file, 'r')
    data = fp.readframes(fp.getnframes())
    channels = fp.getnchannels()
    freq = fp.getframerate()
    bits = fp.getsampwidth()

    # Unpack bytes -- warning currently only tested with 16 bit wavefiles. 32
    # bit not supported.
    data = struct.unpack(('%sh' % fp.getnframes()) * channels, data)

    # Only use first channel
    channel1 = []
    n = 0
    for d in data:
        if n % channels == 0:
            channel1.append(d)
        n += 1
    fp.close()
    return (channel1, freq, bits)
267381
def _gen_txt_data(self, f):
		
		reader = iter(f)

		for line_num, line in enumerate(reader):
			if line_num == 0 and self.has_header:
				continue

			datum = line.rstrip('\r\n')

			yield datum, line_num+1