Reads data from disk and generates CSV files.
965860
def run(self):
        """"""
        # Try to create the directory
        if not os.path.exists(self.output):
            try:
                os.mkdir(self.output)
            except:
                print 'failed to create output directory %s' % self.output

        # Be sure it is a directory
        if not os.path.isdir(self.output):
            print 'invalid output directory %s' % self.output
            sys.exit(1)

        # Create the CSV handlers
        visitors = [
            _CompaniesCSV(self.output),
            _ActivitiesCSV(self.output),
            _ActivitiesSeenCSV(self.output),
            _QSACSV(self.output),
        ]

        # Run by each company populating the CSV files
        for path in glob.glob(os.path.join(self.input, '*.json')):
            with open(path, 'r') as f:
                try:
                    data = json.load(f, encoding='utf-8')
                except ValueError:
                    continue

                for visitor in visitors:
                    visitor.visit(data)
564842
def _write_csv(self, datasets, filename):
        
        with open('/'.join([self.output, filename]), mode='w', encoding=self.encoding) as write_file:
            writer = csv.writer(write_file, delimiter=',')
            for i, row in enumerate(datasets):
                if i == 0:
                    # header
                    writer.writerow(list(row.keys()))
                writer.writerow(list(row.values()))
153276
def csv(self, text=TEXT, sep=',', index=True, float_fmt="%.2g"):
        
        return self._data.to_csv(sep=sep, index=index, float_format=float_fmt)
1290441
def generate_csv(in_dir, out):
    
    writer = UnicodeWriter(open(out, 'wb'), delimiter=';')
    writer.writerow(('Reference ID', 'Created', 'Origin', 'Subject'))
    for cable in cables_from_source(in_dir):
        writer.writerow((cable.reference_id, cable.created, cable.origin, titlefy(cable.subject)))
507416
def main():
    """"""

    parser = argparse.ArgumentParser(description='Convert ULog to CSV')
    parser.add_argument('filename', metavar='file.ulg', help='ULog input file')

    parser.add_argument(
        '-m', '--messages', dest='messages',
        help=("Only consider given messages. Must be a comma-separated list of"
              " names, like 'sensor_combined,vehicle_gps_position'"))
    parser.add_argument('-d', '--delimiter', dest='delimiter', action='store',
                        help="Use delimiter in CSV (default is ',')", default=',')


    parser.add_argument('-o', '--output', dest='output', action='store',
                        help='Output directory (default is same as input file)',
                        metavar='DIR')

    args = parser.parse_args()

    if args.output and not os.path.isdir(args.output):
        print('Creating output directory {:}'.format(args.output))
        os.mkdir(args.output)

    convert_ulog2csv(args.filename, args.messages, args.output, args.delimiter)
62611
def main():
    """"""
    parser = argparse.ArgumentParser(description='Generate a TPIP report as a CSV file.')
    parser.add_argument('output_filename', type=str, metavar='output-file',
                        help='the output path and filename', nargs='?')
    parser.add_argument('--only', type=str, help='only parse this package')
    args = parser.parse_args()

    output_path = os.path.abspath(args.output_filename) if args.output_filename else None

    skips = []
    tpip_pkgs = []
    for pkg_name, pkg_item in sorted(pkg_resources.working_set.by_key.items()):
        if args.only and args.only not in pkg_name.lower():
            continue
        if pkg_name in EXCLUDED_PACKAGES:
            skips.append(pkg_name)
            continue
        metadata_lines = get_metadata(pkg_item)
        tpip_pkg = process_metadata(pkg_name, metadata_lines)
        tpip_pkgs.append(force_ascii_values(tpip_pkg))

    print(json.dumps(tpip_pkgs, indent=2, sort_keys=True))
    print('Parsed %s packages\nOutput to CSV: `%s`\nIgnored packages: %s' % (
        len(tpip_pkgs),
        output_path,
        ', '.join(skips),
    ))
    output_path and write_csv_file(output_path, tpip_pkgs)
1124889
def generate_csv(data):
  """"""
  print(",".join([
      "ability_id",
      "link_name",
      "link_index",
      "button_name",
      "hotkey",
      "friendly_name",
      "remap_to",
      "mismatch",
  ]))
  for ability in sorted(six.itervalues(data.abilities),
                        key=lambda a: sort_key(data, a)):
    ab_id = ability.ability_id
    if ab_id in skip_abilities or (ab_id not in data.general_abilities and
                                   ab_id not in used_abilities):
      continue

    general = ""
    if ab_id in data.general_abilities:
      general = "general"
    elif ability.remaps_to_ability_id:
      general = ability.remaps_to_ability_id

    mismatch = ""
    if ability.remaps_to_ability_id:
      def check_mismatch(ability, parent, attr):
        if getattr(ability, attr) != getattr(parent, attr):
          return "%s: %s" % (attr, getattr(ability, attr))

      parent = data.abilities[ability.remaps_to_ability_id]
      mismatch = "; ".join(filter(None, [
          check_mismatch(ability, parent, "available"),
          check_mismatch(ability, parent, "target"),
          check_mismatch(ability, parent, "allow_minimap"),
          check_mismatch(ability, parent, "allow_autocast"),
          check_mismatch(ability, parent, "is_building"),
          check_mismatch(ability, parent, "footprint_radius"),
          check_mismatch(ability, parent, "is_instant_placement"),
          check_mismatch(ability, parent, "cast_range"),
      ]))

    print(",".join(str(s) for s in [
        ability.ability_id,
        ability.link_name,
        ability.link_index,
        ability.button_name,
        ability.hotkey,
        ability.friendly_name,
        general,
        mismatch,
    ]))
326999
def generate_data(self, data_dir, tmp_dir, task_id=-1):
    """"""
    filepath_fns = {
        problem.DatasetSplit.TRAIN: self.training_filepaths,
        problem.DatasetSplit.EVAL: self.dev_filepaths,
        problem.DatasetSplit.TEST: self.test_filepaths,
    }

    # We set shuffled=True as we don't want to shuffle on disk later.
    split_paths = [(split["split"], filepath_fns[split["split"]](
        data_dir, split["shards"], shuffled=True))
                   for split in self.dataset_splits]
    all_paths = []
    for _, paths in split_paths:
      all_paths.extend(paths)

    if self.is_generate_per_split:
      for split, paths in split_paths:
        generator_utils.generate_files(
            self.generate_encoded_samples(data_dir, tmp_dir, split),
            paths,
            cycle_every_n=self.total_number_of_frames // len(paths))
    else:
      generator_utils.generate_files(
          self.generate_encoded_samples(data_dir, tmp_dir,
                                        problem.DatasetSplit.TRAIN),
          all_paths,
          cycle_every_n=self.total_number_of_frames // len(all_paths))
884292
def _create_from_csv(self):
        
       
        import aikif.dataTools.cls_datatable as cl
        fle = cl.DataTable(self.input_data, ',')
        fle.load_to_array()
        self.content['data'] = fle.arr
        
        
        lg.record_process('_create_from_csv', 'read ' + self._calc_size_stats() + ' from ' + self.input_data)
758955
def produce_csv(input, output):
    
    '''
    output.write('"model","mean","std"\n')
    for model_data in input:
        output.write('"%s",%f,%f\n' % (model_data['name'], model_data['mean'], model_data['stddev']))
    output.flush()
    output.close()
    print("Wrote as %s" % output.name)
477684
def import_csv(csv_file, **kwargs):
    """"""
    records = get_imported_data(csv_file, **kwargs)
    _check_required_columns(csv_file, records.results)
    return records
755131
def self_generate(output_filename, filename="iso3166-1.csv"):  # pragma: no cover
    
    import csv
    import re

    countries = []
    alt_codes = []
    with open(filename, "r") as csv_file:
        for row in csv.reader(csv_file):
            name = row[0].rstrip("*")
            name = re.sub(r"\(the\)", "", name)
            if name:
                countries.append((name, row[1]))
                alt_codes.append((row[1], row[2], int(row[3])))
    with open(__file__, "r") as source_file:
        contents = source_file.read()
    # Write countries.
    bits = re.match("(.*\nCOUNTRIES = \{\n)(.*?)(\n\}.*)", contents, re.DOTALL).groups()
    country_list = []
    for name, code in countries:
        name = name.replace('"', r"\"").strip()
        country_list.append('    "{code}": _("{name}"),'.format(name=name, code=code))
    content = bits[0]
    content += "\n".join(country_list)
    # Write alt codes.
    alt_bits = re.match(
        "(.*\nALT_CODES = \{\n)(.*)(\n\}.*)", bits[2], re.DOTALL
    ).groups()
    alt_list = []
    for code, code3, codenum in alt_codes:
        name = name.replace('"', r"\"").strip()
        alt_list.append(
            '    "{code}": ("{code3}", {codenum}),'.format(
                code=code, code3=code3, codenum=codenum
            )
        )
    content += alt_bits[0]
    content += "\n".join(alt_list)
    content += alt_bits[2]
    # Generate file.
    with open(output_filename, "w") as output_file:
        output_file.write(content)
    return countries
525746
def from_csv(self, input_data):
        
        reformatted_data = []
        for (i,row) in enumerate(input_data):
            if i==0:
                headers = row
            else:
                data_row = {}
                for (j,h) in enumerate(headers):
                    data_row.update({h : row[j]})
                reformatted_data.append(data_row)
        return reformatted_data
610288
def _load_objects(csv_paths, csv_positions, prefix):
  """"""
  logging.info('Loading CSVs %s from positions %s with prefix %s',
               csv_paths, csv_positions, prefix)
  objects = collections.defaultdict(list)
  for i, labels_path in enumerate(csv_paths):
    with tf.io.gfile.GFile(labels_path) as csv_f:
      if csv_positions[i] > 0:
        csv_f.seek(csv_positions[i])
      else:
        csv_f.readline()  # Drop headers
      reader = csv.reader(csv_f)
      for image_id, source, label, confidence in reader:
        if prefix and image_id[0] != prefix:
          break
        csv_positions[i] = csv_f.tell()
        image_id = int(image_id, 16)
        current_obj = _Object(label, int(float(confidence) * 10), source)
        objects[image_id].append(current_obj)
  return dict(objects)
779530
def writer(datasets, dataset_names, output_dir):
    """"""
    for dataset, dataset_name in zip(datasets, dataset_names):
        if dataset:
            filepath = output_dir + '/' + dataset_name + '.txt'
            with open(filepath, 'w+') as out_file:
                joined = '\n'.join(dataset)
                out_file.write(str(joined.encode('utf-8').decode('utf-8')))
                out_file.write('\n')
307907
def main(left_path, left_column, right_path, right_column,
         outfile, titles, join, minscore, count, warp):
    """"""
    right_file = csv.reader(open(right_path, 'r'))
    if titles:
        right_header = next(right_file)
    index = NGram((tuple(r) for r in right_file),
                  threshold=minscore,
                  warp=warp, key=lambda x: lowstrip(x[right_column]))
    left_file = csv.reader(open(left_path, 'r'))
    out = csv.writer(open(outfile, 'w'), lineterminator='\n')
    if titles:
        left_header = next(left_file)
        out.writerow(left_header + ["Rank", "Similarity"] + right_header)
    for row in left_file:
        if not row: continue # skip blank lines
        row = tuple(row)
        results = index.search(lowstrip(row[left_column]), threshold=minscore)
        if results:
            if count > 0:
                results = results[:count]
            for rank, result in enumerate(results, 1):
                out.writerow(row + (rank, result[1]) + result[0])
        elif join == "outer":
            out.writerow(row)
723815
def GenerateFileData(self):
    """"""
    # Handle chunked encoding:
    # https://www.w3.org/Protocols/rfc2616/rfc2616-sec3.html#sec3.6.1
    while 1:
      line = self.rfile.readline()
      # We do not support chunked extensions, just ignore them.
      chunk_size = int(line.split(";")[0], 16)
      if chunk_size == 0:
        break

      for chunk in self._GenerateChunk(chunk_size):
        yield chunk

      # Chunk is followed by \r\n.
      lf = self.rfile.read(2)
      if lf != "\r\n":
        raise IOError("Unable to parse chunk.")

    # Skip entity headers.
    for header in self.rfile.readline():
      if not header:
        break