/*
given a list of meta file paths, create a combined stream that reads all the
records via the csv parser
188703
function createMetaRecordStream(metaFilePaths, types) {
  const metaRecordStream = combinedStream.create();

  metaFilePaths.forEach((metaFilePath) => {
    metaRecordStream.append( (next) => {
      logger.info( `Loading ${path.basename(metaFilePath)} records from ${path.dirname(metaFilePath)}` );
      next(createOneMetaRecordStream(metaFilePath));
    });
  });

  return metaRecordStream;
}
1076450
function visitAll(state, paths, cwd, one, done) {
  var expected = paths.length
  var actual = -1
  var result = []

  paths.forEach(each)

  next()

  function each(filePath) {
    visit(state, join(cwd || '', filePath), one, onvisit)
  }

  function onvisit(files) {
    result = result.concat(files)
    next()
  }

  function next() {
    actual++

    if (actual === expected) {
      done(result)
    }
  }
}
430761
function copyReadmeFiles() {
    const resolvedSrcReadmePaths = path.resolve(rootDir, srcReadmePaths);

    glob(resolvedSrcReadmePaths, void(0), function(error, files) {
        files.forEach((filePath) => {
        const destReadmePath = filePath.replace(/(\bsrc\b)(?!.*\1)/, destDir);
            fs.copyFileSync(filePath, destReadmePath);
        });
    });
}
298719
function createSQLiteRecordStream(dbPaths, importPlace) {
  const sqliteStream = combinedStream.create();
  const sqliteStatement = importPlace ?
    SQLiteStream.findGeoJSONByPlacetypeAndWOFId(getPlacetypes(), importPlace) :
    SQLiteStream.findGeoJSONByPlacetype(getPlacetypes());

  dbPaths.forEach((dbPath) => {
    sqliteStream.append( (next) => {
      logger.info( `Loading ${path.basename(dbPath)} database from ${path.dirname(dbPath)}` );
      next(new SQLiteStream(dbPath, sqliteStatement));
    });
  });

  return sqliteStream;
}
561555
function getMetaFilePaths(wofRoot, bundles) {
  return bundles.map((bundle) => {
    return path.join(wofRoot, 'meta', bundle);
  });
}
266091
function createFileDecoder(path, opts) {
  return fs.createReadStream(path)
    .pipe(new containers.streams.BlockDecoder(opts));
}
231751
def _parse(self):
        
        return MetadataFile(
            self.filename,
            self.raw_data,
            decoders.parse_meta(self.filename, self.raw_data)
        )
1293687
def csv_from_items(items, stream=None):
        
        items = iter(items)
        first = next(items)
        cls = first.__class__
        if stream is None:
            stream = sys.stdout
        fields = [f.name for f in attr.fields(cls)]
        writer = csv.DictWriter(stream, fieldnames=fields)
        writer.writeheader()
        writer.writerow(attr.asdict(first))
        writer.writerows((attr.asdict(x) for x in items))
567988
def iter_csv_stream(input_stream, fieldnames=None, sniff=False, *args, **kwargs):
    '''  '''
    if 'dialect' not in kwargs and sniff:
        kwargs['dialect'] = csv.Sniffer().sniff(input_stream.read(1024))
        input_stream.seek(0)
    if 'quoting' in kwargs and kwargs['quoting'] is None:
        kwargs['quoting'] = csv.QUOTE_MINIMAL
    if fieldnames:
        # read csv using dictreader
        if isinstance(fieldnames, bool):
            reader = csv.DictReader(input_stream, *args, **kwargs)
        else:
            reader = csv.DictReader(input_stream, *args, fieldnames=fieldnames, **kwargs)
        for row in reader:
            yield row
    else:
        csvreader = csv.reader(input_stream, *args, **kwargs)
        for row in csvreader:
            yield row
687073
def Read(self, meta_only=False, allowed=None, cast=False):
        """"""
        if allowed is not None and not isinstance(allowed, (list, tuple)):
            raise RuntimeError('`allowed` must be a list of str names.')

        meta = xmltodict.parse(
                open(self.filename, 'r').read()
            ).get('espa_metadata')

        # Handle bands seperately
        bands = meta.get('bands').get('band')
        del(meta['bands'])

        if not isinstance(bands, (list)):
            bands = [bands]
        meta = self.CleanDict(meta)

        # Get spatial refernce
        ras = SetProperties(RasterSet, meta)

        if allowed is not None:
            # Remove non-allowed arrays from bdict
            for k in list(self.bdict.keys()):
                if k not in allowed:
                    del(self.bdict[k])

        for i in range(len(bands)):
            info = self.GenerateBand(bands[i], meta_only=True, cast=cast)
            if allowed is not None and info.name not in allowed:
                continue
            if info.name not in self.bdict.keys() or self.bdict[info.name].data is None:
                b = self.GenerateBand(bands[i], meta_only=meta_only, cast=cast)
                self.bdict[b.name] = b
            elif cast and self.bdict[info.name].data.dtype != np.float32:
                b = self.GenerateBand(bands[i], meta_only=meta_only, cast=cast)
                self.bdict[b.name] = b
            elif not cast and self.bdict[info.name].data.dtype == np.float32:
                b = self.GenerateBand(bands[i], meta_only=meta_only, cast=cast)
                self.bdict[b.name] = b
        ras.bands = self.bdict

        if not meta_only:
            ras.validate()

        return ras
979713
def _read(self, directory, filename, session, path, name, extension, spatial, spatialReferenceID, replaceParamFile):
        
        # Set file extension property
        self.fileExtension = extension

        # Dictionary of keywords/cards and parse function names
        KEYWORDS = ('NUM_LINKS',
                    'TIME_STEP',
                    'NUM_TS',
                    'START_TIME',
                    'TS')

        # Parse file into chunks associated with keywords/cards
        with open(path, 'r') as f:
            self.name = f.readline().strip()
            chunks = pt.chunk(KEYWORDS, f)

        # Parse chunks associated with each key
        for card, chunkList in iteritems(chunks):
            # Parse each chunk in the chunk list
            for chunk in chunkList:
                schunk = chunk[0].strip().split()

                # Cases
                if card == 'NUM_LINKS':
                    # NUM_LINKS handler
                    self.numLinks = schunk[1]

                elif card == 'TIME_STEP':
                    # TIME_STEP handler
                    self.timeStepInterval = schunk[1]

                elif card == 'NUM_TS':
                    # NUM_TS handler
                    self.numTimeSteps = schunk[1]

                elif card == 'START_TIME':
                    # START_TIME handler
                    self.startTime = '%s  %s    %s  %s  %s  %s' % (
                        schunk[1],
                        schunk[2],
                        schunk[3],
                        schunk[4],
                        schunk[5],
                        schunk[6])

                elif card == 'TS':
                    # TS handler
                    for line in chunk:
                        sline = line.strip().split()
                        token = sline[0]

                        # Cases
                        if token == 'TS':
                            # Time Step line handler
                            timeStep = LinkNodeTimeStep(timeStep=sline[1])
                            timeStep.linkNodeDataset = self

                        else:
                            # Split the line
                            spLinkLine = line.strip().split()

                            # Create LinkDataset GSSHAPY object
                            linkDataset = LinkDataset()
                            linkDataset.numNodeDatasets = int(spLinkLine[0])
                            linkDataset.timeStep = timeStep
                            linkDataset.linkNodeDatasetFile = self

                            # Parse line into NodeDatasets
                            NODE_VALUE_INCREMENT = 2
                            statusIndex = 1
                            valueIndex = statusIndex + 1

                            # Parse line into node datasets
                            if linkDataset.numNodeDatasets > 0:
                                for i in range(0, linkDataset.numNodeDatasets):
                                    # Create NodeDataset GSSHAPY object
                                    nodeDataset = NodeDataset()
                                    nodeDataset.status = int(spLinkLine[statusIndex])
                                    nodeDataset.value = float(spLinkLine[valueIndex])
                                    nodeDataset.linkDataset = linkDataset
                                    nodeDataset.linkNodeDatasetFile = self

                                    # Increment to next status/value pair
                                    statusIndex += NODE_VALUE_INCREMENT
                                    valueIndex += NODE_VALUE_INCREMENT
                            else:
                                nodeDataset = NodeDataset()
                                nodeDataset.value = float(spLinkLine[1])
                                nodeDataset.linkDataset = linkDataset
                                nodeDataset.linkNodeDatasetFile = self
713331
def ReadPathInfosHistories(self, client_id, path_type, components_list):
    """"""
    results = {}

    for components in components_list:
      try:
        path_record = self.path_records[(client_id, path_type, components)]
      except KeyError:
        results[components] = []
        continue

      entries_by_ts = {}
      for ts, stat_entry in path_record.GetStatEntries():
        pi = rdf_objects.PathInfo(
            path_type=path_type,
            components=components,
            timestamp=ts,
            stat_entry=stat_entry)
        entries_by_ts[ts] = pi

      for ts, hash_entry in path_record.GetHashEntries():
        try:
          pi = entries_by_ts[ts]
        except KeyError:
          pi = rdf_objects.PathInfo(
              path_type=path_type, components=components, timestamp=ts)
          entries_by_ts[ts] = pi

        pi.hash_entry = hash_entry

      results[components] = [
          entries_by_ts[k] for k in sorted(iterkeys(entries_by_ts))
      ]

    return results
1291027
def _read_csv_lines(path):
    
    csv_file = open(path, 'r')
    csv_lines_raw = csv_file.readlines()
    csv_lines_clean = [line for line in csv_lines_raw if len(line.strip()) > 0]
    return csv_lines_clean
998018
def parse_meta_data(self, line):
        """"""
        line = line.rstrip()
        logger.debug("Parsing metadata line:{0}".format(line))
        line_info = line[2:].split('=')
        match = False

        if line_info[0] == 'fileformat':
            logger.debug("Parsing fileformat")
            try:
                self.fileformat = line_info[1]
                logger.debug("Found fileformat {0}".format(self.fileformat))
            except IndexError:
                raise SyntaxError("fileformat must have a value")

        elif line_info[0] == 'INFO':
            match = self.info_pattern.match(line)
            if not match:
                raise SyntaxError("One of the INFO lines is malformed:{0}".format(line))

            matches = [
                match.group('id'), match.group('number'),
                match.group('type'), match.group('desc')
            ]

            # extra_info is a dictionary to check the metadata about the INFO values:
            self.extra_info[matches[0]] = dict(
                zip(self.header_keys['info'][1:], matches[1:])
            )

            info_line = dict(list(zip(self.header_keys['info'],matches)))

            if len(info_line['Description'].split('Format:')) > 1:
                info_line['Format'] = [
                    info.strip() for info in info_line['Description'].split('Format:')
                ][-1]
            self.info_lines.append(info_line)

            # Store the vep columns:
            if info_line['ID'] == 'CSQ':
                self.vep_columns = info_line.get('Format', '').split('|')

            if info_line['ID'] == 'ANN':
                self.snpeff_columns = [
                    annotation.strip("' ") for annotation in
                    info_line.get('Description', '').split(':')[-1].split('|')]

            self.info_dict[match.group('id')] = line

        elif line_info[0] == 'FILTER':
            match = self.filter_pattern.match(line)
            if not match:
                raise SyntaxError("One of the FILTER lines is malformed: {0}".format(line))
            matches = [match.group('id'), match.group('desc')]
            self.filter_lines.append(dict(
                list(zip(self.header_keys['filt'],matches)))
            )
            self.filter_dict[match.group('id')] = line

        elif line_info[0] == 'contig':
            match = self.contig_pattern.match(line)
            if not match:
                print()
                raise SyntaxError("One of the contig lines is malformed: {0}".format(line))

            matches = [match.group('id'), match.group('length')]
            self.contig_lines.append(dict(
                list(zip(self.header_keys['contig'],matches)))
            )
            self.contig_dict[match.group('id')] = line

        elif line_info[0] == 'FORMAT':
            match = self.format_pattern.match(line)
            if not match:
                raise SyntaxError("One of the FORMAT lines is malformed: {0}".format(line))

            matches = [
                match.group('id'), match.group('number'),
                match.group('type'), match.group('desc')
            ]
            self.format_lines.append(dict(
                list(zip(self.header_keys['form'],matches)))
            )
            self.format_dict[match.group('id')] = line

        elif line_info[0] == 'ALT':
            match = self.alt_pattern.match(line)
            if not match:
                raise SyntaxError("One of the ALT lines is malformed: {0}".format(line))

            matches = [match.group('id'), match.group('desc')]
            self.alt_lines.append(dict(
                list(zip(self.header_keys['alt'],matches)))
            )
            self.alt_dict[match.group('id')] = line

        else:
            match = self.meta_pattern.match(line)
            if not match:
                raise SyntaxError("One of the meta data lines is malformed: {0}".format(line))

            self.other_lines.append({match.group('key'): match.group('val')})
            self.other_dict[match.group('key')] = line
1208741
def read_csv(fpath):
    
    import csv
    import utool as ut
    #csvfile = open(fpath, 'rb')
    with open(fpath, 'rb') as csvfile:
        row_iter = csv.reader(csvfile, delimiter=str(','), quotechar=str('|'))
        row_list = [ut.lmap(ut.ensure_unicode, row) for row in row_iter]
    return row_list
1040261
@SuppressWarnings("rawtypes")
    public static <E extends Exception> DataSet loadCSV(final Reader csvReader, long offset, long count, final Try.Predicate<String[], E> filter,
            final Map<String, ? extends Type> columnTypeMap) throws UncheckedIOException, E {
        N.checkArgument(offset >= 0 && count >= 0, "'offset'=%s and 'count'=%s can't be negative", offset, count);

        if (N.isNullOrEmpty(columnTypeMap)) {
            throw new IllegalArgumentException("columnTypeMap can't be null or empty");
        }

        final BufferedReader br = csvReader instanceof BufferedReader ? (BufferedReader) csvReader : Objectory.createBufferedReader(csvReader);

        try {
            List<String> tmp = new ArrayList<>();
            String line = br.readLine();
            jsonParser.readString(tmp, line);
            final String[] titles = tmp.toArray(new String[tmp.size()]);

            final int columnCount = titles.length;
            final Type<?>[] columnTypes = new Type<?>[columnCount];
            final List<String> columnNameList = new ArrayList<>(columnTypeMap.size());
            final List<List<Object>> columnList = new ArrayList<>(columnTypeMap.size());

            for (int i = 0; i < columnCount; i++) {
                if (columnTypeMap.containsKey(titles[i])) {
                    columnTypes[i] = columnTypeMap.get(titles[i]);
                    columnNameList.add(titles[i]);
                    columnList.add(new ArrayList<>());
                }
            }

            if (columnNameList.size() != columnTypeMap.size()) {
                final List<String> keys = new ArrayList<>(columnTypeMap.keySet());
                keys.removeAll(columnNameList);
                throw new AbacusException(keys + " are not included in titles: " + N.toString(titles));
            }

            final String[] strs = new String[titles.length];

            while (offset-- > 0 && br.readLine() != null) {
            }

            while (count > 0 && (line = br.readLine()) != null) {
                jsonParser.readString(strs, line);

                if (filter != null && filter.test(strs) == false) {
                    continue;
                }

                for (int i = 0, columnIndex = 0; i < columnCount; i++) {
                    if (columnTypes[i] != null) {
                        columnList.get(columnIndex++).add(columnTypes[i].valueOf(strs[i]));
                    }
                }

                count--;
            }

            return new RowDataSet(columnNameList, columnList);
        } catch (IOException e) {
            throw new UncheckedIOException(e);
        } finally {
            if (br != csvReader) {
                Objectory.recycle(br);
            }
        }
    }
711574
function parse_files(files, options, callback) {
  var nodes = {
    // root section node
    '': {
      id: '',
      type: 'section',
      children: [],
      description: '',
      short_description: '',
      href: '#',
      root: true,
      file: '',
      line: 0
    }
  };

  var reportObject = { };
  
  async.forEachSeries(files, function (file, next_file) {
    var fn = parsers[path.extname(file)];

    if (!fn) {
      next_file();
      return;
    }

    console.info('Parsing file: ' + file);
    fn(file, options, function (err, file_nodes, file_report) {
      // TODO:  fail on name clash here as well -- as we might get name clash
      //        from different parsers, or even different files
      _.extend(nodes, file_nodes);
      _.extend(reportObject, file_report);
      next_file(err);
    });
  }, function (err) {
    callback(err, nodes, reportObject);
  });
}